diff --git a/.github/APPROVED_CONTRIBUTORS b/.github/APPROVED_CONTRIBUTORS index 23ba87df7..9f4a04a02 100644 --- a/.github/APPROVED_CONTRIBUTORS +++ b/.github/APPROVED_CONTRIBUTORS @@ -407,3 +407,5 @@ Jay-Madden issue lalyos pr Lincoln504 pr + +vimtor pr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8c980e5a..71e526c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true jobs: - build-check-test: + build-check: runs-on: ubuntu-latest steps: - name: Checkout @@ -23,12 +23,6 @@ jobs: node-version: 22 cache: npm - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep - sudo ln -s $(which fdfind) /usr/local/bin/fd - - name: Install dependencies run: npm ci --ignore-scripts @@ -55,11 +49,61 @@ jobs: - name: Check run: npm run check + # Tests are split across parallel jobs so the slowest suites do not run back to back. + test: + name: test (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: coding-agent + - name: durable + - name: other + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + + # fd and ripgrep are used by coding-agent tools tests and tui autocomplete tests. + - name: Install system dependencies + if: matrix.name != 'durable' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends fd-find ripgrep + sudo ln -s $(which fdfind) /usr/local/bin/fd + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Build + run: npm run build + - name: Build the pi-env daemon + if: matrix.name == 'other' run: npm run build:daemon --workspace packages/env - name: Test - run: npm test + env: + SUITE: ${{ matrix.name }} + run: | + if [ "$SUITE" != other ]; then + npm run test --workspace "packages/$SUITE" + exit 0 + fi + # Every other workspace, so new packages are tested without editing this workflow. + args=() + for dir in packages/*/; do + case "$dir" in packages/coding-agent/ | packages/durable/) continue ;; esac + args+=(--workspace "${dir%/}") + done + npm run test:scripts + npm run test --if-present "${args[@]}" mcp-conformance: runs-on: ubuntu-latest diff --git a/.pi/extensions/prompt-url-widget.ts b/.pi/extensions/prompt-url-widget.ts index 20a41a509..6901663b1 100644 --- a/.pi/extensions/prompt-url-widget.ts +++ b/.pi/extensions/prompt-url-widget.ts @@ -218,7 +218,7 @@ export default function promptUrlWidgetExtension(pi: ExtensionAPI) { }; pi.on("before_agent_start", async (event, ctx) => { - if (!ctx.hasUI) return; + if (!ctx.hasUI || event.messageType !== "user") return; const match = extractPromptMatch(event.prompt); if (!match) { return; diff --git a/.pi/skills/add-llm-provider.md b/.pi/skills/add-llm-provider.md index 884d3da7f..d137e548e 100644 --- a/.pi/skills/add-llm-provider.md +++ b/.pi/skills/add-llm-provider.md @@ -23,11 +23,11 @@ Most providers reuse an existing API such as `openai-completions`; skip this ste - Register the lazy wrapper in the builtin API list in `src/compat.ts`. - Add `export type` re-exports in `src/index.ts` for option types that should be available from the root entry. The `./api/*` subpath export in `package.json` already covers the new module. -## 3. Model Generation (`packages/ai/scripts/generate-models.ts`) +## 3. Model Catalog (`packages/ai/catalog/providers/.ts`) -- Add logic to fetch/parse models from the provider source. -- Map to the standardized `Model` interface. -- Regenerate. This writes the stable catalog wrapper `src/providers/.models.ts` (exporting `_MODELS`) that the provider factory imports, and updates `src/models.generated.ts`. +- Add a provider definition with `defineProvider()` and register it in `catalog/providers/index.ts`. Read `packages/ai/catalog/README.md` first. +- For a models.dev provider, use `modelsDevModels()`; set compat defaults per API in `compat` and provider-specific behavior in `finalize`. +- Regenerate with `npm run generate-models` in `packages/ai`. This writes the stable catalog wrapper `src/providers/.models.ts` (exporting `_MODELS`) that the provider factory imports, and updates `src/models.generated.ts`. ## 4. Provider Factory and Auth (`packages/ai/src/providers/`) diff --git a/AGENTS.md b/AGENTS.md index 190d394f1..43a9cf8c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,12 +25,27 @@ - Always ask before removing functionality or code that appears intentional. - Do not preserve backward compatibility unless the user asks for it. - Never hardcode key checks (e.g. `matchesKey(keyData, "ctrl+x")`). Add defaults to `DEFAULT_EDITOR_KEYBINDINGS` or `DEFAULT_APP_KEYBINDINGS` so they stay configurable. -- Never modify `packages/ai/src/models.generated.ts` directly; update `packages/ai/scripts/generate-models.ts` instead, then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes. The generated model data in `packages/ai/src/providers/data/` is gitignored: never stage it, and don't look for it in git history. +- Never modify `packages/ai/src/models.generated.ts` directly; update the catalog definitions in `packages/ai/catalog/` instead (see its README), then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes. The generated model data in `packages/ai/src/providers/data/` is gitignored: never stage it, and don't look for it in git history. + +## Chord v2 Code Style + +Binding for `packages/chord/src/next/` and its tests, when writing and when reviewing. The model is pi-durable's source. + +- Files: one concept per file, named after it; no one-helper files, no grab-bag files. First line: a comment saying what the file is for. Order: imports, exported types, then the public entry point first and its helpers below it in the order they are used; constants right above their first user. A blank line between top-level declarations. Split files by concept, never by size: one concept may take a long file (up to about 1,000 lines is fine); a second concept in a file is the reason to split. +- Functions: every export has a one-line doc comment saying what it does or guarantees. Inline comments explain a non-obvious why (cost, race, decided rule), never what the next line does. Short functions with one job (about 40 lines needs a reason; one `switch` over a tagged union is fine). No boolean flag parameters that switch behaviour: two functions or a tagged value. Never write the same lines twice. Context last: required args, then `context`/`ctx` (always required, as in Durable; optional only on error-reporting paths), then optional args. +- Types and checks: tagged unions for every state and result. Casts only where TypeScript cannot express a decided invariant, each with a short why. Every remaining runtime check names the trust boundary it serves (wire input, schema-checked params and arguments, declaration policy the types cannot express). Same-host values are plain data, immutable by convention: never copied, frozen or walked to enforce it; freeze only Chord-owned metadata when a decision says so. No defensive code against broken adapters, callers or our own modules. +- Cost and memory: no hot-path work a decision did not ask for (walks, copies, compilations, per-call allocations). The function doing counted work counts it through its context; call sites never count for others. Caches are `WeakMap`s keyed by their owner, each with a GC test; no module-level strong caches. +- Names and API: the design's vocabulary (`params`, `binding`, `principal key`), no invented synonyms; decided `ChordError` codes and reasons. Export only what a later layer or app needs. No dev or test modes, no unrequested options. +- Tests: one test file per source file, one `describe` per mechanism, the layer spec's test list case by case; cost guards through the in-memory telemetry adapter; GC tests for every cache and binding. Every slice also adds one or more runnable examples in `packages/chord/test/next/examples/NN-topic.ts`, one topic each, numbered across slices, written as references for Chord's users (Durable's `test/examples/` format: header with what it shows and how to run it, commented steps, printed results). +- Slices: Chord v2 is built in vertical slices (`packages/chord/docs/next/slices.md`). Implementers and reviewers read the slice spec, the feature map in `slices.md` and the whole of `packages/chord/docs/next/design.md` in full, and check the work against all three; slice specs cite design sections for orientation, not as a reading limit. +- Deferred work: anything left to a later slice goes into the section "Carry-over" of `packages/chord/docs/next/slices.md` under that slice, in the same change; a slice's spec and build include every entry of its section. +- Before reporting: re-read every changed file top to bottom as a stranger would; each file must be understandable without jumping around. Then `npm run check` and the layer's tests. ## Commands - After code changes (not docs): `npm run check` (full output, no tail). Fix all errors, warnings, and infos before committing. Does not run tests. -- Never run `npm run build` or `npm test` unless requested by the user. +- Run `npm run build` whenever something needs built output (`dist/`): tests or benchmarks that import built packages, Durable's workerd suite. Never use it to type check or lint: that is `npm run check`, the preferred check. +- Never run `npm test` unless requested by the user. - Never run the full vitest suite directly: it includes e2e tests that activate when endpoint/auth env vars are present. For all non-e2e tests, run `./test.sh` from the repo root. Otherwise run specific tests from the package root: - Vitest: `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/specific.test.ts` - `packages/tui` (`node:test`): `node --test test/specific.test.ts` diff --git a/biome.json b/biome.json index a27fb2366..2c5b8a2fa 100644 --- a/biome.json +++ b/biome.json @@ -29,6 +29,7 @@ "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*.ts", "packages/*/test/**/*.ts", + "packages/ai/catalog/**/*.ts", "packages/coding-agent/examples/**/*.ts", "packages/agent/examples/**/*.ts", "!**/node_modules/**/*", diff --git a/nix/model-catalog.json b/nix/model-catalog.json index 3c31c2cca..3b4641ee5 100644 --- a/nix/model-catalog.json +++ b/nix/model-catalog.json @@ -1,3 +1,3 @@ { - "revision": "sha256-7fe1b41feaf43a7aad4bc832e6b306c7ced56e6d41bbc724b74e91eeacff6edf" + "revision": "sha256-f4dbabe79cda4b38aee4a761b23fd5eaf526ac2e28fed3c9ee86e37ca0e5d71e" } diff --git a/package-lock.json b/package-lock.json index 62665c62f..a0749b93b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,18 +13,15 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, @@ -32,26 +29,6 @@ "node": ">=22.19.0" } }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@anthropic-ai/sdk": { "version": "0.129.0", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz", @@ -73,6 +50,116 @@ } } }, + "node_modules/@ata-validator/native-darwin-arm64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-arm64/-/native-darwin-arm64-1.61.3.tgz", + "integrity": "sha512-DC+iozKe5mNpsmT4MLuMxG2pIeGPVR3YBYuBmops0AF5UG6eSxg/X0484hW/Nmc2dil54+b4N41hIhS5U9rExg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-darwin-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-x64/-/native-darwin-x64-1.61.3.tgz", + "integrity": "sha512-q63+k48mWlvrNaUXmtGVxp2NWTl3wXlkkWkc2q4MerHzjIfeFBca30tfTRcyVXtDXPCj8BRJkrI69aJJXk+pLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-gnu/-/native-linux-arm64-gnu-1.61.3.tgz", + "integrity": "sha512-D9OG2oKJd/hW9F8IFoqIfK+SDfyryzSGr5YRVJB39LwvvWNNfh6pxsYpjthEjwqGuJdoBtYIqOUFnjgBqDWh9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-musl/-/native-linux-arm64-musl-1.61.3.tgz", + "integrity": "sha512-L1NhRd5/i6v7VQw28gR5L53+Zlo09CnYfXBapgGUC9I3uNppOw56/dihU5cPrd5T/k9VeljosllOP+dWA9cLpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-gnu/-/native-linux-x64-gnu-1.61.3.tgz", + "integrity": "sha512-Dlp+hcvQqKkAQEyrr7RohCbYoLYHdSjeB5puU3Ju92kSeH4MaF451q0VqwOm3StaMQB3LrLA03GDb4/G3b5gsA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-musl/-/native-linux-x64-musl-1.61.3.tgz", + "integrity": "sha512-2TutVxyhzbpUyBFzuGNp4fjyC3Suhi1idFfbk883tPdfcmp2/j0Q5QT5ySk4LPLGrNzV2haqjESZvha/QKctkw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-win32-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-win32-x64/-/native-win32-x64-1.61.3.tgz", + "integrity": "sha512-Ac2fqD70b/lQSuLDdz03xiGPhn2FK7oKXdQ7JbCq9ZYiJBbymhFazMKkRaI+3Y5XKIV3zalAfNGzUJAzMUsXow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@aws-sdk/client-bedrock-runtime": { "version": "3.1127.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1127.0.tgz", @@ -593,6 +680,13 @@ "dev": true, "license": "MIT OR Apache-2.0" }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "dev": true, + "license": "MIT" + }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", @@ -620,26 +714,26 @@ } }, "node_modules/@cloudflare/vitest-plugin": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.4.tgz", - "integrity": "sha512-nXNIlPiLvYFx/AH4/Bb/yntVkeloYcf1+tBi3l6Gp3TDCDtLxXAugrUtO7eTOWbHP6UIx5f9npy05OrACbZ5wg==", + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.10.tgz", + "integrity": "sha512-F6klOeT/Wx0gKvZ4zpYVBixP6TjcT4OD/L/RJK3Yq3SwRmd+v96/y037xUEsvGytdUisL9X5Jw3ADuNdvwBHdg==", "dev": true, "license": "MIT", "dependencies": { "cjs-module-lexer": "1.2.3", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", - "wrangler": "4.151.1", - "zod": "4.6.5" + "miniflare": "5.20261023.2-beta", + "wrangler": "4.155.0", + "zod": "4.6.6" }, "peerDependencies": { "vitest": "^4.1.0 || ^5.0.0" } }, "node_modules/@cloudflare/vitest-plugin/node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "dev": true, "license": "MIT", "funding": { @@ -647,9 +741,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261012.1.tgz", - "integrity": "sha512-dkTL9aY3V4D40b+jBpGPCistWUPFQ/M61wlWi6i1o8MvHFFgEw7EQkcaYEv6Yd3bkPDD/3TFT4hRsMb2xVZSwQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261023.1.tgz", + "integrity": "sha512-lsEx0i7nR0PNG/erUFFHmQmiidEhLfDE5uQmAL9421/y4/ycuxDDOSaxu3VGGYFFPK5sduyUeMmjIlX4CwOBFg==", "cpu": [ "x64" ], @@ -664,9 +758,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261012.1.tgz", - "integrity": "sha512-2/nVGcKXjNNI+DhxKSepTKvFng9H7aN2WXv6Zu67iqgI6JrxypNt5kTPv4ZHosxrcfyKMci/2RfN7GIrN0BitQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261023.1.tgz", + "integrity": "sha512-5e79fHgabdDD8EgmjOhJ1S+xMsf6DbC7fL7IgTBizMnzOp0P2isDn7pYmUxI9AJZE9FVQ4RUilH/I36YNg+GQQ==", "cpu": [ "arm64" ], @@ -681,9 +775,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261012.1.tgz", - "integrity": "sha512-uMe7ONHnLxStD1Ye0YFHBO36uj4cStv5P53rW+5GVWXFT8BRjqVixlH0IqCAHe1Pmcwyd1+7fVoGde89MZSCJg==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261023.1.tgz", + "integrity": "sha512-VHQ6jKFSihjR2+x1buby8XTdBLLJztVnYiB7OLOTYO8aDXB+c1Rd2410PdgFuFYc4AgFEk6PCJLyCLL5L1KBnw==", "cpu": [ "x64" ], @@ -698,9 +792,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261012.1.tgz", - "integrity": "sha512-I0VMdcXk7ygG5vYDce0ePTBFdnBXC0i9WiBY5mWiv0iTinbfea6/rHqSQ7B1UMpjgrYVs9E0m4bZ9ufsk5GCFA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261023.1.tgz", + "integrity": "sha512-ilpRHzfDN8YFRM5CLRKuFw40x/LJBS89yVdhdQmxehlVDuBqjKpJB++zPFbTgKRRFaNupgnXc8RHktXAIhgCmQ==", "cpu": [ "arm64" ], @@ -715,9 +809,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261012.1.tgz", - "integrity": "sha512-c5hQvNx574UEoknaCFvMa/NdLrrYiALNbe6sx1okW8uOiRYWxrFuA8WpXFnBJ5XJypm0TEAHYaUTGv3rRLpY7Q==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261023.1.tgz", + "integrity": "sha512-279WLVjNsMycAwr0EZNiNEf0l2SY6NUtsN80NwZzbTHzakmiN+WSx4w38muxPm+PAU0oSPPAPXKVD+H6Hya8xw==", "cpu": [ "x64" ], @@ -851,6 +945,10 @@ "resolved": "packages/mcp", "link": true }, + "node_modules/@earendil-works/pi-schema": { + "resolved": "packages/schema", + "link": true + }, "node_modules/@earendil-works/pi-telemetry": { "resolved": "packages/telemetry", "link": true @@ -877,7 +975,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -894,7 +991,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -911,7 +1007,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -928,7 +1023,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -945,7 +1039,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -962,7 +1055,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -979,7 +1071,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -996,7 +1087,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1013,7 +1103,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1030,7 +1119,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1047,7 +1135,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1064,7 +1151,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1081,7 +1167,6 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1098,7 +1183,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1115,7 +1199,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1132,7 +1215,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1149,7 +1231,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1166,7 +1247,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1183,7 +1263,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1200,7 +1279,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1217,7 +1295,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1234,7 +1311,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1251,7 +1327,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1268,7 +1343,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1285,7 +1359,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1302,7 +1375,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1454,6 +1526,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1471,6 +1546,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1488,6 +1566,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1505,6 +1586,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1522,6 +1606,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1539,6 +1626,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1556,6 +1646,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1573,6 +1666,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1590,6 +1686,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1613,6 +1712,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1636,6 +1738,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1659,6 +1764,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1682,6 +1790,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1705,6 +1816,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1728,6 +1842,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1751,6 +1868,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1863,6 +1983,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1891,6 +2017,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@oh-my-pi/omptype": { + "version": "18.13.3", + "resolved": "https://registry.npmjs.org/@oh-my-pi/omptype/-/omptype-18.13.3.tgz", + "integrity": "sha512-ZLi+kb3ff554n2ndH4iEviVRWHI1t78LZu7qf/NSXkYf4STcl6vEnJ5A2FBhfUaTLVk/HRs09LzyFnySOkwpvQ==", + "dev": true, + "license": "MIT", + "engines": { + "bun": ">=1.4.4", + "node": ">=20" + } + }, "node_modules/@oxc-project/types": { "version": "0.154.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.154.0.tgz", @@ -1901,12 +2038,6 @@ "url": "https://github.com/sponsors/oxc-project" } }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -2441,21 +2572,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/lodash": { - "version": "4.17.24", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.24.tgz", - "integrity": "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, "node_modules/@types/ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", @@ -2995,6 +3111,41 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3014,6 +3165,36 @@ "node": ">=12" } }, + "node_modules/ata-validator": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/ata-validator/-/ata-validator-1.61.3.tgz", + "integrity": "sha512-6o3V8lpiVQzzHe6blbQZj9saD0Qxmhdv2TTyO5BgPcUWeP3EZj572f9w1b93E6c+BmUce0QOf7GvwOffRUf3tw==", + "dev": true, + "license": "MIT", + "bin": { + "ata": "bin/ata.js" + }, + "engines": { + "node": ">=20.0.0" + }, + "optionalDependencies": { + "@ata-validator/native-darwin-arm64": "1.61.3", + "@ata-validator/native-darwin-x64": "1.61.3", + "@ata-validator/native-linux-arm64-gnu": "1.61.3", + "@ata-validator/native-linux-arm64-musl": "1.61.3", + "@ata-validator/native-linux-x64-gnu": "1.61.3", + "@ata-validator/native-linux-x64-musl": "1.61.3", + "@ata-validator/native-win32-x64": "1.61.3" + }, + "peerDependencies": { + "yaml": "^2.0.0" + }, + "peerDependenciesMeta": { + "yaml": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3201,22 +3382,6 @@ "dev": true, "license": "MIT" }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -3366,16 +3531,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/es-module-lexer": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", @@ -3387,7 +3542,7 @@ "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -3471,12 +3626,36 @@ "node": ">=18" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-sha256": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3537,13 +3716,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "license": "ISC" - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3559,16 +3731,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gaxios": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", @@ -3616,59 +3778,6 @@ "dev": true, "license": "MIT" }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.21", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", - "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/google-auth-library": { "version": "10.6.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", @@ -3710,19 +3819,6 @@ "node": ">=18" } }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", @@ -3826,18 +3922,6 @@ "node": ">= 4" } }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3852,32 +3936,6 @@ "dev": true, "license": "ISC" }, - "node_modules/interpret": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-core-module": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", - "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3915,6 +3973,13 @@ "node": ">=16" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -4219,12 +4284,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -4288,16 +4347,16 @@ } }, "node_modules/miniflare": { - "version": "5.20261012.1-beta", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261012.1-beta.tgz", - "integrity": "sha512-8Canhy6g9+vAhqMc0o80YphqYIwpiA3aAp8aX+uDXU7J/qbUeoZzcII9c2V1P4uT/eHD4SB93qet1o1bRY3BuQ==", + "version": "5.20261023.2-beta", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261023.2-beta.tgz", + "integrity": "sha512-sXpZloQnIUe4raXXdwwqd2ioRHox9a/1B4nB2/p4JWKDwGuVgEK4DugCNgIeQf0iVuSgBItdYpodJe4iwY6SrQ==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.5", - "undici": "7.29.1", - "workerd": "1.20261012.1", + "undici": "7.30.1", + "workerd": "1.20261023.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, @@ -4306,9 +4365,9 @@ } }, "node_modules/miniflare/node_modules/undici": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", - "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", "dev": true, "license": "MIT", "engines": { @@ -4316,12 +4375,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" @@ -4493,16 +4553,6 @@ "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", "license": "MIT" }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4512,13 +4562,6 @@ "node": ">=8" } }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -4545,10 +4588,6 @@ "resolved": "packages/coding-agent/examples/extensions/gondolin", "link": true }, - "node_modules/pi-extension-sandbox": { - "resolved": "packages/coding-agent/examples/extensions/sandbox", - "link": true - }, "node_modules/pi-extension-with-deps": { "resolved": "packages/coding-agent/examples/extensions/with-deps", "link": true @@ -4757,38 +4796,14 @@ "node": ">= 6" } }, - "node_modules/rechoir": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", - "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "dev": true, - "dependencies": { - "resolve": "^1.1.6" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/resolve": { - "version": "1.22.12", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", - "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", "dev": true, "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "node": ">=0.10.0" } }, "node_modules/retry": { @@ -4943,53 +4958,6 @@ "node": ">=8" } }, - "node_modules/shell-quote": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.12.0.tgz", - "integrity": "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/shelljs": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", - "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "glob": "^7.0.0", - "interpret": "^1.0.0", - "rechoir": "^0.6.2" - }, - "bin": { - "shjs": "bin/shjs" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/shx": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", - "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.3", - "shelljs": "^0.8.5" - }, - "bin": { - "shx": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -5134,19 +5102,6 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/tar-fs": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", @@ -5294,9 +5249,9 @@ } }, "node_modules/undici": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", - "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "license": "MIT", "engines": { "node": ">=22.19.0" @@ -5563,9 +5518,9 @@ } }, "node_modules/workerd": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261012.1.tgz", - "integrity": "sha512-tEHoMPsMga0seVjxfsR8Pd3f8IRqYwkq++dTUKI5Gdgj6zuSDluYwisydq+dn90SsIACwBquJUQ1nBpcOqaWTA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261023.1.tgz", + "integrity": "sha512-NeofJFyaYNpqKpFRJW3ApuonaerPd6/JWlfTtLDYf2A6ygqrDEFnYgEXmc0hhOxqnJsSM+kXJN6VebPojzRrsg==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -5576,17 +5531,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20261012.1", - "@cloudflare/workerd-darwin-arm64": "1.20261012.1", - "@cloudflare/workerd-linux-64": "1.20261012.1", - "@cloudflare/workerd-linux-arm64": "1.20261012.1", - "@cloudflare/workerd-windows-64": "1.20261012.1" + "@cloudflare/workerd-darwin-64": "1.20261023.1", + "@cloudflare/workerd-darwin-arm64": "1.20261023.1", + "@cloudflare/workerd-linux-64": "1.20261023.1", + "@cloudflare/workerd-linux-arm64": "1.20261023.1", + "@cloudflare/workerd-windows-64": "1.20261023.1" } }, "node_modules/wrangler": { - "version": "4.151.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.151.1.tgz", - "integrity": "sha512-OAsa9UAHZshM6mq2PXcVoyxXgWThAanZMBc/IirtqEWd/BauRtmUKzAniOu/pU0pd3pxKuMZ2vsbF4eo6RiNDA==", + "version": "4.155.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.155.0.tgz", + "integrity": "sha512-gBKtAg2e0abQDvXPRQu611YMTd/cfAaJTOLAs/89tTxFxEeUyC260MRJRK+m3gLmQjDYWFwJv+ErwuJ/GuALpA==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { @@ -5594,10 +5549,10 @@ "@cloudflare/unenv-preset": "2.16.2", "blake3-wasm": "2.1.5", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", + "miniflare": "5.20261023.2-beta", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20261012.1" + "workerd": "1.20261023.1" }, "bin": { "cf-wrangler": "bin/cf-wrangler.js", @@ -5611,7 +5566,7 @@ "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^5.20261006.1" + "@cloudflare/workers-types": "^5.20261023.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -5688,9 +5643,10 @@ } }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -5698,11 +5654,11 @@ }, "packages/agent": { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "devDependencies": { "@types/node": "22.19.19", @@ -5721,19 +5677,19 @@ }, "packages/ai": { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -5741,6 +5697,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" }, "engines": { @@ -5809,31 +5766,41 @@ }, "packages/chord": { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" }, "devDependencies": { "@types/ws": "8.18.2", - "shx": "0.3.4", "vitest": "4.1.12", "ws": "8.21.0" }, "engines": { "node": ">=22.19.0" + }, + "peerDependencies": { + "esbuild": "0.28.2", + "ws": "8.21.0" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + }, + "ws": { + "optional": true + } } }, "packages/codemode": { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5842,14 +5809,14 @@ }, "packages/coding-agent": { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -5860,12 +5827,12 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "bin": { @@ -5878,7 +5845,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5887,32 +5853,25 @@ }, "packages/coding-agent/examples/extensions/custom-provider-anthropic": { "name": "pi-extension-custom-provider-anthropic", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "0.52.0" } }, "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo": { "name": "pi-extension-custom-provider-gitlab-duo", - "version": "1.4.0" + "version": "1.5.0" }, "packages/coding-agent/examples/extensions/gondolin": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } }, - "packages/coding-agent/examples/extensions/sandbox": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } - }, "packages/coding-agent/examples/extensions/with-deps": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "2.1.3" }, @@ -5966,21 +5925,20 @@ }, "packages/durable": { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5989,14 +5947,13 @@ }, "packages/env": { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -6005,36 +5962,52 @@ }, "packages/evals": { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } }, "packages/mcp": { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { "node": ">=22.19.0" } }, + "packages/schema": { + "name": "@earendil-works/pi-schema", + "version": "1.5.0", + "license": "MIT", + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", + "typebox": "1.3.27", + "vitest": "4.1.12", + "zod": "4.6.7" + }, + "engines": { + "node": ">=22.19.0" + } + }, "packages/telemetry": { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@types/node": "22.19.19", @@ -6046,7 +6019,7 @@ }, "packages/tui": { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/package.json b/package.json index 04462a1d5..daa608b30 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,16 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "scripts": { "clean": "npm run clean --workspaces", - "build": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", - "build:offline": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build:offline": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", "build:native:darwin": "npm --prefix packages/tui run build:native:darwin", "build:native:linux": "npm --prefix packages/tui run build:native:linux", "build:native:win32": "npm --prefix packages/tui run build:native:win32", - "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && npm run check:browser-smoke", + "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && tsc --noEmit -p packages/coding-agent/examples/tsconfig.json && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", "check:runtime-deps": "node scripts/check-runtime-deps.mjs", @@ -57,14 +56,12 @@ "prepare": "husky" }, "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 2ae23784d..9e80e2abf 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Breaking Changes + +- `AgentTool` takes any JSON Schema object as `parameters` (`AgentTool`) and types `params` and `prepareArguments` through `@earendil-works/pi-schema` (`AgentToolParams`): pi-schema's builders give their static type, TypeBox schemas give TypeBox's static type for the common kinds (String, Number, Integer, Boolean, Null, Literal, Array, Object with optional and readonly properties, Union, Record, Unsafe, Unknown, Any; other kinds such as Enum, Tuple and Intersect give `unknown`), other schemas give `unknown`, `AgentTool` stays `any`. `outputSchema` is any JSON Schema object. `typebox` is no longer a dependency. Migration: build parameters with pi-schema (`obj({ path: str() })` instead of `Type.Object({ path: Type.String() })`), or keep TypeBox schemas (narrow `params` yourself only where they use other kinds). + ### Fixed - Fixed `streamProxy()` re-parsing the accumulated tool-call arguments on every delta and keeping their raw text on the partial content block ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/agent/README.md b/packages/agent/README.md index 38625ce11..a6fa8256b 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -455,18 +455,20 @@ const agent = new Agent({ ## Tools -Define tools using `AgentTool`: +Define tools using `AgentTool`. `parameters` is any JSON Schema object; built with `@earendil-works/pi-schema`'s builders, it also types `params` (other schemas give `unknown`, `AgentTool` gives `any`): ```typescript -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; -const readFileTool: AgentTool = { +const readFileParameters = obj({ + path: str({ description: "File path" }), +}); + +const readFileTool: AgentTool = { name: "read_file", label: "Read File", // For UI display description: "Read a file's contents", - parameters: Type.Object({ - path: Type.String({ description: "File path" }), - }), + parameters: readFileParameters, // Override execution mode for this tool (optional). // "sequential" forces the entire batch to run one at a time. // "parallel" allows concurrent execution with other tool calls. diff --git a/packages/agent/examples/mcp-codemode/tools.ts b/packages/agent/examples/mcp-codemode/tools.ts index 37e358c65..06d678670 100644 --- a/packages/agent/examples/mcp-codemode/tools.ts +++ b/packages/agent/examples/mcp-codemode/tools.ts @@ -18,7 +18,7 @@ import { renderDeclarations, } from "@earendil-works/pi-codemode"; import { type McpClient, toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; /** Provider tool names are limited to 64 characters of `[A-Za-z0-9_-]`. */ function toToolName(prefix: string, name: string): string { @@ -45,12 +45,12 @@ export async function createMcpTools(client: McpClient, prefix: string): Promise label: tool.title ?? tool.name, description: tool.description ?? tool.title ?? `MCP tool ${tool.name}`, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {}, - }), - outputSchema: tool.outputSchema ? Type.Unsafe>(tool.outputSchema) : undefined, + }, + outputSchema: tool.outputSchema, execute: async (_toolCallId, params, signal, onUpdate) => { const result = await client.callTool(tool.name, params as Record, { signal, @@ -104,9 +104,11 @@ export function createNestedToolRunner( export const CODEMODE_TOOL_NAME = "codemode"; -const codemodeParameters = Type.Object({ - code: Type.String({ description: "JavaScript source. Top-level await and return work." }), -}); +const codemodeParameters = obj( + { code: str({ description: "JavaScript source. Top-level await and return work." }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); /** * A tool that runs model-written JavaScript in the codemode sandbox, where `tools.(args)` @@ -121,7 +123,6 @@ export function createCodemodeTool( const sandboxTools: CodemodeTool[] = callable.map((tool) => ({ name: tool.name, description: tool.description, - // TypeBox schemas are JSON Schema. inputSchema: tool.parameters as CodemodeJsonSchema, // What the script receives: the structured result if the tool declares one, else its text. outputSchema: (tool.outputSchema as CodemodeJsonSchema | undefined) ?? { type: "string" }, diff --git a/packages/agent/package.json b/packages/agent/package.json index d1231d95d..e00a8e1e7 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "type": "module", "main": "./dist/index.js", @@ -17,14 +17,14 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run build" }, "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "keywords": [ "ai", diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index dbfba74a0..24bdd7531 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -14,7 +14,7 @@ import type { TranscriptContext, Usage, } from "@earendil-works/pi-ai"; -import type { Static, TSchema } from "typebox"; +import type { Static } from "@earendil-works/pi-schema"; /** * Stream function used by the agent loop. `Models.streamSimple` satisfies @@ -462,27 +462,39 @@ export interface AgentToolCallOutcome { */ export type AgentToolUpdateCallback = (partialResult: AgentToolResult) => void; -/** Tool definition used by the agent runtime. */ -export interface AgentTool extends Tool { +/** + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `AgentTool`. + */ +export type AgentToolParams = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + +/** + * Tool definition used by the agent runtime. `parameters` is any JSON Schema object; see {@link AgentToolParams} for + * the type `execute` receives. + */ +export interface AgentTool extends Tool { /** Human-readable label for UI display. */ label: string; /** * Optional compatibility shim for raw tool-call arguments before schema validation. * Must return an object that matches `TParameters`. */ - prepareArguments?: (args: unknown) => Static; + prepareArguments?: (args: unknown) => AgentToolParams; /** * JSON Schema of `structuredContent` in successful results. Tools that declare it should always * set `structuredContent`. */ - outputSchema?: TSchema; + outputSchema?: object; /** * Execute the tool call. Throw on failure, or return a result with `isError: true`; do not only * describe the failure in `content`. */ execute: ( toolCallId: string, - params: Static, + params: AgentToolParams, signal?: AbortSignal, onUpdate?: AgentToolUpdateCallback, ) => Promise>; diff --git a/packages/agent/test/agent-loop.test.ts b/packages/agent/test/agent-loop.test.ts index 4a9386a13..b3ea8c259 100644 --- a/packages/agent/test/agent-loop.test.ts +++ b/packages/agent/test/agent-loop.test.ts @@ -6,7 +6,7 @@ import { type ToolResultMessage, type UserMessage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentLoop, agentLoopContinue, runAgentLoop, runToolCall } from "../src/agent-loop.ts"; import { setDefaultStreamFn } from "../src/index.ts"; @@ -303,7 +303,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should handle tool calls and results", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const toolUsage = { input: 1, @@ -400,7 +400,7 @@ describe("agentLoop with AgentMessage", () => { // #10549 it("records how long execute() took on the tool result, excluding hooks", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const tool: AgentTool = { name: "echo", @@ -454,7 +454,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should not execute tool calls from a length-truncated assistant message", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -526,7 +526,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should execute mutated beforeToolCall args without revalidation", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: Array = []; const tool: AgentTool = { name: "echo", @@ -587,8 +587,8 @@ describe("agentLoop with AgentMessage", () => { }); it("should prepare tool arguments for validation", async () => { - const replaceSchema = Type.Object({ oldText: Type.String(), newText: Type.String() }); - const toolSchema = Type.Object({ edits: Type.Array(replaceSchema) }); + const replaceSchema = obj({ oldText: str(), newText: str() }); + const toolSchema = obj({ edits: arr(replaceSchema) }); const executed: Array> = []; const tool: AgentTool = { name: "edit", @@ -666,7 +666,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should emit tool_execution_end in completion order but persist tool results in source order", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -760,7 +760,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should inject queued messages after all tool calls complete", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -865,7 +865,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when a tool has executionMode=sequential even with default parallel config", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -947,7 +947,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when one of multiple tools has executionMode=sequential", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executionOrder: string[] = []; let releaseSlow: (() => void) | undefined; const slowDone = new Promise((resolve) => { @@ -1033,7 +1033,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow parallel execution when all tools have executionMode=parallel", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -1106,7 +1106,7 @@ describe("agentLoop with AgentMessage", () => { }); it("runs finishTurn after tool-result messages and before turn_end", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1215,7 +1215,7 @@ describe("agentLoop with AgentMessage", () => { ); it("action:end skips queue polling and next-turn preparation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1305,7 +1305,7 @@ describe("agentLoop with AgentMessage", () => { }); it("lets a natural tool-result request satisfy continuation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1501,7 +1501,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should use prepareNextTurn snapshot before continuing", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1578,7 +1578,7 @@ describe("agentLoop with AgentMessage", () => { }); it("picks up steering queued during prepareNextTurn before the next request", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1631,7 +1631,7 @@ describe("agentLoop with AgentMessage", () => { }); it("action:end receives finalized turn context and stops before queue polling", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1730,7 +1730,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a tool batch when every tool result sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1781,7 +1781,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a blocked tool call when beforeToolCall sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let executed = false; const tool: AgentTool = { name: "echo", @@ -1839,7 +1839,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after a mixed batch with one terminating blocked call", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1897,7 +1897,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after parallel tool calls when not all tool results terminate", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1962,7 +1962,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow afterToolCall to mark a tool batch as terminating", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -2131,13 +2131,13 @@ describe("agentLoopContinue with AgentMessage", () => { }); describe("runToolCall", () => { - const echoSchema = Type.Object({ value: Type.String() }); + const echoSchema = obj({ value: str() }); const echo: AgentTool = { name: "echo", label: "Echo", description: "Echo tool", parameters: echoSchema, - outputSchema: Type.Object({ value: Type.String() }), + outputSchema: obj({ value: str() }), async execute(_toolCallId, params, _signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "partial" }], details: {} }); return { @@ -2151,7 +2151,7 @@ describe("runToolCall", () => { name: "failing", label: "Failing", description: "Returns an error result", - parameters: Type.Object({}), + parameters: obj({}), async execute() { return { content: [{ type: "text", text: "bad" }], details: { partial: true }, isError: true }; }, diff --git a/packages/agent/test/agent-tool-typebox.test.ts b/packages/agent/test/agent-tool-typebox.test.ts new file mode 100644 index 000000000..1c859b7ae --- /dev/null +++ b/packages/agent/test/agent-tool-typebox.test.ts @@ -0,0 +1,65 @@ +// AgentTool params typed from a TypeBox-built parameter schema. pi-agent-core does not depend on TypeBox: the schema +// below has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { describe, expectTypeOf, it } from "vitest"; +import type { AgentTool } from "../src/types.ts"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TLiteral { + "~kind": "Literal"; + const: Value; +} + +interface TUnion { + "~kind": "Union"; + anyOf: Branches; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ command: TString; mode: TOptional, TLiteral<"deep">]>> }> = { + "~kind": "Object", + type: "object", + properties: { + command: { "~kind": "String", type: "string" }, + mode: { + "~kind": "Union", + anyOf: [ + { "~kind": "Literal", const: "fast" }, + { "~kind": "Literal", const: "deep" }, + ], + "~optional": true, + }, + }, + required: ["command"], +}; + +type Params = { command: string; mode?: "fast" | "deep" }; + +describe("AgentTool params of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool: AgentTool = { + name: "bash", + label: "Bash", + description: "Runs a command", + parameters, + async execute(_toolCallId, params) { + expectTypeOf(params).toEqualTypeOf(); + return { content: [], details: undefined }; + }, + }; + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + }); +}); diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 6fad6109f..9c8bddc39 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -7,7 +7,7 @@ import { toToolDeclaration, type UserMessage, } from "@earendil-works/pi-ai/compat"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { Agent, @@ -49,7 +49,7 @@ function createTool(name: string): AgentTool { name, label: name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: name }], details: {} }), }; } @@ -146,7 +146,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -200,7 +200,7 @@ describe("Agent", () => { expect(update).toEqual({ role: "system", content: "", - toolsAdded: [{ name: "second", description: "second tool", parameters: Type.Object({}) }], + toolsAdded: [{ name: "second", description: "second tool", parameters: obj({}) }], toolsRemoved: [{ name: "first" }], timestamp: expect.any(Number), }); @@ -214,7 +214,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -239,7 +239,7 @@ describe("Agent", () => { role: "system", content: "", sections: { skills: "x" }, - toolsAdded: [{ name: "echo", description: "Echo input", parameters: Type.Object({}) }], + toolsAdded: [{ name: "echo", description: "Echo input", parameters: obj({}) }], timestamp: 1, }); }); @@ -284,7 +284,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -471,7 +471,7 @@ describe("Agent", () => { }); it("should ignore tool updates after the tool execution settles", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); let delayedUpdate: AgentToolUpdateCallback<{ status: string }> | undefined; const events: AgentEvent[] = []; const unhandledRejections: unknown[] = []; @@ -536,7 +536,7 @@ describe("Agent", () => { }); it("should ignore a settled parallel tool update while another tool is still running", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const slowStarted = createDeferred(); const settledToolEnded = createDeferred(); const releaseSlow = createDeferred(); @@ -858,7 +858,7 @@ describe("Agent", () => { }); it("keeps legacy prepareNextTurn signal callback behavior", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -899,7 +899,7 @@ describe("Agent", () => { }); it("forwards finishTurn through AgentOptions with the active abort signal", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", diff --git a/packages/agent/test/utils/calculate.ts b/packages/agent/test/utils/calculate.ts index 0a4fefc58..333cd5a0e 100644 --- a/packages/agent/test/utils/calculate.ts +++ b/packages/agent/test/utils/calculate.ts @@ -1,5 +1,5 @@ import type { Usage } from "@earendil-works/pi-ai"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface CalculateResult extends AgentToolResult { @@ -16,9 +16,11 @@ export function calculate(expression: string): CalculateResult { } } -const calculateSchema = Type.Object({ - expression: Type.String({ description: "The mathematical expression to evaluate" }), -}); +const calculateSchema = obj( + { expression: str({ description: "The mathematical expression to evaluate" }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); type CalculateParams = Static; diff --git a/packages/agent/test/utils/get-current-time.ts b/packages/agent/test/utils/get-current-time.ts index a83a82af3..11996cc41 100644 --- a/packages/agent/test/utils/get-current-time.ts +++ b/packages/agent/test/utils/get-current-time.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, opt, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface GetCurrentTimeResult extends AgentToolResult<{ utcTimestamp: number }> {} @@ -27,11 +27,11 @@ export async function getCurrentTime(timezone?: string): Promise; diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9ea77c63b..1823bff41 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,18 +2,54 @@ ## [Unreleased] +### Fixed + +- Fixed OpenCode Go Claude Haiku 5.5 failing with a 400 (`Invalid signature in thinking block ... bound to a different conversation`) after the tool list, system prompt, or earlier history changed, which left the session failing on every retry. OpenCode Go and Zen Claude 5.5 models, Opus 5, and Fable 5.1 now use per-message effort and send `thinking.block_binding: drop_block`, like the Anthropic provider ([#10954](https://github.com/earendil-works/pi/issues/10954)) +- Fixed Anthropic non-strict tool schemas dropping root `$defs` and `definitions`, which left property `$ref`s pointing at nothing. Claude then produced wrong arguments for such tools (common in MCP servers), for example a JSON string instead of an object, and some Anthropic-compatible gateways rejected the request outright ([#9557](https://github.com/earendil-works/pi/issues/9557)) + +## [1.5.0] - 2026-11-02 + ### Breaking Changes +- The main entry no longer re-exports TypeBox (`Type`, `Static`, `TSchema`), and `typebox` is no longer a dependency. `Tool.parameters` is any JSON Schema object (`Tool`): `@earendil-works/pi-schema`'s builders, which also type the arguments through pi-schema's `Static`, TypeBox schemas, or plain objects. `validateToolCall()`/`validateToolArguments()` validate with pi-schema (`compile(schema, { coerce: true }).parse()`): every schema gets the same coercion, the superset of TypeBox's `Value.Convert` rules (which only applied to TypeBox-built schemas) and pi-ai's own, so plain JSON Schema tools (MCP tools) now also get, for example, `null` to `"null"` for a required string, `"3.5"` to 3 for an integer and a single value wrapped into an array. The error text lists pi-schema's messages (`- edits.0.oldText: missing`, `- limit: expected number, got string`) instead of TypeBox's. `StringEnum()` returns a pi-schema `Schema` with the same JSON (`{ type: "string", enum }`). The model and compat schemas (`providers/model-schema`, `providers/compat-schema`) are built with pi-schema, with the same JSON Schema, and their types come from pi-schema's `Static`; `ProviderCompat` is exported. Migration: + + ```ts + // Before + import { Type, type Static, StringEnum } from "@earendil-works/pi-ai"; + const params = Type.Object({ path: Type.String(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // After: pi-schema builders (closed objects; pass { open: true } to allow other properties) + import { StringEnum } from "@earendil-works/pi-ai"; + import { obj, type Static, str } from "@earendil-works/pi-schema"; + const params = obj({ path: str(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // Or keep TypeBox, imported from typebox itself (its schemas are JSON Schema) + import { Type, type Static } from "typebox"; + ``` +- Tool arguments are validated as JSON Schema only; nothing runs through TypeBox's own validator any more, so TypeBox features outside JSON Schema no longer apply to tool arguments: + - `Type.Refine()` refinements are not run: arguments they would reject are accepted. + - `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every JSON argument (TypeBox converted to them). + - Codecs (`Type.Codec()`) do not decode: the tool receives the encoded JSON value. + - Formats registered with TypeBox's `Format.Set()` are not checked; the standard formats plus `url` and `json-pointer-uri-fragment` are. + - Property names are names, not regular expressions: `"a.b"` no longer also converts the argument `axb`. + - `additionalProperties` converts only undeclared properties; TypeBox also converted declared ones with it when an object had more than one property. +- A `$ref` that names nothing (a missing definition, an external document) rejects only arguments at its place (`- t: unresolved reference #/definitions/Missing`) instead of failing every call of the tool, as TypeBox's rejection did before. - `appendGrammarToolInputJsonDelta()` (`api/constrained-sampling`) takes the input delta instead of the whole input so far and no longer closes the input; close it with the new `closeGrammarToolInputJson()` ([#9265](https://github.com/earendil-works/pi/issues/9265)) ### Added - Added optional `openBrowserOnConfirm` and `copyCodeOnConfirm` to `device_code` auth events and `OAuthDeviceCodeInfo`. Both default to `true`; set them to `false` to stop clients from offering to open the verification page or copy the code ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) - Added `StreamingJsonParser`, an incremental, tolerant parser for JSON text that arrives in pieces (each piece is read once; snapshots of the partial value never change once returned), and `ToolCallArgumentsStream`, which every provider now uses for streamed tool-call arguments; its `append()` also returns the changes each piece made (`JsonChange`: a value set at a path, or text appended to a string), for writers that keep their own copy ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Added GPT-6.1 Terra to the OpenAI, Azure OpenAI Responses, and OpenAI Codex providers with official pricing ($1 input, $0.05 cached input, $1.25 cache writes, $5 output, long-context tier above 272k), tool search, and mid-conversation effort changes. Unlike GPT-6.1 Sol it supports `off` (reasoning effort `none`); Codex serves it with a 500K context window +- Added `Usage.cost.totalFromProvider` to distinguish provider-reported billed totals from catalog estimates. Missing or `false` means an estimate; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added `responseModel` to pi-messages terminal events, preserving the concrete model reported by providers such as Radius when a virtual model is requested ([#11265](https://github.com/earendil-works/pi/pull/11265)) ### Changed - The faux provider streams partial tool-call arguments, so `toolcall_delta` partials show growing arguments as real providers do ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- The built-in model catalog is generated from models.dev's v2 catalog, and its per-provider rules moved from `scripts/generate-models.ts` into one TypeScript definition per provider under `catalog/`. Models that models.dev types as realtime or image models (`gpt-realtime-2.1`, `gemini-3.1-flash-live-preview`, `gemini-3.1-flash-lite-image`) are no longer offered as chat models, and models.dev's newer OpenAI and Google chat models (`chat-latest`, `gpt-5.6-cyber`, `gemini-robotics-er-2-preview`) are now included ### Fixed @@ -22,6 +58,20 @@ - Fixed streamed tool-call arguments being re-parsed in full on every delta, which made CPU time and memory grow quadratically with the size of a tool call (a 90 KB `write` on Anthropic peaked at about 140 MB of heap; a 500 KB one took minutes). Every provider and the pi-messages client now read each delta once, and keep the raw argument text and their other stream state (`partialJson`, `partialArgs`, `index`, `customInput`, `streamIndex`, Bedrock's encrypted reasoning bytes) off the content blocks, so it no longer reaches partial messages or transcripts of failed streams ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed grammar (custom) tool input streaming copying the whole input so far on every delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed bounded error texts (telemetry span messages, truncated provider error bodies, pi-messages diagnostics, Radius gateway errors) keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed Kimi K3.1 Pro and K3.1 Pro Highspeed on Kimi Coding, Moonshot, and OpenCode Go using Kimi K2 request settings. They now send `reasoning_effort` (low/high/max), return earlier reasoning, accept tools added mid-conversation, and report Moonshot rates on Kimi Coding. Kimi Coding `k3-256k` now reports Kimi K3 rates. +- Fixed OAuth logins opening server-supplied URLs that are not `http:` or `https:` URLs. Radius now checks its browser authorization endpoint and device verification URI, and GitHub Copilot, Kimi Coding, and Meta share one check +- Fixed pi-messages streams accepting a `contentIndex` that is not the next or an existing content block, such as `__proto__` or a huge index +- Fixed `pi-ai` CLI auth file being readable by other users. It is now written with mode `0600` +- Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention +- Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) +- Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing Claude's earlier thinking after each tool call. They used the Chat Completions API, where pi dropped the thinking signature, and OpenRouter then ignored the unsigned thinking. They now use the Anthropic Messages API with adaptive thinking, like the versioned Claude models ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/README.md b/packages/ai/README.md index 37fbcb7e2..7af343eab 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -103,15 +103,16 @@ Unified LLM API with provider collections, automatic auth resolution, token and npm install @earendil-works/pi-ai ``` -TypeBox exports are re-exported from `@earendil-works/pi-ai`: `Type`, `Static`, and `TSchema`. +Tool parameters are JSON Schema. The examples build them with `@earendil-works/pi-schema`, which also types the arguments (`npm install @earendil-works/pi-schema`); TypeBox schemas and plain JSON Schema objects work as well. ## Quick Start You build a `Models` collection of providers and stream through it. The quickest start registers every built-in provider; apps that care about bundle size register individual providers instead (see [Provider Factories](#provider-factories) and [Bundling and Tree Shaking](#bundling-and-tree-shaking)). ```typescript -import { Type, type Context, type Tool } from '@earendil-works/pi-ai'; +import { type Context, type Tool } from '@earendil-works/pi-ai'; import { builtinModels } from '@earendil-works/pi-ai/providers/all'; +import { obj, opt, str } from '@earendil-works/pi-schema'; // A Models collection with every built-in provider registered const models = builtinModels(); @@ -119,12 +120,12 @@ const models = builtinModels(); // Sync lookup against the collection const model = models.getModel('openai', 'gpt-4o-mini')!; -// Define tools with TypeBox schemas for type safety and validation +// Define tools with JSON Schema; pi-schema's builders also type the arguments const tools: Tool[] = [{ name: 'get_time', description: 'Get the current time', - parameters: Type.Object({ - timezone: Type.Optional(Type.String({ description: 'Optional timezone (e.g., America/New_York)' })) + parameters: obj({ + timezone: opt(str({ description: 'Optional timezone (e.g., America/New_York)' })) }) }]; @@ -538,34 +539,35 @@ Amazon Bedrock resolves ambient AWS credentials (`AWS_PROFILE`, access key pairs ## Tools -Tools enable LLMs to interact with external systems. This library uses TypeBox schemas for type-safe tool definitions with automatic validation using TypeBox's built-in validator and value conversion utilities. TypeBox schemas can be serialized and deserialized as plain JSON, making them ideal for distributed systems. +Tools enable LLMs to interact with external systems. A tool's `parameters` is a JSON Schema object: built with `@earendil-works/pi-schema`'s builders (which also type the arguments through its `Static`), with TypeBox, or written by hand. `validateToolCall()` and `validateToolArguments()` validate with pi-schema, coercing loosely typed arguments first (numeric strings, `"true"`, a single value for an array, `null` for an omitted optional property). Schemas are plain JSON, so they serialize with the rest of a context. ### Defining Tools ```typescript -import { Type, type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { arr, obj, str } from '@earendil-works/pi-schema'; -// Define tool parameters with TypeBox +// Define tool parameters with pi-schema's builders const weatherTool: Tool = { name: 'get_weather', description: 'Get current weather for a location', - parameters: Type.Object({ - location: Type.String({ description: 'City name or coordinates' }), + parameters: obj({ + location: str({ description: 'City name or coordinates' }), units: StringEnum(['celsius', 'fahrenheit'], { default: 'celsius' }) }) }; -// Note: For Google API compatibility, use StringEnum helper instead of Type.Enum -// Type.Enum generates anyOf/const patterns that Google doesn't support +// Note: StringEnum gives { type: "string", enum: [...] }, which every provider accepts, +// including Google's, which rejects anyOf/const patterns const bookMeetingTool: Tool = { name: 'book_meeting', description: 'Schedule a meeting', - parameters: Type.Object({ - title: Type.String({ minLength: 1 }), - startTime: Type.String({ format: 'date-time' }), - endTime: Type.String({ format: 'date-time' }), - attendees: Type.Array(Type.String({ format: 'email' }), { minItems: 1 }) + parameters: obj({ + title: str({ minLength: 1 }), + startTime: str({ format: 'date-time' }), + endTime: str({ format: 'date-time' }), + attendees: arr(str({ format: 'email' }), { minItems: 1 }) }) }; ``` @@ -578,10 +580,7 @@ Tools can opt in to provider-side constrained sampling. For JSON-schema tools, ` const strictTool: Tool = { name: 'edit_file', description: 'Edit a file', - parameters: Type.Object({ - path: Type.String(), - content: Type.String() - }, { additionalProperties: false }), + parameters: obj({ path: str(), content: str() }), // obj() is closed: additionalProperties: false constrainedSampling: { type: 'json_schema', strict: 'prefer' } }; ``` @@ -592,9 +591,7 @@ Strict JSON-schema constrained sampling is supported for OpenAI, Anthropic, supp const patchTool: Tool = { name: 'apply_patch', description: 'Apply a patch', - parameters: Type.Object({ - input: Type.String() - }, { additionalProperties: false }), + parameters: obj({ input: str() }), constrainedSampling: { type: 'grammar', variants: { @@ -698,7 +695,7 @@ for await (const event of s) { ### Validating Tool Arguments -When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools: +When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools. It coerces loosely typed arguments with pi-schema (`compile(schema, { coerce: true }).parse()`), returns the coerced arguments, and throws an error listing pi-schema's errors (`path: message`) followed by the arguments it received: ```typescript import { validateToolCall, type Tool } from '@earendil-works/pi-ai'; @@ -1726,8 +1723,8 @@ models.setProvider(openaiProvider()); Rules: -- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without TypeBox, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. -- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports TypeBox and schema validation. Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. +- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without schema validation, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. +- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports schema validation (pi-schema). Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. - `@earendil-works/pi-ai/providers/` imports that provider's catalog and lazy API wrapper only. - `@earendil-works/pi-ai/providers/all` imports every built-in provider factory and all catalogs. Use it only when you want the full built-in set. - With code splitting, provider SDKs stay in lazy chunks and load on first request. @@ -1928,12 +1925,12 @@ Create a new API implementation file (for example `bedrock-converse-stream.ts`) Add a lazy wrapper `src/api/.lazy.ts` (`Api()` via `lazyApi()`) so providers can reference the implementation without importing its SDK. Add any root-level `export type` re-exports in `src/index.ts` that should remain available from `@earendil-works/pi-ai`. -#### 3. Model Generation (`scripts/generate-models.ts`) +#### 3. Model Catalog (`catalog/providers/.ts`) -- Add logic to fetch and parse models from the provider's source (e.g., models.dev API) +- Add a provider definition and register it in `catalog/providers/index.ts`; see `catalog/README.md` for the build order and helpers - Map chat/tool-capable provider data to `Model`, image-generation data to `ImageModel`, and models.dev `type: "decision"` entries to `ClassifierModel`; hydration groups the ignored `src/providers/data/.json` values by API while stable `src/providers/.models.ts` wrappers derive exact model/API types directly from those JSON keys - Keep model ids unique within each provider and model type; emit separate entries when an upstream model supports multiple operations -- Handle provider-specific quirks (pricing format, capability flags, model ID transformations) +- Handle provider-specific quirks (pricing format, capability flags, model ID transformations) in the provider definition #### 4. Provider Factory (`src/providers/.ts`) diff --git a/packages/ai/bedrock-provider.d.ts b/packages/ai/bedrock-provider.d.ts deleted file mode 100644 index a66eabee6..000000000 --- a/packages/ai/bedrock-provider.d.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./dist/bedrock-provider.js"; diff --git a/packages/ai/bedrock-provider.js b/packages/ai/bedrock-provider.js deleted file mode 100644 index a66eabee6..000000000 --- a/packages/ai/bedrock-provider.js +++ /dev/null @@ -1 +0,0 @@ -export * from "./dist/bedrock-provider.js"; diff --git a/packages/ai/catalog/README.md b/packages/ai/catalog/README.md new file mode 100644 index 000000000..24f3641a4 --- /dev/null +++ b/packages/ai/catalog/README.md @@ -0,0 +1,47 @@ +# Model catalog + +`node catalog/generate.ts` (`npm run generate-models`) fetches upstream catalogs and writes the built-in model catalog: + +- `src/providers/data/.json`: model values, grouped by API (gitignored) +- `src/providers/.models.ts` and `src/models.generated.ts`: typed wrappers (committed) +- with `--json-output `: the public catalog JSON served by pi.dev + +## Layout + +| Path | Contents | +| --- | --- | +| `sources/` | Upstream fetchers. `models-dev.ts` normalizes the [models.dev v2](https://models.dev/experimental/v2.0/api.json) catalog; OpenRouter, Vercel AI Gateway, NVIDIA NIM, and Radius have their own modules. All sources are fetched before anything is built. | +| `providers/.ts` | One file per built-in provider: where its models come from, its API and base URL, compat defaults, corrections, and models upstream does not list. Register new files in `providers/index.ts`. | +| `families/` | Facts about a model family that hold on every host, such as Claude effort levels or GPT-6 thinking levels. They apply to every provider's models. | +| `define.ts` | `defineProvider()` and the helpers provider files use. | +| `build.ts` | Runs the provider definitions in a fixed order. | + +## Build order + +For each provider: + +1. `models(ctx)` returns its chat models: upstream entries mapped to pi models, data corrections (limits, prices), and authored models. `ctx.providerModels(id)` returns another provider's models from this step, e.g. Azure derives from OpenAI. +2. `compat` defaults for each API are merged under each model's own compat. Then the shared rules run: models.dev effort levels for APIs that send them natively, and the rules in `families/`. +3. `finalize(model, ctx)` applies provider-specific behavior. It runs after the shared rules, so its values win. +4. Image input limits get the default resize profile, then `finalizeAll(models, ctx)` runs cross-model logic such as Anthropic's allowed fallback models. + +## Adding a provider + +Most providers are a models.dev provider served on one API: + +```ts +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "example", + models: (ctx) => + modelsDevModels(ctx, "example", { api: "openai-completions", baseUrl: "https://api.example.com/v1" }), + compat: { "openai-completions": { supportsStore: false, supportsDeveloperRole: false } }, +}); +``` + +`modelsDevModels()` keeps tool-capable chat models and maps names, input modalities, costs (including context tiers), and limits. Use its `include` and `map` options for provider-specific filtering and fields. Prefer fixing wrong upstream data in models.dev; keep local corrections small and say why they exist. + +## Checking changes + +`npm run diff:model-catalog -- [provider ...]` from the repository root generates the catalog at `HEAD` and in the worktree and prints the JSON differences. diff --git a/packages/ai/catalog/build.ts b/packages/ai/catalog/build.ts new file mode 100644 index 000000000..1a47dce05 --- /dev/null +++ b/packages/ai/catalog/build.ts @@ -0,0 +1,164 @@ +import type { + AnyModel, + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + KnownApi, + Model, + OpenAICompletionsCompat, +} from "../src/types.ts"; +import type { ApiCompatDefaults, ChatModel, ProviderContext, ProviderDefinition } from "./define.ts"; +import { mergeThinking } from "./define.ts"; +import { applyClaudeFamilyRules } from "./families/anthropic.ts"; +import { applyDeepSeekCompatRules, applyDeepSeekFamilyRules } from "./families/deepseek.ts"; +import { applyOpenAiFamilyRules } from "./families/openai.ts"; +import { getEffortThinkingLevelMap } from "./reasoning.ts"; +import type { CatalogSources } from "./sources/index.ts"; +import type { ModelsDevModel } from "./sources/models-dev.ts"; + +export interface ProviderCatalog { + chat: (Model & { type: "chat" })[]; + image: ImageModel[]; + classifier: ClassifierModel[]; +} + +// Keep the generated default no less restrictive than coding-agent's historical +// image preprocessing. Provider limits can narrow this profile, but unknown +// providers retain the cache-safe 2000px / 4.5 MiB behavior. +const DEFAULT_IMAGE_RESIZE = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 4.5 * 1024 * 1024, + jpegQuality: 80, +} as const; + +function modelKey(model: Pick): string { + return `${model.provider}:${model.id}`; +} + +function applyCompatDefaults(model: ChatModel, defaults: ApiCompatDefaults | undefined): void { + const entry = defaults?.[model.api as KnownApi] as + | Model["compat"] + | ((model: Model) => Model["compat"]) + | undefined; + if (!entry) return; + const compat = typeof entry === "function" ? entry(model) : entry; + model.compat = { ...compat, ...model.compat } as Model["compat"]; +} + +/** Whether the model's API sends models.dev effort values to the host as-is. */ +function sendsNativeEffort(model: ChatModel): boolean { + if (model.api === "anthropic-messages") { + return (model as Model<"anthropic-messages">).compat?.forceAdaptiveThinking === true; + } + if ( + model.api === "openai-responses" || + model.api === "azure-openai-responses" || + model.api === "openai-codex-responses" + ) { + return true; + } + if (model.api !== "openai-completions") return false; + const compat: OpenAICompletionsCompat = (model as Model<"openai-completions">).compat ?? {}; + return (compat.thinkingFormat ?? "openai") === "openai" && (compat.supportsReasoningEffort ?? true); +} + +function applyModelsDevEffort(model: ChatModel, source: ModelsDevModel | undefined): void { + if (!source?.reasoningOptions || !sendsNativeEffort(model)) return; + const map = getEffortThinkingLevelMap(source.reasoningOptions); + if (map) mergeThinking(model, map); +} + +function applyImageResizeDefaults(model: AnyModel): void { + if (!model.input.includes("image")) return; + const images = model.inputLimits?.images; + model.inputLimits = { + ...model.inputLimits, + images: { ...images, resize: { ...DEFAULT_IMAGE_RESIZE, ...images?.resize } }, + }; +} + +/** Build every provider's catalog from the fetched sources. */ +export function buildCatalog( + definitions: readonly ProviderDefinition[], + sources: CatalogSources, + strict: boolean, +): Record { + const definitionsById = new Map(definitions.map((definition) => [definition.id, definition])); + const modelsDevSources = new Map(); + const stageOne = new Map(); + const building = new Set(); + + function context(providerId: string): ProviderContext { + return { + providerId, + sources, + strict, + providerModels: (id) => structuredClone(buildModels(id)), + modelsDevSource: (model) => modelsDevSources.get(modelKey(model)), + recordModelsDevSource: (model, source) => { + modelsDevSources.set(modelKey(model), source); + }, + }; + } + + function buildModels(providerId: string): ChatModel[] { + const built = stageOne.get(providerId); + if (built) return built; + const definition = definitionsById.get(providerId); + if (!definition) throw new Error(`Unknown provider: ${providerId}`); + if (building.has(providerId)) throw new Error(`Provider ${providerId} depends on itself`); + building.add(providerId); + const models = definition.models?.(context(providerId)) ?? []; + building.delete(providerId); + for (const model of models) { + if (model.provider !== providerId) { + throw new Error(`${providerId} produced ${modelKey(model)} for another provider`); + } + } + stageOne.set(providerId, models); + return models; + } + + for (const definition of definitions) buildModels(definition.id); + + const catalogs: Record = {}; + for (const definition of definitions) { + const ctx = context(definition.id); + const chat: ChatModel[] = []; + const seen = new Set(); + for (const model of stageOne.get(definition.id) ?? []) { + // The first source listing a model wins. + if (seen.has(model.id)) continue; + seen.add(model.id); + chat.push(model); + } + + for (const model of chat) { + applyCompatDefaults(model, definition.compat); + applyDeepSeekCompatRules(model); + applyModelsDevEffort(model, ctx.modelsDevSource(model)); + applyOpenAiFamilyRules(model); + applyClaudeFamilyRules(model); + applyDeepSeekFamilyRules(model); + } + for (const model of chat) definition.finalize?.(model, ctx); + for (const model of chat) { + if (model.compat && Object.keys(model.compat).length === 0) delete model.compat; + applyImageResizeDefaults(model); + } + definition.finalizeAll?.(chat, ctx); + + const image = definition.images?.(ctx) ?? []; + for (const model of image) applyImageResizeDefaults(model); + const classifier = definition.classifiers?.(ctx) ?? []; + catalogs[definition.id] = { + chat: chat.map((model) => ({ ...model, type: "chat" as const })), + image, + classifier, + }; + } + return catalogs; +} diff --git a/packages/ai/catalog/define.ts b/packages/ai/catalog/define.ts new file mode 100644 index 000000000..71517c6e1 --- /dev/null +++ b/packages/ai/catalog/define.ts @@ -0,0 +1,142 @@ +import type { + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + KnownApi, + Model, + ModelInputLimits, + ThinkingLevelMap, +} from "../src/types.ts"; +import type { CatalogSources } from "./sources/index.ts"; +import { type ModelsDevModel, modelsDevChatModels, modelsDevCost } from "./sources/models-dev.ts"; + +export type ChatModel = Model; +export type ModelCompat = NonNullable["compat"]>; + +/** + * Compat defaults per API. They sit under each model's own compat, so model + * values win. A function receives the model to vary defaults by model. + */ +export type ApiCompatDefaults = { + [TApi in KnownApi]?: ModelCompat | ((model: Model) => ModelCompat); +}; + +/** Passed to every provider hook. */ +export interface ProviderContext { + /** The provider being built. */ + readonly providerId: string; + readonly sources: CatalogSources; + /** Fail instead of degrading when upstream data is unexpected. */ + readonly strict: boolean; + /** Copies of another provider's chat models before shared rules run. */ + providerModels(providerId: string): ChatModel[]; + /** The models.dev entry a model was built from, if it was recorded. */ + modelsDevSource(model: ChatModel): ModelsDevModel | undefined; + /** Record the models.dev entry so shared rules can use its reasoning metadata. */ + recordModelsDevSource(model: ChatModel, source: ModelsDevModel): void; +} + +/** + * One built-in provider's catalog. The build runs, in order: + * 1. `models()`: upstream models plus authored ones, with data corrections. + * 2. `compat` defaults and the shared model-family rules in `families/`. + * 3. `finalize()`: provider-specific behavior that must win over shared rules. + * 4. Shared normalization (image input limits), then `finalizeAll()`. + */ +export interface ProviderDefinition { + id: string; + models?(ctx: ProviderContext): ChatModel[]; + images?(ctx: ProviderContext): ImageModel[]; + classifiers?(ctx: ProviderContext): ClassifierModel[]; + compat?: ApiCompatDefaults; + finalize?(model: ChatModel, ctx: ProviderContext): void; + finalizeAll?(models: ChatModel[], ctx: ProviderContext): void; +} + +export function defineProvider(definition: ProviderDefinition): ProviderDefinition { + return definition; +} + +/** Shallow-merge compat flags into a model. */ +export function mergeCompat(model: Model, compat: ModelCompat): void { + model.compat = { ...model.compat, ...compat } as ModelCompat; +} + +/** Shallow-merge thinking level mappings into a model. Later values win. */ +export function mergeThinking(model: ChatModel, map: ThinkingLevelMap): void { + model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map }; +} + +/** Provider input limits for image-capable models. Limits the model already carries take precedence. */ +export function withInputLimits(model: ChatModel, limits: ModelInputLimits): void { + if (!model.input.includes("image")) return; + model.inputLimits = { + ...limits, + ...model.inputLimits, + images: { ...limits.images, ...model.inputLimits?.images }, + }; +} + +/** Add models only when upstream has not listed them (yet). */ +export function addMissing(models: ChatModel[], additions: readonly ChatModel[]): ChatModel[] { + for (const addition of additions) { + if (!models.some((model) => model.id === addition.id)) models.push(addition); + } + return models; +} + +export interface ModelsDevOptions { + api: TApi; + baseUrl: string; + compat?: ModelCompat; + headers?: Record; + /** Only models models.dev lists on this API protocol (e.g. "responses"). */ + protocol?: string; + /** Skip models models.dev marks deprecated. */ + skipDeprecated?: boolean; + /** Expose models.dev effort levels through the shared rules. Defaults to true. */ + recordReasoning?: boolean; + /** Fallback limits when models.dev has none. Defaults to 4096 each. */ + defaultContextWindow?: number; + defaultMaxTokens?: number; + /** Extra filter. Models without tool support are always skipped. */ + include?(source: ModelsDevModel): boolean; + /** Adjust or replace the mapped model. Return undefined to skip it. */ + map?(model: Model, source: ModelsDevModel): ChatModel | undefined; +} + +/** Map one models.dev provider's tool-capable chat models onto pi models. */ +export function modelsDevModels( + ctx: ProviderContext, + sourceProvider: string, + options: ModelsDevOptions, +): ChatModel[] { + const models: ChatModel[] = []; + for (const source of modelsDevChatModels(ctx.sources.modelsDev, sourceProvider)) { + if (!source.toolCall) continue; + if (options.protocol && !source.apis.some((api) => api.protocol === options.protocol)) continue; + if (options.skipDeprecated && source.status === "deprecated") continue; + if (options.include && !options.include(source)) continue; + const base: Model = { + id: source.id, + name: source.name || source.id, + api: options.api, + provider: ctx.providerId, + baseUrl: options.baseUrl, + reasoning: source.reasoning, + input: source.imageInput ? ["text", "image"] : ["text"], + cost: modelsDevCost(source.cost), + contextWindow: source.contextWindow || options.defaultContextWindow || 4096, + maxTokens: source.maxTokens || options.defaultMaxTokens || 4096, + ...(options.headers ? { headers: { ...options.headers } } : {}), + ...(options.compat ? { compat: { ...options.compat } } : {}), + }; + const model = options.map ? options.map(base, source) : base; + if (!model) continue; + if (options.recordReasoning !== false) ctx.recordModelsDevSource(model, source); + models.push(model); + } + return models; +} diff --git a/packages/ai/catalog/families/anthropic.ts b/packages/ai/catalog/families/anthropic.ts new file mode 100644 index 000000000..f9d6d8a64 --- /dev/null +++ b/packages/ai/catalog/families/anthropic.ts @@ -0,0 +1,134 @@ +// Anthropic Claude models: facts that hold on every host that serves them. +// Ids appear dashed (claude-opus-4-8), dotted (claude-opus-4.8), prefixed +// (anthropic/claude-opus-4.8), and dated (claude-opus-5-20260723). + +import type { Model, ThinkingLevelMap } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; + +function stripPrefix(modelId: string): string { + return modelId.toLowerCase().replace(/^~?anthropic\//, ""); +} + +/** Models that accept per-message `output_config` effort changes. */ +export function supportsClaudeMidConvoEffort(modelId: string): boolean { + const id = stripPrefix(modelId); + return ( + /^claude-opus-(?:5|5[.-]5)(?:-\d{8})?$/.test(id) || + /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(id) || + /^claude-(?:fable|mythos)-5(?:[.-]1)(?:-\d{8})?$/.test(id) + ); +} + +// Claude models that accept `thinking: {type: "disabled"}` alongside per-message effort. +// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 reject it. +// https://platform.claude.com/docs/en/build-with-claude/thinking#configuring-thinking +const DISABLED_THINKING_MODELS = new Set(["claude-opus-5", "claude-haiku-5-5", "claude-haiku-5.5"]); + +/** Models that accept system messages and tool changes after the conversation starts. */ +export function supportsClaudeMidConvoSystemMessages(modelId: string): boolean { + return ( + /^claude-opus-(?:4[.-]8|5(?:[.-]5)?)(?:-\d{8})?$/.test(modelId) || + /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(modelId) || + /^claude-(?:fable|mythos)-5(?:[.-]1)?(?:-\d{8})?$/.test(modelId) + ); +} + +/** Enable per-message effort changes. Models that cannot disable thinking with it lose "off". */ +export function enableClaudeMidConvoEffort(model: Model<"anthropic-messages">): void { + mergeCompat(model, { supportsMidConvoEffort: true }); + if (!DISABLED_THINKING_MODELS.has(stripPrefix(model.id).replace(/-\d{8}$/, ""))) { + mergeThinking(model, { off: null }); + } +} + +function idIncludesAny(modelId: string, fragments: readonly string[]): boolean { + return fragments.some((fragment) => modelId.includes(fragment)); +} + +const ADAPTIVE_THINKING = [ + "opus-4-6", + "opus-4.6", + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-4-6", + "sonnet-4.6", + "sonnet-5", + "sonnet.5", + "haiku-5", + "haiku.5", + "fable-5", + "mythos-5", +]; +/** + * OpenRouter's `~anthropic/claude-*-latest` aliases follow the newest model of each family, + * all of which use adaptive thinking. + */ +function isClaudeLatestAlias(modelId: string): boolean { + return /^~anthropic\/claude-[a-z]+-latest$/.test(modelId); +} + +const TEMPERATURE_UNSUPPORTED = [ + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-5-5", + "sonnet-5.5", + "haiku-5-5", + "haiku-5.5", +]; +const MAX_EFFORT_ONLY = ["opus-4-6", "opus-4.6", "sonnet-4-6", "sonnet-4.6"]; +const XHIGH_AND_MAX_EFFORT = [ + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-5", + "sonnet.5", + "haiku-5", + "haiku.5", +]; + +/** + * Adaptive thinking effort levels (per Anthropic adaptive thinking docs): + * "max" is available on all adaptive-thinking Claude models; "xhigh" only on + * Opus 4.7/4.8/5, Sonnet 5, Haiku 5.5, and Fable 5. + */ +export function applyClaudeFamilyRules(model: ChatModel): void { + if (idIncludesAny(model.id, MAX_EFFORT_ONLY)) mergeThinking(model, { max: "max" }); + if (idIncludesAny(model.id, XHIGH_AND_MAX_EFFORT)) mergeThinking(model, { xhigh: "xhigh", max: "max" }); + if (model.id.includes("fable-5")) mergeThinking(model, { off: null, xhigh: "xhigh", max: "max" }); + if (model.api !== "anthropic-messages") return; + const messagesModel = model as Model<"anthropic-messages">; + if (isClaudeLatestAlias(model.id) || idIncludesAny(model.id, ADAPTIVE_THINKING)) { + mergeCompat(messagesModel, { forceAdaptiveThinking: true }); + } + if (idIncludesAny(model.id.toLowerCase(), TEMPERATURE_UNSUPPORTED)) { + mergeCompat(messagesModel, { supportsTemperature: false }); + } +} + +/** + * models.dev may list Opus 5.5, Sonnet 5.5, and Haiku 5.5 before their effort + * metadata is complete. Haiku 5.5 accepts disabled thinking; Opus 5.5 and + * Sonnet 5.5 reject it. + */ +export function claude55ThinkingLevels(modelId: string): ThinkingLevelMap { + return { + ...(modelId.includes("haiku") ? {} : { off: null }), + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: "xhigh", + max: "max", + }; +} diff --git a/packages/ai/catalog/families/deepseek.ts b/packages/ai/catalog/families/deepseek.ts new file mode 100644 index 000000000..630ed51c9 --- /dev/null +++ b/packages/ai/catalog/families/deepseek.ts @@ -0,0 +1,67 @@ +// DeepSeek V4 models on OpenAI-compatible hosts. + +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; + +export const DEEPSEEK_COMPAT: OpenAICompletionsCompat = { + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", +}; + +export const DEEPSEEK_V4_THINKING_LEVEL_MAP = { + minimal: null, + low: null, + medium: null, + high: "high", + max: "max", +} as const; +export const DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP = { + ...DEEPSEEK_V4_THINKING_LEVEL_MAP, + low: "low", +} as const; +// Azure Foundry rejects DeepSeek's own max effort. +const AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP = { + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: null, + max: null, +} as const; + +// Qwen Token Plan serves DeepSeek V4 through Qwen's own thinking format. +const QWEN_FORMAT_PROVIDERS = new Set(["qwen-token-plan", "qwen-token-plan-cn", "qwen-token-plan-individual"]); +// These hosts accept native reasoning efforts instead of DeepSeek's thinking toggle. +const NATIVE_EFFORT_PROVIDERS = new Set(["openrouter", "opencode"]); + +function isDeepSeekV4(model: ChatModel): model is Model<"openai-completions"> { + return model.api === "openai-completions" && model.id.includes("deepseek-v4"); +} + +/** DeepSeek V4 needs its reasoning content replayed on every host. */ +export function applyDeepSeekCompatRules(model: ChatModel): void { + if (!isDeepSeekV4(model) || QWEN_FORMAT_PROVIDERS.has(model.provider)) return; + mergeCompat( + model, + NATIVE_EFFORT_PROVIDERS.has(model.provider) + ? { requiresReasoningContentOnAssistantMessages: DEEPSEEK_COMPAT.requiresReasoningContentOnAssistantMessages } + : DEEPSEEK_COMPAT, + ); +} + +/** DeepSeek V4 effort levels when the host did not report its own. */ +export function applyDeepSeekFamilyRules(model: ChatModel): void { + if (!isDeepSeekV4(model) || model.thinkingLevelMap !== undefined) return; + if (model.provider === "openrouter") { + mergeThinking(model, { ...DEEPSEEK_V4_THINKING_LEVEL_MAP, xhigh: "xhigh", max: null }); + } else if (model.provider === "azure") { + mergeThinking(model, AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP); + } else if ( + (model.provider === "deepseek" || model.provider === "opencode" || model.provider === "opencode-go") && + model.id.includes("deepseek-v4-flash") + ) { + mergeThinking(model, DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP); + } else { + mergeThinking(model, DEEPSEEK_V4_THINKING_LEVEL_MAP); + } +} diff --git a/packages/ai/catalog/families/google.ts b/packages/ai/catalog/families/google.ts new file mode 100644 index 000000000..8e6bd9936 --- /dev/null +++ b/packages/ai/catalog/families/google.ts @@ -0,0 +1,22 @@ +// Google Gemini and Gemma models. + +import type { ThinkingLevelMap } from "../../src/types.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import type { ModelsDevReasoningOption } from "../sources/models-dev.ts"; + +function isGemma4Model(modelId: string): boolean { + return /gemma-?4/.test(modelId.toLowerCase()); +} + +/** Gemini effort levels from models.dev, with a Gemma 4 fallback. */ +export function googleThinkingLevels( + modelId: string, + reasoningOptions: readonly ModelsDevReasoningOption[] = [], +): ThinkingLevelMap | undefined { + const effortMap = getEffortThinkingLevelMap(reasoningOptions); + if (effortMap) return effortMap; + if (isGemma4Model(modelId)) { + return { off: null, minimal: "MINIMAL", low: null, medium: null, high: "HIGH" }; + } + return undefined; +} diff --git a/packages/ai/catalog/families/kimi.ts b/packages/ai/catalog/families/kimi.ts new file mode 100644 index 000000000..01e9f9067 --- /dev/null +++ b/packages/ai/catalog/families/kimi.ts @@ -0,0 +1,38 @@ +// Moonshot Kimi models, shared by Moonshot, Kimi Coding, and gateways. + +import type { ModelCost, OpenAICompletionsCompat } from "../../src/types.ts"; + +/** + * Kimi K3 and its point releases (kimi-k3.1-pro, kimi-k3.1-pro-highspeed) share + * K3's request format. https://api.moonshot.ai/v1/models reports effort levels + * and dynamic tool loading for all of them. + */ +export function isKimiK3FamilyId(modelId: string): boolean { + return /^kimi-k3(?:\.\d+)?(?:-|$)/.test(modelId); +} + +export const KIMI_K3_MAX_TOKENS = 131072; +export const KIMI_K3_COST: ModelCost = { + input: 3, + output: 15, + cacheRead: 0.3, + cacheWrite: 0, +}; + +// Kimi Chat Completions settings shared by Moonshot and Kimi Coding. +export const KIMI_OPENAI_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + thinkingFormat: "deepseek", +}; + +// Kimi K3-style models take reasoning_effort and need earlier reasoning sent back. +export const KIMI_K3_OPENAI_COMPAT: OpenAICompletionsCompat = { + ...KIMI_OPENAI_COMPAT, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", + supportsReasoningEffort: true, +}; diff --git a/packages/ai/catalog/families/openai.ts b/packages/ai/catalog/families/openai.ts new file mode 100644 index 000000000..3f3a6fa11 --- /dev/null +++ b/packages/ai/catalog/families/openai.ts @@ -0,0 +1,141 @@ +// OpenAI GPT models: facts that hold on every host that serves them. + +import type { Api, Model, ModelCost } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; +import { roundCost } from "../sources/models-dev.ts"; + +export const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; + +/** OpenAI bills prompts above 272K input tokens at 2x input and 1.5x output rates. */ +export function withOpenAiLongContextPricing(cost: ModelCost): ModelCost { + return { + ...cost, + tiers: [ + { + inputTokensAbove: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, + input: roundCost(cost.input * 2), + output: roundCost(cost.output * 1.5), + cacheRead: roundCost(cost.cacheRead * 2), + cacheWrite: roundCost(cost.cacheWrite * 2), + }, + ], + }; +} + +// Keep current OpenAI prices authoritative until models.dev and passthrough +// catalogs catch up. +// https://developers.openai.com/api/docs/pricing +export const OPENAI_STANDARD_COSTS: Record = { + "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, + "gpt-5.6-sol": { input: 4, output: 20, cacheRead: 0.4, cacheWrite: 5 }, + "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-6-astra": { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }, + "gpt-6-luna": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, + "gpt-6-sol": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-6.1-sol": { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, + "gpt-6.1-terra": { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, +}; + +/** The standard price of an OpenAI model with long-context pricing. */ +export function openAiStandardCost(id: string): ModelCost { + return withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[id]); +} + +// GPT-5.5 and older put tools loaded through tool search or additional_tools in a namespace named +// after the tool and reject replayed calls without it, so history from other models fails there +// (#11415). Like Codex, dynamic tool loading therefore starts with GPT-5.6, which does not do this. +/** Models that accept tool search and `additional_tools`. */ +export const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); +/** Models that accept mid-conversation developer messages. */ +export const OPENAI_MID_CONVO_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-pro", + "gpt-5.5", + ...OPENAI_TOOL_SEARCH_MODEL_IDS, +]); + +const RESPONSES_APIS = new Set(["openai-responses", "azure-openai-responses", "openai-codex-responses"]); + +function isResponsesApi(model: ChatModel): boolean { + return RESPONSES_APIS.has(model.api); +} + +export function supportsOpenAiXhigh(modelId: string): boolean { + return ( + modelId.includes("gpt-5.2") || + modelId.includes("gpt-5.3") || + modelId.includes("gpt-5.4") || + modelId.includes("gpt-5.5") || + modelId.includes("gpt-5.6") || + modelId.includes("gpt-6") + ); +} + +const GPT_6_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"]); + +/** Thinking levels of GPT models on any host. */ +export function applyOpenAiFamilyRules(model: ChatModel): void { + if ((model.api === "openai-responses" || model.api === "azure-openai-responses") && model.id.startsWith("gpt-5")) { + mergeThinking(model, { off: null }); + } + if (GPT_6_MODEL_IDS.has(model.id) && isResponsesApi(model)) { + mergeThinking(model, { + // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none"; GPT-6.1 Terra accepts it. All reject + // "minimal" (verified on Codex, October 2026). + off: model.id === "gpt-6-astra" || model.id === "gpt-6.1-sol" ? null : "none", + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: "xhigh", + max: "max", + }); + } + if (supportsOpenAiXhigh(model.id)) { + mergeThinking(model, { xhigh: "xhigh" }); + } + if ( + (model.id.includes("gpt-5.6") || model.id.includes("gpt-6")) && + (isResponsesApi(model) || model.api === "openai-completions") + ) { + mergeThinking(model, { max: "max" }); + } + if (model.id.endsWith("gpt-5.5-pro")) { + mergeThinking(model, { off: null, minimal: null, low: null }); + } +} + +/** + * OpenAI custom grammar tools on Responses endpoints. OpenAI rejects + * `type: "custom"` tools for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series). + */ +export function enableOpenAiGrammarTools(model: ChatModel): void { + if (!isResponsesApi(model)) return; + const match = /^gpt-(\d+)/.exec(model.id); + if (!match || Number(match[1]) < 5) return; + mergeCompat(model as Model<"openai-responses">, { supportsOpenAIGrammarTools: true }); +} + +/** + * Newer OpenAI Responses models accept developer messages after the + * conversation has started. Proxies that pass them through also pass + * `additional_tools` items through unchanged, for the models that accept them. + */ +export function enableOpenAiMidConvoSystemMessages(model: ChatModel, options: { proxy: boolean }): void { + if (model.api !== "openai-responses" && model.api !== "openai-codex-responses") return; + if (!OPENAI_MID_CONVO_MODEL_IDS.has(model.id)) return; + mergeCompat(model as Model<"openai-responses">, { + supportsMidConvoSystemMessages: true, + ...(options.proxy && OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), + }); +} diff --git a/packages/ai/catalog/generate.ts b/packages/ai/catalog/generate.ts new file mode 100644 index 000000000..718159674 --- /dev/null +++ b/packages/ai/catalog/generate.ts @@ -0,0 +1,308 @@ +#!/usr/bin/env node + +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + createModelDataManifest, + groupProviderModelData, + MODEL_DATA_MANIFEST_FILE, + type ModelDataStructure, + readModelDataProviderIds, + validateGeneratedModelData, + validateModelDataDirectory, +} from "../scripts/model-data.ts"; +import type { + AnyModel, + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + Model, + ModelType, +} from "../src/types.ts"; +import { buildCatalog, type ProviderCatalog } from "./build.ts"; +import { PROVIDERS } from "./providers/index.ts"; +import { loadCatalogSources } from "./sources/index.ts"; + +/** A catalog entry with its model type spelled out. */ +type CatalogEntry = AnyModel & { type: ModelType }; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); + +interface GeneratorOptions { + strict: boolean; + dataOnly: boolean; + jsonOnly: boolean; + jsonOutputDir: string | undefined; + pretty: boolean; +} + +function readGeneratorOptions(args: string[]): GeneratorOptions { + let strict = false; + let dataOnly = false; + let jsonOnly = false; + let jsonOutputDir: string | undefined; + let pretty = false; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--strict") { + strict = true; + continue; + } + if (arg === "--data-only") { + dataOnly = true; + continue; + } + if (arg === "--json-only") { + jsonOnly = true; + continue; + } + if (arg === "--pretty") { + pretty = true; + continue; + } + if (arg === "--json-output") { + const value = args[++index]; + if (!value) throw new Error("--json-output requires a directory"); + jsonOutputDir = resolve(value); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); + if (dataOnly && (jsonOnly || jsonOutputDir)) + throw new Error("--data-only cannot be combined with JSON catalog output"); + return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty }; +} + +/** Key models by id, keeping the first entry per id, sorted by id. */ +function sortedById(models: readonly T[]): [string, T][] { + const byId = new Map(); + for (const model of models) if (!byId.has(model.id)) byId.set(model.id, model); + return [...byId].sort(([left], [right]) => left.localeCompare(right)); +} + +async function generateModels(generatorOptions: GeneratorOptions): Promise { + const sources = await loadCatalogSources(generatorOptions.strict); + const catalogs = buildCatalog(PROVIDERS, sources, generatorOptions.strict); + // Providers without any models are left out, as if they did not exist. + const providers: Record = Object.fromEntries( + Object.entries(catalogs).filter( + ([, catalog]) => catalog.chat.length + catalog.image.length + catalog.classifier.length > 0, + ), + ); + + const sortedProviderIds = Object.keys(providers).sort(); + const jsonChatProviders: Record & { type: "chat" }>> = {}; + const jsonImageProviders: Record>> = {}; + const jsonClassifierProviders: Record>> = {}; + const jsonAllProviders: Record = {}; + for (const providerId of sortedProviderIds) { + jsonChatProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].chat)); + jsonImageProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].image)); + jsonClassifierProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].classifier)); + jsonAllProviders[providerId] = [ + ...Object.values(jsonChatProviders[providerId]), + ...Object.values(jsonImageProviders[providerId]), + ...Object.values(jsonClassifierProviders[providerId]), + ]; + } + + const serializeJson = (value: unknown) => + `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`; + const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value)); + const generatedDataProviderIds = generatorOptions.dataOnly + ? readModelDataProviderIds(packageRoot) + : sortedProviderIds; + const missingProviderIds = generatedDataProviderIds.filter((providerId) => !jsonAllProviders[providerId]); + if (missingProviderIds.length > 0) { + throw new Error(`Cannot hydrate missing providers: ${missingProviderIds.join(", ")}`); + } + + // Only the ignored internal data is grouped by API for type derivation. + const generatedDataProviders: Record>> = {}; + const modelDataStructure: ModelDataStructure = {}; + for (const providerId of generatedDataProviderIds) { + const { groups, structure } = groupProviderModelData(providerId, jsonAllProviders[providerId]); + generatedDataProviders[providerId] = groups; + modelDataStructure[providerId] = structure; + } + + const generatedAt = new Date().toISOString(); + + if (!generatorOptions.jsonOnly) { + // Stage and validate all provider values before replacing the current generated data. + const providersDir = join(packageRoot, "src/providers"); + const dataDir = join(providersDir, "data"); + const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-")); + const stagedDataDir = join(stagingRoot, "data"); + const previousDataDir = join(stagingRoot, "previous-data"); + let restoreGeneratedCatalog: (() => void) | undefined; + try { + mkdirSync(stagedDataDir, { recursive: true }); + const fileContents: Record = {}; + for (const providerId of generatedDataProviderIds) { + const filename = `${providerId}.json`; + const content = serializeJson(generatedDataProviders[providerId]); + fileContents[filename] = content; + writeFileSync(join(stagedDataDir, filename), content); + } + writeJson( + join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), + createModelDataManifest(modelDataStructure, fileContents, generatedAt), + ); + validateModelDataDirectory(modelDataStructure, stagedDataDir); + + if (!generatorOptions.dataOnly) { + const previousShardContents = new Map( + readdirSync(providersDir) + .filter((entry) => entry.endsWith(".models.ts")) + .map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const), + ); + const aggregatorPath = join(packageRoot, "src/models.generated.ts"); + const previousAggregator = readFileSync(aggregatorPath, "utf8"); + restoreGeneratedCatalog = () => { + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry)); + } + for (const [entry, content] of previousShardContents) { + writeFileSync(join(providersDir, entry), content); + } + writeFileSync(aggregatorPath, previousAggregator); + }; + + const generatedHeader = `// This file is auto-generated by catalog/generate.ts +// Do not edit manually - run 'npm run generate-models' to update + +`; + const catalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; + const imageCatalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_IMAGE_MODELS`; + const classifierCatalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_CLASSIFIER_MODELS`; + const generatedShardFiles = new Set(); + for (const providerId of sortedProviderIds) { + let output = generatedHeader; + output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; + output += `import { flattenChatModelCatalog, flattenClassifierModelCatalog, flattenImageModelCatalog, type ChatModelCatalog, type ClassifierModelCatalog, type ImageModelCatalog } from "../model-catalog.ts";\n\n`; + output += `export const ${catalogConstName(providerId)}: ChatModelCatalog =\n`; + output += `\tflattenChatModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; + output += `export const ${imageCatalogConstName(providerId)}: ImageModelCatalog =\n`; + output += `\tflattenImageModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; + output += `export const ${classifierCatalogConstName(providerId)}: ClassifierModelCatalog =\n`; + output += `\tflattenClassifierModelCatalog(${JSON.stringify(providerId)}, values);\n`; + const filename = `${providerId}.models.ts`; + generatedShardFiles.add(filename); + writeFileSync(join(providersDir, filename), output); + } + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry)); + } + + let output = generatedHeader; + for (const providerId of sortedProviderIds) { + output += `import { ${classifierCatalogConstName(providerId)}, ${imageCatalogConstName(providerId)}, ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; + } + output += `\nexport const MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${catalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; + } + output += `};\n\nexport const IMAGE_MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${imageCatalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${imageCatalogConstName(providerId)},\n`; + } + output += `};\n\nexport const CLASSIFIER_MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${classifierCatalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${classifierCatalogConstName(providerId)},\n`; + } + output += `};\n`; + writeFileSync(aggregatorPath, output); + console.log("Generated provider catalogs and src/models.generated.ts"); + } + + const hadPreviousData = existsSync(dataDir); + if (hadPreviousData) renameSync(dataDir, previousDataDir); + try { + renameSync(stagedDataDir, dataDir); + validateGeneratedModelData(packageRoot); + } catch (error) { + rmSync(dataDir, { recursive: true, force: true }); + if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir); + throw error; + } + restoreGeneratedCatalog = undefined; + console.log( + generatorOptions.dataOnly + ? "Hydrated JSON model values under src/providers/data/" + : "Generated JSON model values under src/providers/data/", + ); + } catch (error) { + restoreGeneratedCatalog?.(); + throw error; + } finally { + rmSync(stagingRoot, { recursive: true, force: true }); + } + } + + if (generatorOptions.jsonOutputDir) { + // `models.json` and `providers/{id}.json` retain the legacy keyed chat catalog. + // The `.all` variants are arrays so the same upstream id can appear once per type. + const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); + rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); + mkdirSync(providerOutputDir, { recursive: true }); + writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonChatProviders); + writeJson(join(generatorOptions.jsonOutputDir, "models.all.json"), jsonAllProviders); + writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); + for (const providerId of sortedProviderIds) { + writeJson(join(providerOutputDir, `${providerId}.json`), jsonChatProviders[providerId]); + writeJson(join(providerOutputDir, `${providerId}.all.json`), jsonAllProviders[providerId]); + } + console.log(`Generated JSON model catalog under ${generatorOptions.jsonOutputDir}`); + } + + // Print statistics + const chatModels = Object.values(providers).flatMap((catalog) => catalog.chat); + const totalModels = chatModels.length; + const reasoningModels = chatModels.filter((model) => model.reasoning).length; + + console.log(`\nModel Statistics:`); + console.log(` Total tool-capable models: ${totalModels}`); + console.log(` Reasoning-capable models: ${reasoningModels}`); + + for (const [provider, models] of Object.entries(providers)) { + console.log( + ` ${provider}: ${models.chat.length} chat models, ${models.image.length} image models, ${models.classifier.length} classifier models`, + ); + } +} + +generateModels(readGeneratorOptions(process.argv.slice(2))).catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/ai/catalog/providers/amazon-bedrock.ts b/packages/ai/catalog/providers/amazon-bedrock.ts new file mode 100644 index 000000000..28bc0c71c --- /dev/null +++ b/packages/ai/catalog/providers/amazon-bedrock.ts @@ -0,0 +1,32 @@ +import { defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; + +// Opus 5 is only callable through cross-region inference profiles. +const INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); + +function bedrockBaseUrl(modelId: string): string { + return modelId.startsWith("eu.") + ? "https://bedrock-runtime.eu-central-1.amazonaws.com" + : "https://bedrock-runtime.us-east-1.amazonaws.com"; +} + +export default defineProvider({ + id: "amazon-bedrock", + models: (ctx) => + modelsDevModels(ctx, "amazon-bedrock", { + api: "bedrock-converse-stream", + baseUrl: "", + include: (source) => + !INFERENCE_PROFILE_ONLY_MODEL_IDS.has(source.id) && + // Jamba does not support tool use in streaming mode. + !source.id.startsWith("ai21.jamba") && + // Mistral 7B Instruct does not support system messages. + !source.id.startsWith("mistral.mistral-7b-instruct-v0"), + // Includes models.dev pricing tiers, e.g. the long-context tier for OpenAI models (#10326). + map: (model, source) => ({ + ...model, + baseUrl: bedrockBaseUrl(model.id), + ...(source.structuredOutput ? { compat: { supportsStrictMode: true } } : {}), + }), + }), + finalize: (model) => withInputLimits(model, { images: { maxPerMessage: 20 } }), +}); diff --git a/packages/ai/catalog/providers/ant-ling.ts b/packages/ai/catalog/providers/ant-ling.ts new file mode 100644 index 000000000..b9cf58cc2 --- /dev/null +++ b/packages/ai/catalog/providers/ant-ling.ts @@ -0,0 +1,52 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, mergeThinking } from "../define.ts"; + +const COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, +}; + +function model(id: string, name: string, reasoning: boolean, cost: Model<"openai-completions">["cost"]) { + return { + id, + name, + api: "openai-completions", + baseUrl: "https://api.ant-ling.com/v1", + provider: "ant-ling", + reasoning, + input: ["text"], + cost, + contextWindow: 262144, + maxTokens: 65536, + compat: reasoning ? { ...COMPAT, thinkingFormat: "ant-ling" } : COMPAT, + } satisfies Model<"openai-completions">; +} + +export default defineProvider({ + id: "ant-ling", + models: () => [ + model("Ling-2.6-flash", "Ling 2.6 Flash", false, { input: 0.01, output: 0.02, cacheRead: 0, cacheWrite: 0 }), + model("Ling-2.6-1T", "Ling 2.6 1T", false, { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }), + model("Ring-2.6-1T", "Ring 2.6 1T", true, { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }), + ], + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "ant-ling", + supportsStrictMode: true, + supportsLongCacheRetention: false, + }, + }, + finalize: (model) => { + // Ring reasons by default. Only high/xhigh have documented explicit effort controls. + if (model.reasoning) { + mergeThinking(model, { off: null, minimal: null, low: null, medium: null, high: "high", xhigh: "xhigh" }); + } + }, +}); diff --git a/packages/ai/catalog/providers/anthropic.ts b/packages/ai/catalog/providers/anthropic.ts new file mode 100644 index 000000000..5b97aff2d --- /dev/null +++ b/packages/ai/catalog/providers/anthropic.ts @@ -0,0 +1,66 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking, modelsDevModels, withInputLimits } from "../define.ts"; +import { + claude55ThinkingLevels, + enableClaudeMidConvoEffort, + supportsClaudeMidConvoEffort, + supportsClaudeMidConvoSystemMessages, +} from "../families/anthropic.ts"; + +// Models the API may fall back to, with the price the caller is billed. +const ALLOWED_FALLBACK_MODELS: Record = { + "claude-fable-5": ["claude-opus-4-8", "claude-opus-5"], + "claude-opus-5": ["claude-opus-4-8"], +}; + +const MODELS_WITH_1M_CONTEXT = new Set(["claude-opus-4-6", "claude-sonnet-4-6"]); +const MODELS_55 = new Set(["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"]); + +export default defineProvider({ + id: "anthropic", + models: (ctx) => + modelsDevModels(ctx, "anthropic", { + api: "anthropic-messages", + baseUrl: "https://api.anthropic.com", + map: (model) => { + if (MODELS_55.has(model.id)) mergeThinking(model, claude55ThinkingLevels(model.id)); + if (MODELS_WITH_1M_CONTEXT.has(model.id)) model.contextWindow = 1000000; + return model; + }, + }), + finalize: (model) => { + if (model.api !== "anthropic-messages") return; + const messagesModel = model as Model<"anthropic-messages">; + if (supportsClaudeMidConvoEffort(model.id)) enableClaudeMidConvoEffort(messagesModel); + if (supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true }); + } + mergeCompat(messagesModel, { supportsStrictTools: true }); + // Ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` extends + // it to one hour. Only direct Anthropic is annotated so cache warming does + // not assume equivalent behavior through proxies. + // https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching + model.promptCache = { short: 300, long: 3600 }; + withInputLimits(model, { + maxRequestBytes: 32 * 1024 * 1024, + images: { maxPerRequest: model.contextWindow === 200000 ? 100 : 600 }, + }); + }, + finalizeAll: (models) => { + const modelsById = new Map(models.map((model) => [model.id, model as Model<"anthropic-messages">])); + for (const [modelId, fallbackModelIds] of Object.entries(ALLOWED_FALLBACK_MODELS)) { + const model = modelsById.get(modelId); + if (!model) continue; + const compatibleFallbackModelIds = model.compat?.supportsMidConvoEffort + ? fallbackModelIds.filter(supportsClaudeMidConvoEffort) + : fallbackModelIds; + const allowedFallbackModels = compatibleFallbackModelIds.flatMap((fallbackModelId) => { + const fallbackModel = modelsById.get(fallbackModelId); + return fallbackModel + ? [{ provider: fallbackModel.provider, model: fallbackModel.id, cost: fallbackModel.cost }] + : []; + }); + if (allowedFallbackModels.length > 0) mergeCompat(model, { allowedFallbackModels }); + } + }, +}); diff --git a/packages/ai/catalog/providers/azure.ts b/packages/ai/catalog/providers/azure.ts new file mode 100644 index 000000000..94bd77a65 --- /dev/null +++ b/packages/ai/catalog/providers/azure.ts @@ -0,0 +1,61 @@ +import type { Model, ModelCost } from "../../src/types.ts"; +import { type ChatModel, defineProvider, mergeCompat } from "../define.ts"; +import { enableOpenAiGrammarTools } from "../families/openai.ts"; + +// Azure Foundry deploys these with larger context windows than OpenAI's own short-tier defaults. +// See models-sold-directly-by-azure docs. +const CONTEXT_WINDOW_OVERRIDES: Record = { + "gpt-5.4": 1050000, + "gpt-5.5": 1050000, + "gpt-5.6-luna": 1050000, + "gpt-5.6-sol": 1050000, + "gpt-5.6-terra": 1050000, +}; + +// Azure resells DeepSeek at its own rates. US data zone, checked 2026-09-16. +// https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/ +const DEEPSEEK_V4_PRO_COST: ModelCost = { input: 1.925, output: 3.828, cacheRead: 0.165, cacheWrite: 0 }; + +/** OpenAI models as Azure deploys them. Azure bills without OpenAI's long-context tier. */ +function openAiModels(models: ChatModel[]): ChatModel[] { + return models + .filter((model) => model.api === "openai-responses") + .map((model) => ({ + ...model, + api: "azure-openai-responses", + provider: "azure", + baseUrl: "", + cost: { + input: model.cost.input, + output: model.cost.output, + cacheRead: model.cost.cacheRead, + cacheWrite: model.cost.cacheWrite, + }, + contextWindow: CONTEXT_WINDOW_OVERRIDES[model.id] ?? model.contextWindow, + })); +} + +export default defineProvider({ + id: "azure", + models: (ctx) => [ + ...openAiModels(ctx.providerModels("openai")), + ...ctx + .providerModels("deepseek") + .filter((model) => model.id === "deepseek-v4-pro") + .map((model) => ({ ...model, provider: "azure", baseUrl: "", cost: DEEPSEEK_V4_PRO_COST })), + ], + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + enableOpenAiGrammarTools(model); + if (model.api === "openai-completions") { + // Azure 400s on DeepSeek's `thinking` field and on every prompt cache parameter, discards a + // `developer` system message unbilled once reasoning_effort is set, and honours mid-convo ones (#9645). + mergeCompat(model as Model<"openai-completions">, { + supportsDeveloperRole: false, + supportsMidConvoSystemMessages: true, + thinkingFormat: "openai", + supportsLongCacheRetention: false, + }); + } + }, +}); diff --git a/packages/ai/catalog/providers/baseten.ts b/packages/ai/catalog/providers/baseten.ts new file mode 100644 index 000000000..4aaaaca13 --- /dev/null +++ b/packages/ai/catalog/providers/baseten.ts @@ -0,0 +1,89 @@ +import type { OpenAICompletionsCompat, ThinkingLevelMap } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +const BASE_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + supportsUsageInStreaming: true, + maxTokensField: "max_tokens", + supportsStrictMode: true, + // Baseten automatic prompt caching needs session affinity so related + // requests land on the same replica. See: + // https://docs.baseten.co/inference/model-apis/pricing-and-limits + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, +}; +const REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + supportsReasoningEffort: true, + thinkingFormat: "openai", +}; +const TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + thinkingFormat: "baseten", + chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, +}; +const TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...REASONING_EFFORT_COMPAT, + thinkingFormat: "baseten", + chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, +}; +const TOGGLE_LEVELS: ThinkingLevelMap = { + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: null, +}; +const GLM_52_LEVELS: ThinkingLevelMap = { + off: "none", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", +}; +const GLM_52_MODEL_IDS = new Set(["zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast"]); + +export default defineProvider({ + id: "baseten", + models: (ctx) => + modelsDevModels(ctx, "baseten", { + api: "openai-completions", + baseUrl: "https://inference.baseten.co/v1", + skipDeprecated: true, + recordReasoning: false, + map: (model, source) => { + const options = source.reasoningOptions ?? []; + const isGlm52 = GLM_52_MODEL_IDS.has(model.id); + const supportsToggle = isGlm52 || options.some((option) => option.type === "toggle"); + const supportsEffort = isGlm52 || options.some((option) => option.type === "effort"); + const compat = + supportsToggle && supportsEffort + ? TOGGLE_REASONING_EFFORT_COMPAT + : supportsToggle + ? TOGGLE_REASONING_COMPAT + : supportsEffort + ? REASONING_EFFORT_COMPAT + : BASE_COMPAT; + const thinkingLevelMap = isGlm52 + ? GLM_52_LEVELS + : supportsToggle + ? TOGGLE_LEVELS + : getEffortThinkingLevelMap(options); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + // Baseten's GLM-5.2 endpoints are text-only despite models.dev reporting image input. + input: !isGlm52 && source.imageInput ? ["text", "image"] : ["text"], + compat, + }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/cerebras.ts b/packages/ai/catalog/providers/cerebras.ts new file mode 100644 index 000000000..18340cbd7 --- /dev/null +++ b/packages/ai/catalog/providers/cerebras.ts @@ -0,0 +1,8 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "cerebras", + models: (ctx) => + modelsDevModels(ctx, "cerebras", { api: "openai-completions", baseUrl: "https://api.cerebras.ai/v1" }), + compat: { "openai-completions": { supportsStore: false, supportsDeveloperRole: false } }, +}); diff --git a/packages/ai/catalog/providers/cloudflare-ai-gateway.ts b/packages/ai/catalog/providers/cloudflare-ai-gateway.ts new file mode 100644 index 000000000..ed4fb95d5 --- /dev/null +++ b/packages/ai/catalog/providers/cloudflare-ai-gateway.ts @@ -0,0 +1,84 @@ +import { + CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, + CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, +} from "../../src/api/cloudflare.ts"; +import type { Model } from "../../src/types.ts"; +import { type ChatModel, defineProvider, mergeCompat, modelsDevModels } from "../define.ts"; +import { enableOpenAiGrammarTools, OPENAI_STANDARD_COSTS, withOpenAiLongContextPricing } from "../families/openai.ts"; + +/** models.dev lists gateway models as `/`; route each upstream to its passthrough. */ +function routeGatewayModel(model: ChatModel): ChatModel | undefined { + const slash = model.id.indexOf("/"); + if (slash === -1) return undefined; + const upstream = model.id.slice(0, slash); + const nativeId = model.id.slice(slash + 1); + if (upstream === "openai") { + // Cloudflare AI Gateway passes OpenAI usage through at OpenAI list prices. + const standardCost = OPENAI_STANDARD_COSTS[nativeId]; + return { + ...model, + id: nativeId, + api: "openai-responses", + baseUrl: CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, + ...(standardCost ? { cost: withOpenAiLongContextPricing(standardCost) } : {}), + }; + } + if (upstream === "anthropic") { + // The /anthropic passthrough forwards the model ID to Anthropic unchanged. + // models.dev lists dotted versions (claude-opus-5.5), but Anthropic only + // accepts dashed IDs (claude-opus-5-5). + return { + ...model, + id: nativeId.replaceAll(".", "-"), + api: "anthropic-messages", + baseUrl: CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, + // Gateway passthroughs forward session affinity headers to upstreams that + // use them for cache/routing affinity. + compat: { sendSessionAffinityHeaders: true }, + }; + } + if (upstream === "workers-ai") { + return { ...model, compat: { sendSessionAffinityHeaders: true } }; + } + return undefined; +} + +export default defineProvider({ + id: "cloudflare-ai-gateway", + models: (ctx) => { + const models = modelsDevModels(ctx, "cloudflare-ai-gateway", { + api: "openai-completions", + baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + map: routeGatewayModel, + }); + // The gateway proxies Workers AI through its OpenAI-compatible /compat endpoint, + // but models.dev may omit or intermittently drop those `workers-ai/*` entries + // from the AI Gateway catalog. Mirror the Workers AI catalog under the documented + // prefix so the gateway keeps its OpenAI-compatible models stable. + const ids = new Set(models.map((model) => model.id)); + const mirrored = modelsDevModels(ctx, "cloudflare-workers-ai", { + api: "openai-completions", + baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + compat: { sendSessionAffinityHeaders: true }, + include: (source) => !ids.has(`workers-ai/${source.id}`), + map: (model) => ({ ...model, id: `workers-ai/${model.id}` }), + }); + return [...models, ...mirrored]; + }, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, + }, + }, + finalize: (model) => { + if (model.api === "openai-responses") { + mergeCompat(model as Model<"openai-responses">, { supportsStrictMode: true }); + } + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/cloudflare-workers-ai.ts b/packages/ai/catalog/providers/cloudflare-workers-ai.ts new file mode 100644 index 000000000..6274661b6 --- /dev/null +++ b/packages/ai/catalog/providers/cloudflare-workers-ai.ts @@ -0,0 +1,61 @@ +import { CLOUDFLARE_WORKERS_AI_BASE_URL, CLOUDFLARE_WORKERS_AI_REST_BASE_URL } from "../../src/api/cloudflare.ts"; +import type { ClassifierModel } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; + +function classifier( + id: string, + name: string, + input: number, + contextWindow: number, + imageInput: "images" | "assets", +): ClassifierModel<"cloudflare-workers-ai-system-one"> { + return { + type: "classifier", + id, + name, + api: "cloudflare-workers-ai-system-one", + provider: "cloudflare-workers-ai", + baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, + input: ["text", "image"], + cost: { input, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow, + compat: { imageInput }, + }; +} + +// Workers AI has no unauthenticated catalog, so its System One models are listed here from +// Cloudflare's model pages. +const CLASSIFIER_MODELS = [ + // Cloudflare-hosted Clef decision models. They take up to four PNG, JPEG, or WebP images as data URLs. + // Clef Omni also takes audio and video, which classifier contexts do not carry. + // Pricing and context: https://developers.cloudflare.com/workers-ai/models/clef/, + // https://developers.cloudflare.com/workers-ai/models/clef-flash/, + // and https://developers.cloudflare.com/workers-ai/models/clef-omni/ + classifier("@cf/cloudflare/clef", "Clef", 0.24, 65536, "images"), + classifier("@cf/cloudflare/clef-flash", "Clef Flash", 0.038, 24576, "images"), + classifier("@cf/cloudflare/clef-omni", "Clef Omni", 0.15, 64000, "images"), + // TypeSafe's Jev models take PNG and JPEG images as System One assets. + // https://developers.cloudflare.com/ai/models/typesafe/jev/ (jev-latest) + // and https://developers.cloudflare.com/ai/models/typesafe/jev-1.14.0/ + classifier("typesafe/jev", "Jev", 0.042, 32000, "assets"), + classifier("typesafe/jev-1.14.0", "Jev 1.14", 0.042, 32000, "assets"), +]; + +export default defineProvider({ + id: "cloudflare-workers-ai", + models: (ctx) => + modelsDevModels(ctx, "cloudflare-workers-ai", { + api: "openai-completions", + baseUrl: CLOUDFLARE_WORKERS_AI_BASE_URL, + compat: { sendSessionAffinityHeaders: true }, + }), + classifiers: () => CLASSIFIER_MODELS, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsStrictMode: true, + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/deepseek.ts b/packages/ai/catalog/providers/deepseek.ts new file mode 100644 index 000000000..af048844d --- /dev/null +++ b/packages/ai/catalog/providers/deepseek.ts @@ -0,0 +1,57 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat } from "../define.ts"; +import { DEEPSEEK_COMPAT, DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP } from "../families/deepseek.ts"; + +// models.dev's DeepSeek entries lag the API; pricing from +// https://api-docs.deepseek.com/quick_start/pricing. DeepSeek also offers +// time-based off-peak rates, which the cost schema cannot represent yet. +const MODELS: Model<"openai-completions">[] = [ + { + id: "deepseek-flash", + name: "DeepSeek V4.1 Flash", + api: "openai-completions", + baseUrl: "https://api.deepseek.com", + provider: "deepseek", + reasoning: true, + thinkingLevelMap: DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP, + input: ["text", "image"], + cost: { input: 0.3, output: 1.2, cacheRead: 0.006, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 384000, + compat: DEEPSEEK_COMPAT, + }, + { + id: "deepseek-v4-pro", + name: "DeepSeek V4 Pro", + api: "openai-completions", + baseUrl: "https://api.deepseek.com", + provider: "deepseek", + reasoning: true, + input: ["text"], + cost: { input: 1.32, output: 3.96, cacheRead: 0.044, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 384000, + compat: DEEPSEEK_COMPAT, + }, +]; + +export default defineProvider({ + id: "deepseek", + models: () => structuredClone(MODELS), + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + maxTokensField: "max_tokens", + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", + supportsStrictMode: true, + }, + }, + finalize: (model) => { + // DeepSeek V4 Pro accepts plain system text in place mid-conversation. + if (model.id === "deepseek-v4-pro") { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + }, +}); diff --git a/packages/ai/catalog/providers/fireworks.ts b/packages/ai/catalog/providers/fireworks.ts new file mode 100644 index 000000000..9751c022b --- /dev/null +++ b/packages/ai/catalog/providers/fireworks.ts @@ -0,0 +1,100 @@ +import type { AnthropicMessagesCompat, Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Fireworks Anthropic-compatible API. Prompt caching uses automatic prefix +// matching and session affinity: x-session-affinity routes requests to the same +// replica for cache hits. cache_control on tools and eager_input_streaming are +// not supported. +// https://docs.fireworks.ai/tools-sdks/anthropic-compatibility +const ANTHROPIC_COMPAT: AnthropicMessagesCompat = { + allowEmptySignature: true, + sendSessionAffinityHeaders: true, + supportsEagerToolInputStreaming: false, + supportsCacheControlOnTools: false, + supportsLongCacheRetention: false, +}; +const OPENAI_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, +}; +const KIMI_K3_COMPAT: OpenAICompletionsCompat = { + ...OPENAI_COMPAT, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", +}; + +// Verified against Fireworks Messages raw_output on 2026-09-10 (#9323). +// Fall back to verified support when models.dev omits effort metadata; this is +// not an allowlist. Any Fireworks Messages model advertising effort uses adaptive thinking. +const ADAPTIVE_THINKING_FALLBACK_MODELS = new Set([ + "accounts/fireworks/models/deepseek-v4-flash-0731", + "accounts/fireworks/models/deepseek-v4-flash-vision-exp", + "accounts/fireworks/models/deepseek-v4-pro-0813", + "accounts/fireworks/models/qwen3p8-max", + "accounts/fireworks/models/qwen3p8-2p4t-a95b", +]); + +export default defineProvider({ + id: "fireworks", + models: (ctx) => + modelsDevModels(ctx, "fireworks-ai", { + api: "anthropic-messages", + // SDK appends /v1/messages. + baseUrl: "https://api.fireworks.ai/inference", + map: (model, source) => { + if (model.id.includes("glm-") || model.id.includes("kimi-k3")) { + return { + ...model, + api: "openai-completions", + baseUrl: "https://api.fireworks.ai/inference/v1", + compat: model.id.includes("kimi-k3") ? KIMI_K3_COMPAT : OPENAI_COMPAT, + }; + } + // Use adaptive thinking for cataloged effort controls, with verified + // fallbacks where models.dev is incomplete. New models need no allowlist entry. + const adaptive = + source.reasoningOptions?.some((option) => option.type === "effort") || + ADAPTIVE_THINKING_FALLBACK_MODELS.has(model.id); + return { ...model, compat: { ...ANTHROPIC_COMPAT, ...(adaptive ? { forceAdaptiveThinking: true } : {}) } }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model, ctx) => { + if (model.api === "anthropic-messages" && (model as Model<"anthropic-messages">).compat?.forceAdaptiveThinking) { + // Qwen Max currently advertises only a toggle. Prefer upstream effort + // metadata once available instead of replacing it with this fallback. + if (model.id === "accounts/fireworks/models/qwen3p8-max" && !model.thinkingLevelMap) { + model.thinkingLevelMap = getEffortThinkingLevelMap([ + { type: "effort", values: ["low", "medium", "xhigh"] }, + ]); + } + if ( + ctx.modelsDevSource(model)?.reasoningOptions?.some((option) => option.type === "toggle") || + // The 2.4T alias omits the verified toggle in models.dev. + model.id === "accounts/fireworks/models/qwen3p8-2p4t-a95b" + ) { + mergeThinking(model, { off: "none" }); + } + if (model.id === "accounts/fireworks/models/deepseek-v4-pro-0813") mergeThinking(model, { low: "low" }); + } + if (model.id.includes("glm-5p2")) { + // GLM 5.2 and its fast router support off/high/max. Fireworks maps low + // and medium to high, so do not expose those aliases as distinct levels. + mergeThinking(model, { off: "none", minimal: null, low: null, medium: null, max: "max" }); + } + if (model.id.includes("kimi-k3")) { + // Fireworks maps medium to high on both APIs; do not expose it as a distinct level. + mergeThinking(model, { medium: null }); + // Kimi K3 accepts tool-bearing system messages; Fireworks passes them through. + if (model.api === "openai-completions") { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + supportsMidConvoToolAdditions: true, + }); + } + } + }, +}); diff --git a/packages/ai/catalog/providers/github-copilot.ts b/packages/ai/catalog/providers/github-copilot.ts new file mode 100644 index 000000000..2bea3586b --- /dev/null +++ b/packages/ai/catalog/providers/github-copilot.ts @@ -0,0 +1,122 @@ +import type { Api, Model, ThinkingLevelMap } from "../../src/types.ts"; +import { addMissing, type ChatModel, defineProvider, mergeCompat, mergeThinking, modelsDevModels } from "../define.ts"; +import { claude55ThinkingLevels, supportsClaudeMidConvoSystemMessages } from "../families/anthropic.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + openAiStandardCost, +} from "../families/openai.ts"; + +const BASE_URL = "https://api.individual.githubcopilot.com"; +const STATIC_HEADERS = { + "User-Agent": "GitHubCopilotChat/0.35.0", + "Editor-Version": "vscode/1.107.0", + "Editor-Plugin-Version": "copilot-chat/0.35.0", + "Copilot-Integration-Id": "vscode-chat", +} as const; + +// GitHub's "Models with extended capabilities" table lists these Copilot models as supporting +// the extended 1 million token context window. +const EXTENDED_CONTEXT_MODEL_IDS = new Set([ + "claude-fable-5", + "claude-opus-4.8", + "claude-opus-5", + "claude-opus-5.5", + "claude-sonnet-4.6", + "claude-sonnet-5", + "gpt-5.3-codex", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", +]); + +const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_MODEL_IDS = new Set([ + "claude-haiku-4.5", + "claude-sonnet-4", + "claude-sonnet-4.5", +]); + +// Checked manually against the authenticated GitHub Copilot /models endpoint on 2026-06-15. +// Keep this to narrow corrections over models.dev metadata instead of snapshotting Copilot's catalog. +const THINKING_LEVEL_OVERRIDES: Record = { + "claude-opus-4.8": { minimal: "low" }, + "claude-opus-5": { minimal: "low" }, + "claude-sonnet-4.6": { minimal: "low", max: "max" }, +}; + +/** Claude 4.x and 5.x use Messages; GPT, Grok, OSWE, and MAI-Code are only served on /responses. */ +function copilotApi(modelId: string): Api { + if (/^claude-(haiku|sonnet|opus|fable)-[45]([.-]|$)/.test(modelId)) return "anthropic-messages"; + if (/^(gpt-|grok-|oswe|mai-)/.test(modelId)) return "openai-responses"; + return "openai-completions"; +} + +// The authenticated Copilot catalog advertised these models on 2026-09-22, +// but models.dev did not include them yet. +const GPT_6_LUNA: Model<"openai-responses"> = { + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-responses", + provider: "github-copilot", + baseUrl: BASE_URL, + reasoning: true, + input: ["text", "image"], + cost: openAiStandardCost("gpt-6-luna"), + contextWindow: 1000000, + maxTokens: 128000, + headers: { ...STATIC_HEADERS }, +}; + +export default defineProvider({ + id: "github-copilot", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "github-copilot", { + api: "openai-completions", + baseUrl: BASE_URL, + headers: STATIC_HEADERS, + skipDeprecated: true, + defaultContextWindow: 128000, + defaultMaxTokens: 8192, + map: (model): ChatModel => { + const api = copilotApi(model.id); + const result: ChatModel = { ...model, api }; + if (api === "openai-completions") { + result.compat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + }; + } + if (model.id === "claude-opus-5.5") mergeThinking(result, claude55ThinkingLevels(model.id)); + return result; + }, + }), + [GPT_6_LUNA], + ).map((model) => (EXTENDED_CONTEXT_MODEL_IDS.has(model.id) ? { ...model, contextWindow: 1000000 } : model)), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + if (model.id.startsWith("gpt-5")) mergeThinking(model, { minimal: "low" }); + const override = THINKING_LEVEL_OVERRIDES[model.id]; + if (override) mergeThinking(model, override); + if (model.api === "anthropic-messages") { + const messagesModel = model as Model<"anthropic-messages">; + // Copilot forwards mid-conversation system messages but rejects + // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level. + if (supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true }); + } + if (EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_MODEL_IDS.has(model.id)) { + mergeCompat(messagesModel, { supportsEagerToolInputStreaming: false }); + } + } + // Copilot forwards Kimi K3's system text but silently drops its tool-bearing message. + if (model.api === "openai-completions" && model.id === "kimi-k3") { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + // Copilot passes mid-conversation developer messages and `additional_tools` + // through to OpenAI unchanged; tool search is not verified through it. + enableOpenAiMidConvoSystemMessages(model, { proxy: true }); + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/google-vertex.ts b/packages/ai/catalog/providers/google-vertex.ts new file mode 100644 index 000000000..e74a4ed5f --- /dev/null +++ b/packages/ai/catalog/providers/google-vertex.ts @@ -0,0 +1,43 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { modelsDevChatModels } from "../sources/models-dev.ts"; +import { GOOGLE_ALIAS_TARGETS } from "./google.ts"; + +export default defineProvider({ + id: "google-vertex", + models: (ctx) => { + const sources = new Map( + modelsDevChatModels(ctx.sources.modelsDev, "google-vertex").map((model) => [model.id, model]), + ); + return modelsDevModels(ctx, "google-vertex", { + api: "google-vertex", + baseUrl: "https://{location}-aiplatform.googleapis.com", + recordReasoning: false, + // The models.dev catalog also includes Claude, OpenAI, and other MaaS models + // that do not use the @google/genai Gemini streaming path. + include: (source) => source.id.startsWith("gemini-") && source.id !== "gemini-3.1-flash-lite-preview", + map: (model, aliasSource) => { + const target = GOOGLE_ALIAS_TARGETS[model.id]; + const source = (target && sources.get(target)) || aliasSource; + const thinkingLevelMap = googleThinkingLevels(model.id, source.reasoningOptions); + return { + ...model, + reasoning: source.reasoning, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + input: source.imageInput ? ["text", "image"] : ["text"], + cost: { + input: source.cost.input || 0, + output: source.cost.output || 0, + // models.dev reports Vertex cache_read/cache_write values for Gemini 2.5 Flash that + // do not match the official Gemini API standard pricing table. pi only accounts + // cachedContentTokenCount as cacheRead. + cacheRead: model.id === "gemini-2.5-flash" ? 0.03 : source.cost.cacheRead || 0, + cacheWrite: 0, + }, + contextWindow: source.contextWindow || 4096, + maxTokens: source.maxTokens || 4096, + }; + }, + }); + }, +}); diff --git a/packages/ai/catalog/providers/google.ts b/packages/ai/catalog/providers/google.ts new file mode 100644 index 000000000..15cac1f93 --- /dev/null +++ b/packages/ai/catalog/providers/google.ts @@ -0,0 +1,36 @@ +import { defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { modelsDevChatModels, modelsDevCost } from "../sources/models-dev.ts"; + +// Rolling aliases take their metadata from the model they currently point to. +export const GOOGLE_ALIAS_TARGETS: Record = { + "gemini-flash-latest": "gemini-3.5-flash", + "gemini-flash-lite-latest": "gemini-3.1-flash-lite", +}; + +export default defineProvider({ + id: "google", + models: (ctx) => { + const sources = new Map(modelsDevChatModels(ctx.sources.modelsDev, "google").map((model) => [model.id, model])); + return modelsDevModels(ctx, "google", { + api: "google-generative-ai", + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + recordReasoning: false, + map: (model, aliasSource) => { + const target = GOOGLE_ALIAS_TARGETS[model.id]; + const source = (target && sources.get(target)) || aliasSource; + const thinkingLevelMap = googleThinkingLevels(model.id, source.reasoningOptions); + return { + ...model, + reasoning: source.reasoning, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + input: source.imageInput ? ["text", "image"] : ["text"], + cost: modelsDevCost(source.cost), + contextWindow: source.contextWindow || 4096, + maxTokens: source.maxTokens || 4096, + }; + }, + }); + }, + finalize: (model) => withInputLimits(model, { maxRequestBytes: 20 * 1024 * 1024, images: { maxPerRequest: 3600 } }), +}); diff --git a/packages/ai/catalog/providers/groq.ts b/packages/ai/catalog/providers/groq.ts new file mode 100644 index 000000000..7f6e15d76 --- /dev/null +++ b/packages/ai/catalog/providers/groq.ts @@ -0,0 +1,13 @@ +import { defineProvider, mergeThinking, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "groq", + models: (ctx) => + modelsDevModels(ctx, "groq", { api: "openai-completions", baseUrl: "https://api.groq.com/openai/v1" }), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + if (model.id === "qwen/qwen3.6-27b") { + mergeThinking(model, { minimal: null, low: null, medium: null, high: "default" }); + } + }, +}); diff --git a/packages/ai/catalog/providers/huggingface.ts b/packages/ai/catalog/providers/huggingface.ts new file mode 100644 index 000000000..69ab9104c --- /dev/null +++ b/packages/ai/catalog/providers/huggingface.ts @@ -0,0 +1,12 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "huggingface", + models: (ctx) => + modelsDevModels(ctx, "huggingface", { + api: "openai-completions", + baseUrl: "https://router.huggingface.co/v1", + compat: { supportsDeveloperRole: false }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/index.ts b/packages/ai/catalog/providers/index.ts new file mode 100644 index 000000000..60b29cbd5 --- /dev/null +++ b/packages/ai/catalog/providers/index.ts @@ -0,0 +1,90 @@ +// Every built-in provider catalog. Add new providers here. + +import type { ProviderDefinition } from "../define.ts"; +import amazonBedrock from "./amazon-bedrock.ts"; +import antLing from "./ant-ling.ts"; +import anthropic from "./anthropic.ts"; +import azure from "./azure.ts"; +import baseten from "./baseten.ts"; +import cerebras from "./cerebras.ts"; +import cloudflareAiGateway from "./cloudflare-ai-gateway.ts"; +import cloudflareWorkersAi from "./cloudflare-workers-ai.ts"; +import deepseek from "./deepseek.ts"; +import fireworks from "./fireworks.ts"; +import githubCopilot from "./github-copilot.ts"; +import google from "./google.ts"; +import googleVertex from "./google-vertex.ts"; +import groq from "./groq.ts"; +import huggingface from "./huggingface.ts"; +import kimiCoding from "./kimi-coding.ts"; +import meta from "./meta.ts"; +import minimax from "./minimax.ts"; +import minimaxCn from "./minimax-cn.ts"; +import mistral from "./mistral.ts"; +import moonshotai from "./moonshotai.ts"; +import moonshotaiCn from "./moonshotai-cn.ts"; +import nvidia from "./nvidia.ts"; +import openai from "./openai.ts"; +import openaiCodex from "./openai-codex.ts"; +import opencode from "./opencode.ts"; +import opencodeGo from "./opencode-go.ts"; +import openrouter from "./openrouter.ts"; +import qwenTokenPlan from "./qwen-token-plan.ts"; +import qwenTokenPlanCn from "./qwen-token-plan-cn.ts"; +import qwenTokenPlanIndividual from "./qwen-token-plan-individual.ts"; +import radius from "./radius.ts"; +import together from "./together.ts"; +import typesafe from "./typesafe.ts"; +import vercelAiGateway from "./vercel-ai-gateway.ts"; +import xai from "./xai.ts"; +import xiaomi from "./xiaomi.ts"; +import xiaomiTokenPlanAms from "./xiaomi-token-plan-ams.ts"; +import xiaomiTokenPlanCn from "./xiaomi-token-plan-cn.ts"; +import xiaomiTokenPlanSgp from "./xiaomi-token-plan-sgp.ts"; +import zai from "./zai.ts"; +import zaiCodingCn from "./zai-coding-cn.ts"; + +export const PROVIDERS: readonly ProviderDefinition[] = [ + amazonBedrock, + antLing, + anthropic, + azure, + baseten, + cerebras, + cloudflareAiGateway, + cloudflareWorkersAi, + deepseek, + fireworks, + githubCopilot, + googleVertex, + google, + groq, + huggingface, + kimiCoding, + meta, + minimaxCn, + minimax, + mistral, + moonshotaiCn, + moonshotai, + nvidia, + openaiCodex, + openai, + opencodeGo, + opencode, + openrouter, + qwenTokenPlanCn, + qwenTokenPlanIndividual, + qwenTokenPlan, + radius, + together, + typesafe, + vercelAiGateway, + xai, + xiaomiTokenPlanAms, + xiaomiTokenPlanCn, + xiaomiTokenPlanSgp, + xiaomi, + zaiCodingCn, + zai, +]; diff --git a/packages/ai/catalog/providers/kimi-coding.ts b/packages/ai/catalog/providers/kimi-coding.ts new file mode 100644 index 000000000..782b44455 --- /dev/null +++ b/packages/ai/catalog/providers/kimi-coding.ts @@ -0,0 +1,55 @@ +import type { Model, ModelCost } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; +import { KIMI_K3_COST, KIMI_K3_OPENAI_COMPAT, KIMI_OPENAI_COMPAT } from "../families/kimi.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Kimi Coding is subscription-backed, so models.dev reports zero cost. Use the +// equivalent Moonshot API rates to estimate the value of subscription usage. +const IMPLIED_COSTS: Record = { + k3: KIMI_K3_COST, + "k3-256k": KIMI_K3_COST, + "k3.1-pro": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 0 }, + "k3.1-pro-highspeed": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 0 }, + "kimi-for-coding": { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 }, + "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, +}; + +// Kimi Coding uses Chat Completions, the same protocol as the official Kimi Code client. +export default defineProvider({ + id: "kimi-coding", + models: (ctx) => + modelsDevModels(ctx, "kimi-code-plan-global", { + api: "openai-completions", + baseUrl: "https://api.kimi.com/coding/v1", + recordReasoning: false, + map: (model, source): Model<"openai-completions"> => { + // K3-style models report effort levels and dynamic tool loading (tools declared in + // later system messages) in https://api.kimi.com/coding/v1/models; models.dev mirrors + // the effort levels. kimi-for-coding-highspeed reports neither and rejects + // tool-bearing system messages with a 400. + const isK3Style = source.reasoningOptions?.some((option) => option.type === "effort") === true; + const impliedCost = IMPLIED_COSTS[model.id]; + // K3-style models send reasoning_effort, so expose the effort values models.dev lists. + const effortLevels = + isK3Style && source.reasoningOptions ? getEffortThinkingLevelMap(source.reasoningOptions) : undefined; + return { + ...model, + compat: { + ...(isK3Style ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT), + // All models accept system text after the conversation starts. + supportsMidConvoSystemMessages: true, + ...(isK3Style ? { supportsMidConvoToolAdditions: true } : {}), + }, + cost: { + input: source.cost.input || impliedCost?.input || 0, + output: source.cost.output || impliedCost?.output || 0, + cacheRead: source.cost.cacheRead || impliedCost?.cacheRead || 0, + cacheWrite: source.cost.cacheWrite || impliedCost?.cacheWrite || 0, + }, + // Kimi Code reports every model as always-thinking. + thinkingLevelMap: { ...effortLevels, off: null }, + }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/meta.ts b/packages/ai/catalog/providers/meta.ts new file mode 100644 index 000000000..7d4bba68d --- /dev/null +++ b/packages/ai/catalog/providers/meta.ts @@ -0,0 +1,6 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "meta", + models: (ctx) => modelsDevModels(ctx, "meta", { api: "openai-responses", baseUrl: "https://api.meta.ai/v1" }), +}); diff --git a/packages/ai/catalog/providers/minimax-cn.ts b/packages/ai/catalog/providers/minimax-cn.ts new file mode 100644 index 000000000..c34cd2316 --- /dev/null +++ b/packages/ai/catalog/providers/minimax-cn.ts @@ -0,0 +1,7 @@ +import { defineProvider } from "../define.ts"; +import { minimaxModels } from "./minimax.ts"; + +export default defineProvider({ + id: "minimax-cn", + models: (ctx) => minimaxModels(ctx, "minimax-cn", "https://api.minimaxi.com/anthropic"), +}); diff --git a/packages/ai/catalog/providers/minimax.ts b/packages/ai/catalog/providers/minimax.ts new file mode 100644 index 000000000..7504b7683 --- /dev/null +++ b/packages/ai/catalog/providers/minimax.ts @@ -0,0 +1,26 @@ +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// MiniMax asked to hide older models from its direct providers (#2596). +const HIDDEN_MODEL_IDS = new Set(["MiniMax-M2", "MiniMax-M2.1", "MiniMax-M2.5", "MiniMax-M2.5-highspeed"]); + +/** MiniMax's Anthropic-compatible API. The SDK appends /v1/messages. */ +export function minimaxModels(ctx: ProviderContext, source: string, baseUrl: string) { + return modelsDevModels(ctx, source, { + api: "anthropic-messages", + baseUrl, + recordReasoning: false, + include: (model) => !HIDDEN_MODEL_IDS.has(model.id), + map: (model, source) => { + // Models with effort levels (e.g. MiniMax-M3.1-Flash) take adaptive thinking with + // output_config.effort and reject thinking.type="disabled" (no "none" effort, so off is null). + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + return thinkingLevelMap ? { ...model, thinkingLevelMap, compat: { forceAdaptiveThinking: true } } : model; + }, + }); +} + +export default defineProvider({ + id: "minimax", + models: (ctx) => minimaxModels(ctx, "minimax", "https://api.minimax.io/anthropic"), +}); diff --git a/packages/ai/catalog/providers/mistral.ts b/packages/ai/catalog/providers/mistral.ts new file mode 100644 index 000000000..8009e4650 --- /dev/null +++ b/packages/ai/catalog/providers/mistral.ts @@ -0,0 +1,46 @@ +import { addMissing, defineProvider, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import { roundCost } from "../sources/models-dev.ts"; + +export default defineProvider({ + id: "mistral", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "mistral", { + api: "mistral-conversations", + baseUrl: "https://api.mistral.ai", + recordReasoning: false, + map: (model, source) => { + // Models with effort values use `reasoning_effort` with these levels. + // Reasoning models without them (Magistral) use `prompt_mode`. + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + cost: { + input: source.cost.input || 0, + output: source.cost.output || 0, + // Mistral bills cached input at 10% when models.dev does not say otherwise. + cacheRead: source.cost.cacheRead ?? (source.cost.input ? roundCost(source.cost.input * 0.1) : 0), + cacheWrite: source.cost.cacheWrite || 0, + }, + }; + }, + }), + [ + { + id: "mistral-medium-3.5", + name: "Mistral Medium 3.5", + api: "mistral-conversations", + provider: "mistral", + baseUrl: "https://api.mistral.ai", + reasoning: true, + thinkingLevelMap: getEffortThinkingLevelMap([{ type: "effort", values: ["none", "high"] }]), + input: ["text", "image"], + cost: { input: 1.5, output: 7.5, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 262144, + maxTokens: 262144, + }, + ], + ), +}); diff --git a/packages/ai/catalog/providers/moonshotai-cn.ts b/packages/ai/catalog/providers/moonshotai-cn.ts new file mode 100644 index 000000000..e8f3d91e1 --- /dev/null +++ b/packages/ai/catalog/providers/moonshotai-cn.ts @@ -0,0 +1,9 @@ +import { defineProvider } from "../define.ts"; +import { finalizeMoonshotModel, MOONSHOT_COMPLETIONS_COMPAT, moonshotModels } from "./moonshotai.ts"; + +export default defineProvider({ + id: "moonshotai-cn", + models: (ctx) => moonshotModels(ctx, "moonshotai-cn", "https://api.moonshot.cn/v1"), + compat: { "openai-completions": MOONSHOT_COMPLETIONS_COMPAT }, + finalize: finalizeMoonshotModel, +}); diff --git a/packages/ai/catalog/providers/moonshotai.ts b/packages/ai/catalog/providers/moonshotai.ts new file mode 100644 index 000000000..11ba2a5cc --- /dev/null +++ b/packages/ai/catalog/providers/moonshotai.ts @@ -0,0 +1,71 @@ +import type { Model } from "../../src/types.ts"; +import { + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + type ProviderContext, +} from "../define.ts"; +import { isKimiK3FamilyId, KIMI_K3_COST, KIMI_K3_OPENAI_COMPAT, KIMI_OPENAI_COMPAT } from "../families/kimi.ts"; + +const ALWAYS_THINKING_MODEL_IDS = new Set(["kimi-k2.7-code", "kimi-k2.7-code-highspeed"]); +// Kimi K2.6/K2.7 accept system text after the conversation starts but reject +// tool-bearing system messages. +const TEXT_ONLY_MID_CONVO_MODEL_IDS = new Set(["kimi-k2.6", ...ALWAYS_THINKING_MODEL_IDS]); + +export function moonshotModels(ctx: ProviderContext, source: string, baseUrl: string) { + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + map: (model, source): Model<"openai-completions"> => { + const isKimiK3 = isKimiK3FamilyId(model.id); + return { + ...model, + reasoning: isKimiK3 || source.reasoning, + // Moonshot does not bill cache writes; models.dev lists the input rate as cache_write for the K3 family. + cost: + model.id === "kimi-k3" + ? { ...KIMI_K3_COST } + : { + input: source.cost.input || 0, + output: source.cost.output || 0, + cacheRead: source.cost.cacheRead || 0, + cacheWrite: isKimiK3 ? 0 : source.cost.cacheWrite || 0, + }, + compat: { ...(isKimiK3 ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT) }, + }; + }, + }); +} + +export const MOONSHOT_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", +} as const; + +export function finalizeMoonshotModel(model: ChatModel): void { + if (ALWAYS_THINKING_MODEL_IDS.has(model.id)) { + // Kimi K2.7 Code is always-thinking. Official docs say + // `thinking: { type: "disabled" }` is rejected, and callers can omit + // the thinking parameter to use the enabled default. + mergeThinking(model, { off: null }); + } + const isKimiK3 = isKimiK3FamilyId(model.id); + if (isKimiK3 || TEXT_ONLY_MID_CONVO_MODEL_IDS.has(model.id)) { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + // The Kimi K3 family also accepts tool-bearing system messages. + ...(isKimiK3 ? { supportsMidConvoToolAdditions: true } : {}), + }); + } +} + +export default defineProvider({ + id: "moonshotai", + models: (ctx) => moonshotModels(ctx, "moonshotai", "https://api.moonshot.ai/v1"), + compat: { "openai-completions": MOONSHOT_COMPLETIONS_COMPAT }, + finalize: finalizeMoonshotModel, +}); diff --git a/packages/ai/catalog/providers/nvidia.ts b/packages/ai/catalog/providers/nvidia.ts new file mode 100644 index 000000000..1d492ff29 --- /dev/null +++ b/packages/ai/catalog/providers/nvidia.ts @@ -0,0 +1,64 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; +import { NVIDIA_BASE_URL, normalizeNvidiaModelId } from "../sources/index.ts"; + +// Listed by NIM but unusable through pi (no tool calls, broken streaming, or retired). +const UNSUPPORTED_MODEL_IDS = new Set([ + "abacusai/dracarys-llama-3.1-70b-instruct", + "bytedance/seed-oss-36b-instruct", + "deepseek-ai/deepseek-v4-flash", + "deepseek-ai/deepseek-v4-pro", + "google/gemma-2-2b-it", + "google/gemma-3n-e2b-it", + "google/gemma-3n-e4b-it", + "google/gemma-4-31b-it", + "meta/llama-3.2-1b-instruct", + "meta/llama-4-maverick-17b-128e-instruct", + "microsoft/phi-4-mini-instruct", + "minimaxai/minimax-m2.7", + "mistralai/mistral-nemotron", + "nvidia/nemotron-mini-4b-instruct", + "qwen/qwen3-next-80b-a3b-instruct", + "qwen/qwen3.5-397b-a17b", + "sarvamai/sarvam-m", + "upstage/solar-10.7b-instruct", +]); + +export default defineProvider({ + id: "nvidia", + models: (ctx) => { + const liveIds = ctx.sources.nvidiaModelIds; + const liveId = (id: string) => liveIds.get(id) ?? liveIds.get(normalizeNvidiaModelId(id)); + return modelsDevModels(ctx, "nvidia", { + api: "openai-completions", + baseUrl: NVIDIA_BASE_URL, + headers: { "NVCF-POLL-SECONDS": "3600" }, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }, + // Only models the live NIM API serves, under the id it uses. + include: (source) => { + if (!source.textInput || !source.textOutput) return false; + const id = liveId(source.id); + return id !== undefined && !UNSUPPORTED_MODEL_IDS.has(id); + }, + map: (model) => { + const id = liveId(model.id) ?? model.id; + return { ...model, id, name: model.name === model.id ? id : model.name }; + }, + }); + }, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/openai-codex.ts b/packages/ai/catalog/providers/openai-codex.ts new file mode 100644 index 000000000..fc120d039 --- /dev/null +++ b/packages/ai/catalog/providers/openai-codex.ts @@ -0,0 +1,68 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking } from "../define.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + OPENAI_TOOL_SEARCH_MODEL_IDS, + openAiStandardCost, + supportsOpenAiXhigh, +} from "../families/openai.ts"; + +// ChatGPT OAuth models. models.dev does not list them; keep a small, explicit +// list without aliases. Only list models the Codex backend accepts for ChatGPT +// accounts: GPT-5.5 and GPT-5.3 Codex Spark are rejected with "not supported +// when using Codex with a ChatGPT account". +const MODELS = [ + // GPT-6.1 uses the 500k default returned by /backend-api/codex/models. + { id: "gpt-6.1-sol", name: "GPT-6.1 Sol", contextWindow: 500000 }, + { id: "gpt-6.1-terra", name: "GPT-6.1 Terra", contextWindow: 500000 }, + // GPT-5.6 and GPT-6 use Codex's 272k default catalog limit. + { id: "gpt-6-astra", name: "GPT-6 Astra", contextWindow: 272000 }, + { id: "gpt-6-sol", name: "GPT-6 Sol", contextWindow: 272000 }, + { id: "gpt-6-luna", name: "GPT-6 Luna", contextWindow: 272000 }, + { id: "gpt-5.6-luna", name: "GPT-5.6 Luna", contextWindow: 272000 }, + { id: "gpt-5.6-sol", name: "GPT-5.6 Sol", contextWindow: 272000 }, + { id: "gpt-5.6-terra", name: "GPT-5.6 Terra", contextWindow: 272000 }, +]; + +// Codex CLI keeps additional_tools behind its own model list. +const ADDITIONAL_TOOLS_MODEL_IDS = new Set([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); + +export default defineProvider({ + id: "openai-codex", + models: () => + MODELS.map( + ({ id, name, contextWindow }): Model<"openai-codex-responses"> => ({ + id, + name, + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text", "image"], + cost: openAiStandardCost(id), + contextWindow, + maxTokens: 128000, + }), + ), + finalize: (model) => { + if (supportsOpenAiXhigh(model.id)) mergeThinking(model, { minimal: "low" }); + enableOpenAiGrammarTools(model); + if (OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) { + mergeCompat(model as Model<"openai-codex-responses">, { + ...(ADDITIONAL_TOOLS_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), + supportsToolSearch: true, + }); + } + enableOpenAiMidConvoSystemMessages(model, { proxy: false }); + }, +}); diff --git a/packages/ai/catalog/providers/openai.ts b/packages/ai/catalog/providers/openai.ts new file mode 100644 index 000000000..873dc11ec --- /dev/null +++ b/packages/ai/catalog/providers/openai.ts @@ -0,0 +1,141 @@ +import type { ClassifierModel, Model } from "../../src/types.ts"; +import { + addMissing, + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + withInputLimits, +} from "../define.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, + OPENAI_STANDARD_COSTS, + OPENAI_TOOL_SEARCH_MODEL_IDS, + withOpenAiLongContextPricing, +} from "../families/openai.ts"; + +const BASE_URL = "https://api.openai.com/v1"; + +// models.dev lists this alias, but it is not accepted by OpenAI APIs. +const UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]); + +// Keep direct OpenAI requests in the short-context pricing tier by default. Users can opt into the +// larger context through model overrides, so retain long-context cost metadata on the capped models. +const SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); +const LONG_CONTEXT_PRICING_MODEL_IDS = new Set([...SHORT_CONTEXT_CAPPED_MODEL_IDS, "gpt-5.4-pro", "gpt-5.5-pro"]); + +const NONE_REASONING_MODEL_IDS = new Set([ + "gpt-5.1", + "gpt-5.2", + "gpt-5.3-codex", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-sol", + "gpt-6-luna", +]); +// OpenAI documents configuration_update for the GPT-6 family. +// https://developers.openai.com/api/docs/guides/reasoning +const MID_CONVO_EFFORT_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"]); + +// Not listed by models.dev. +const GPT_5_CHAT_LATEST: Model<"openai-responses"> = { + id: "gpt-5-chat-latest", + name: "GPT-5 Chat Latest", + api: "openai-responses", + baseUrl: BASE_URL, + provider: "openai", + reasoning: false, + input: ["text", "image"], + cost: { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, +}; + +// OpenAI Decisions API (public beta): gpt-6-luna is its only model. It bills input tokens only, +// with the same long-context multiplier as chat requests. Only API keys work: Sign in with ChatGPT +// tokens are rejected on /v1/decisions, and the Codex backend has no Decisions route. +// The endpoint rejects inputs above 922K tokens (the model's documented maximum input), but +// requests running longer than about five seconds, currently above roughly 600K input tokens, +// fail with a gateway timeout. +// https://developers.openai.com/api/docs/guides/decisions +const CLASSIFIER_MODELS: ClassifierModel<"openai-decisions">[] = [ + { + type: "classifier", + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-decisions", + provider: "openai", + baseUrl: BASE_URL, + input: ["text", "image"], + cost: withOpenAiLongContextPricing({ input: 0.1, output: 0, cacheRead: 0, cacheWrite: 0 }), + contextWindow: 922000, + }, +]; + +function correctUpstream(model: ChatModel): ChatModel { + if (SHORT_CONTEXT_CAPPED_MODEL_IDS.has(model.id)) { + model.contextWindow = OPENAI_LONG_CONTEXT_INPUT_THRESHOLD; + model.maxTokens = 128000; + } + if (LONG_CONTEXT_PRICING_MODEL_IDS.has(model.id)) { + model.cost = withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[model.id] ?? model.cost); + } + // models.dev reports gpt-5-pro output as 272000 (a duplicate of the input sub-limit); + // the actual max output is 128000. Also propagates to the derived Azure clone. + if (model.id === "gpt-5-pro") model.maxTokens = 128000; + return model; +} + +export default defineProvider({ + id: "openai", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "openai", { + api: "openai-responses", + baseUrl: BASE_URL, + // Audio models are only served on Chat Completions. + protocol: "responses", + include: (source) => !UNSUPPORTED_MODEL_IDS.has(source.id), + map: correctUpstream, + }), + [GPT_5_CHAT_LATEST], + ), + classifiers: () => CLASSIFIER_MODELS, + finalize: (model) => { + if (model.api !== "openai-responses") return; + const responsesModel = model as Model<"openai-responses">; + if (NONE_REASONING_MODEL_IDS.has(model.id)) mergeThinking(model, { off: "none" }); + if (model.id === "gpt-5.5") mergeThinking(model, { minimal: null }); + mergeCompat(responsesModel, { supportsStrictMode: true }); + enableOpenAiGrammarTools(model); + if (OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) { + mergeCompat(responsesModel, { supportsAdditionalTools: true, supportsToolSearch: true }); + } + enableOpenAiMidConvoSystemMessages(model, { proxy: false }); + // OpenAI charges prompt-cache writes starting with the GPT-5.6 family, and exactly + // those models accept `prompt_cache_options`; older models reject the parameter. + // https://developers.openai.com/api/docs/guides/prompt-caching + if (model.cost.cacheWrite > 0) mergeCompat(responsesModel, { supportsExplicitPromptCacheMode: true }); + if (MID_CONVO_EFFORT_MODEL_IDS.has(model.id)) mergeCompat(responsesModel, { supportsMidConvoEffort: true }); + withInputLimits(model, { maxRequestBytes: 512 * 1024 * 1024, images: { maxPerRequest: 1500 } }); + }, +}); diff --git a/packages/ai/catalog/providers/opencode-go.ts b/packages/ai/catalog/providers/opencode-go.ts new file mode 100644 index 000000000..b13b778e7 --- /dev/null +++ b/packages/ai/catalog/providers/opencode-go.ts @@ -0,0 +1,9 @@ +import { defineProvider } from "../define.ts"; +import { finalizeOpencodeModel, OPENCODE_COMPLETIONS_COMPAT, opencodeModels } from "./opencode.ts"; + +export default defineProvider({ + id: "opencode-go", + models: opencodeModels, + compat: { "openai-completions": OPENCODE_COMPLETIONS_COMPAT }, + finalize: finalizeOpencodeModel, +}); diff --git a/packages/ai/catalog/providers/opencode.ts b/packages/ai/catalog/providers/opencode.ts new file mode 100644 index 000000000..db981e4c3 --- /dev/null +++ b/packages/ai/catalog/providers/opencode.ts @@ -0,0 +1,213 @@ +import type { Api, ClassifierModel, Model, OpenAICompletionsCompat, OpenAIResponsesCompat } from "../../src/types.ts"; +import { + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + type ProviderContext, +} from "../define.ts"; +import { + enableClaudeMidConvoEffort, + supportsClaudeMidConvoEffort, + supportsClaudeMidConvoSystemMessages, +} from "../families/anthropic.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { isKimiK3FamilyId } from "../families/kimi.ts"; +import { enableOpenAiGrammarTools, enableOpenAiMidConvoSystemMessages } from "../families/openai.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import type { ModelsDevModel } from "../sources/models-dev.ts"; + +// OpenAI-compatible endpoints reject the long prompt-cache retention parameter for these models. +const LONG_CACHE_RETENTION_UNSUPPORTED_MODELS = new Set([ + "opencode:deepseek-v4-flash", + "opencode:deepseek-v4.1-flash", + "opencode:deepseek-v4-pro", + "opencode:kimi-k2.5", + "opencode:kimi-k2.6", + "opencode:minimax-m2.7", + "opencode-go:kimi-k2.6", +]); + +// OpenCode serves each model on one protocol-specific endpoint. Native protocols +// come before generic Chat Completions if models.dev ever lists several. +const OPENCODE_PROTOCOLS = ["responses", "messages", "generate-content", "chat-completions"] as const; + +interface OpencodeApi { + api: Api; + baseUrl: string; + compat?: OpenAICompletionsCompat | OpenAIResponsesCompat; +} + +/** + * The API and base URL models.dev lists for a model. Models served only on a + * protocol pi does not support fail strict generation and are skipped otherwise. + */ +function opencodeApi(ctx: ProviderContext, source: ModelsDevModel): OpencodeApi | undefined { + const entry = OPENCODE_PROTOCOLS.map((protocol) => + source.apis.find((candidate) => candidate.protocol === protocol), + ).find((candidate) => candidate !== undefined); + if (!entry) { + const protocols = source.apis.map((candidate) => candidate.protocol).join(", ") || "none"; + const message = `models.dev lists no supported API for ${ctx.providerId}/${source.id} (${protocols})`; + if (ctx.strict) throw new Error(message); + console.warn(`Skipping model: ${message}`); + return undefined; + } + switch (entry.protocol) { + case "responses": + return { + api: "openai-responses", + baseUrl: entry.baseUrl, + compat: { sessionAffinityFormat: "openai-nosession" }, + }; + case "messages": + // The Anthropic SDK appends /v1/messages itself. + return { api: "anthropic-messages", baseUrl: entry.baseUrl.replace(/\/v1\/?$/, "") }; + case "generate-content": + return { api: "google-generative-ai", baseUrl: entry.baseUrl }; + case "chat-completions": + return { + api: "openai-completions", + baseUrl: entry.baseUrl, + ...(entry.aisdk === "@ai-sdk/alibaba" ? { compat: { cacheControlFormat: "anthropic" } } : {}), + }; + } +} + +/** OpenCode Zen and Go models, on the endpoints models.dev lists for them. */ +export function opencodeModels(ctx: ProviderContext): ChatModel[] { + const provider = ctx.providerId; + return modelsDevModels(ctx, provider, { + api: "openai-completions", + // Replaced by the base URL of the API models.dev lists for each model. + baseUrl: "", + skipDeprecated: true, + // Codex Spark is not usable through OpenCode. + include: (source) => source.id !== "gpt-5.3-codex-spark", + map: (model, source) => { + const served = opencodeApi(ctx, source); + if (!served) return undefined; + let { api, baseUrl, compat } = served; + if (provider === "opencode" && model.id === "grok-build-0.1") { + compat = { ...compat, supportsReasoningEffort: false }; + } + if (model.id === "kimi-k2.6") { + // OpenCode Kimi K2.6 accepts Anthropic-style thinking objects + // and rejects string thinking values or combined reasoning_effort. + compat = { ...compat, thinkingFormat: "deepseek", supportsReasoningEffort: false }; + } + // models.dev reports these as @ai-sdk/anthropic, but OpenCode Go serves + // them through the OpenAI-compatible /openai/v1/chat/completions path. + if ( + provider === "opencode-go" && + api === "anthropic-messages" && + (model.id === "qwen3.5-plus" || model.id === "qwen3.6-plus") + ) { + api = "openai-completions"; + baseUrl = baseUrl.replace(/\/anthropic$/, "/openai/v1"); + // Qwen/DashScope uses enable_thinking at the top level. + compat = { ...compat, thinkingFormat: "qwen" }; + } + if (api === "openai-completions") { + compat = { + ...compat, + maxTokensField: "max_tokens", + ...(LONG_CACHE_RETENTION_UNSUPPORTED_MODELS.has(`${provider}:${model.id}`) + ? { supportsLongCacheRetention: false } + : {}), + }; + } + const thinkingLevelMap = + api === "google-generative-ai" + ? googleThinkingLevels(model.id, source.reasoningOptions) + : provider === "opencode-go" && model.id === "deepseek-v4.1-flash" + ? getEffortThinkingLevelMap(source.reasoningOptions ?? []) + : undefined; + const result: ChatModel = { ...model, api, baseUrl, ...(thinkingLevelMap ? { thinkingLevelMap } : {}) }; + if (compat) result.compat = compat; + // OpenCode variants list Claude Sonnet 4/4.5 with 1M context, actual limit is 200K. + if (model.id === "claude-sonnet-4-5" || model.id === "claude-sonnet-4") result.contextWindow = 200000; + if (["claude-opus-4-6", "claude-sonnet-4-6", "claude-opus-4.6", "claude-sonnet-4.6"].includes(model.id)) { + result.contextWindow = 1000000; + } + if (model.id === "gpt-5.4") { + result.contextWindow = 272000; + result.maxTokens = 128000; + } + return result; + }, + }); +} + +export const OPENCODE_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsStrictMode: true, +} as const; + +/** Behavior shared by OpenCode Zen and Go. */ +export function finalizeOpencodeModel(model: ChatModel): void { + if (model.provider === "opencode-go" && model.id === "glm-5.2") { + mergeThinking(model, { off: null, minimal: null, low: null, medium: null, high: "high", max: "max" }); + } + if (model.provider === "opencode-go" && model.id === "kimi-k2.6") { + // OpenCode Go exposes Kimi K2.6 thinking as on/off, not distinct effort tiers. + mergeThinking(model, { minimal: null, low: null, medium: null }); + } + if (model.provider === "opencode" && model.id === "grok-build-0.1") { + // OpenCode Zen Grok Build reasons by default but rejects explicit reasoningEffort. + mergeThinking(model, { off: null, minimal: null, low: null, medium: null }); + } + if (model.api === "anthropic-messages") { + const messagesModel = model as Model<"anthropic-messages">; + // OpenCode Zen forwards mid-conversation system messages but rejects + // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level. + if (model.provider === "opencode" && supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true }); + } + // Zen and Go forward per-message effort and `thinking.block_binding`. Without + // `drop_block`, Go rejects replayed thinking after the system prompt or tools change. + if (supportsClaudeMidConvoEffort(model.id)) enableClaudeMidConvoEffort(messagesModel); + // OpenCode Qwen 3.8 Flash emits and accepts thinking blocks with empty signatures. + if (model.id === "qwen3.8-flash") mergeCompat(messagesModel, { allowEmptySignature: true }); + } + // The Kimi K3 family accepts tool-bearing system messages; OpenCode passes them through. + if (model.api === "openai-completions" && isKimiK3FamilyId(model.id)) { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + supportsMidConvoToolAdditions: true, + }); + } + // OpenCode passes mid-conversation developer messages and `additional_tools` + // through to OpenAI unchanged; tool search is not verified through it. + enableOpenAiMidConvoSystemMessages(model, { proxy: true }); +} + +// OpenCode Zen serves Jev through its TypeSafe-compatible System One endpoint. +// Neither its /inference/openai/v1/models listing nor models.dev carries metadata for it. +// https://opencode.ai/docs/zen +function jev(id: string, name: string, input: number): ClassifierModel<"typesafe-system-one"> { + return { + type: "classifier", + id, + name, + api: "typesafe-system-one", + provider: "opencode", + baseUrl: "https://opencode.ai/inference/systemone/v1", + input: ["text"], + cost: { input, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }; +} + +export default defineProvider({ + id: "opencode", + models: opencodeModels, + classifiers: () => [jev("jev-1.13", "Jev 1.13", 0.042), jev("jev-1.13-free", "Jev 1.13 Free", 0)], + compat: { "openai-completions": OPENCODE_COMPLETIONS_COMPAT }, + finalize: (model) => { + finalizeOpencodeModel(model); + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/openrouter.ts b/packages/ai/catalog/providers/openrouter.ts new file mode 100644 index 000000000..6d705af93 --- /dev/null +++ b/packages/ai/catalog/providers/openrouter.ts @@ -0,0 +1,108 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { addMissing, type ChatModel, defineProvider, mergeCompat, mergeThinking } from "../define.ts"; +import { enableClaudeMidConvoEffort, supportsClaudeMidConvoEffort } from "../families/anthropic.ts"; +import { isKimiK3FamilyId, KIMI_K3_MAX_TOKENS } from "../families/kimi.ts"; +import { OPENAI_MID_CONVO_MODEL_IDS } from "../families/openai.ts"; + +const BASE_URL = "https://openrouter.ai/api/v1"; +// OpenRouter rejects `configuration_update` system messages on Opus 5 ("Mid-conversation +// reasoning effort (configuration_update) is not supported on anthropic/claude-opus-5-20260723") +// while accepting them on Fable 5.1. +const MID_CONVO_EFFORT_UNSUPPORTED_MODEL_IDS = new Set(["anthropic/claude-opus-5"]); + +/** Keep selected model metadata stable until upstream settles. */ +function correctUpstream(model: ChatModel): ChatModel { + // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. + if ( + model.id === "~moonshotai/kimi-latest" || + (model.id.startsWith("moonshotai/") && isKimiK3FamilyId(model.id.slice("moonshotai/".length))) + ) { + model.maxTokens = KIMI_K3_MAX_TOKENS; + } + if (model.id === "moonshotai/kimi-k2.5") { + model.cost = { ...model.cost, input: 0.41, output: 2.06, cacheRead: 0.07 }; + model.maxTokens = 4096; + } + if (model.id.startsWith("moonshotai/kimi-k2.6")) { + mergeCompat(model as Model<"openai-completions">, { + supportsDeveloperRole: false, + requiresReasoningContentOnAssistantMessages: true, + }); + } + if (model.id === "z-ai/glm-5") model.cost = { ...model.cost, input: 0.6, output: 1.9, cacheRead: 0.119 }; + return model; +} + +// Router aliases. Their price depends on the model they route to, so it is unknown here. +const ROUTER_MODELS: Model<"openai-completions">[] = [ + { + id: "auto", + name: "Auto", + api: "openai-completions", + provider: "openrouter", + baseUrl: BASE_URL, + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 2000000, + maxTokens: 30000, + }, + // OpenRouter exposes Fusion as a router alias/plugin entry point; its model + // metadata does not advertise tools, but the alias resolves to a concrete model + // that can invoke caller tools and has the openrouter:fusion server tool auto-injected. + { + id: "openrouter/fusion", + name: "OpenRouter: Fusion", + api: "openai-completions", + provider: "openrouter", + baseUrl: BASE_URL, + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 30000, + }, +]; + +export default defineProvider({ + id: "openrouter", + models: (ctx) => + addMissing(structuredClone(ctx.sources.openRouter.chat).map(correctUpstream), structuredClone(ROUTER_MODELS)), + images: (ctx) => structuredClone(ctx.sources.openRouter.images), + classifiers: (ctx) => structuredClone(ctx.sources.openRouter.classifiers), + compat: { + "openai-completions": (model): OpenAICompletionsCompat => ({ + // OpenRouter passes developer messages through to Anthropic and OpenAI models only. + ...(/^(anthropic|openai)\//.test(model.id) ? {} : { supportsDeveloperRole: false }), + thinkingFormat: "openrouter", + supportsStrictMode: true, + ...(/^~?anthropic\//.test(model.id) ? { cacheControlFormat: "anthropic" } : {}), + sendSessionAffinityHeaders: true, + }), + }, + finalize: (model) => { + if ( + model.api === "anthropic-messages" && + supportsClaudeMidConvoEffort(model.id) && + !MID_CONVO_EFFORT_UNSUPPORTED_MODEL_IDS.has(model.id) + ) { + enableClaudeMidConvoEffort(model as Model<"anthropic-messages">); + } + if (model.id.startsWith("inception/mercury-2")) { + // Mercury 2 in instant mode (reasoning_effort: "none") disables tool calling. + // Mark "off" unsupported so the openai-completions provider omits the reasoning param + // instead of defaulting to {reasoning:{effort:"none"}}. + // Pi's low/medium/high pass through verbatim; OpenRouter normalizes to Mercury's vocabulary. + mergeThinking(model, { off: null }); + } + if (model.id === "z-ai/glm-5.2") mergeThinking(model, { xhigh: "xhigh" }); + // OpenAI models behind OpenRouter accept plain system text in place. + if ( + model.api === "openai-completions" && + model.id.startsWith("openai/") && + OPENAI_MID_CONVO_MODEL_IDS.has(model.id.slice("openai/".length)) + ) { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan-cn.ts b/packages/ai/catalog/providers/qwen-token-plan-cn.ts new file mode 100644 index 000000000..33b210dd9 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan-cn.ts @@ -0,0 +1,13 @@ +import { defineProvider } from "../define.ts"; +import { QWEN_COMPLETIONS_COMPAT, qwenTokenPlanModels } from "./qwen-token-plan.ts"; + +export default defineProvider({ + id: "qwen-token-plan-cn", + models: (ctx) => + qwenTokenPlanModels( + ctx, + "alibaba-token-plan-cn", + "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", + ), + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan-individual.ts b/packages/ai/catalog/providers/qwen-token-plan-individual.ts new file mode 100644 index 000000000..c172748d1 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan-individual.ts @@ -0,0 +1,34 @@ +import { assertExactModelIds } from "../../scripts/model-data.ts"; +import { defineProvider } from "../define.ts"; +import { INTERNATIONAL_BASE_URL, QWEN_COMPLETIONS_COMPAT, qwenTokenPlanModels } from "./qwen-token-plan.ts"; + +// QwenCloud Token Plan Individual text-model allowlist, verified 2026-09-03. +// It shares the international endpoint with a narrower catalog. +// https://docs.qwencloud.com/token-plan/personal/token-plan-personal-overview +const MODEL_IDS = new Set([ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-flash", + "qwen3.8-max", +]); + +export default defineProvider({ + id: "qwen-token-plan-individual", + models: (ctx) => { + const models = qwenTokenPlanModels(ctx, "alibaba-token-plan", INTERNATIONAL_BASE_URL, MODEL_IDS); + if (ctx.strict) { + assertExactModelIds( + ctx.providerId, + MODEL_IDS, + models.map((model) => model.id), + ); + } + return models; + }, + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan.ts b/packages/ai/catalog/providers/qwen-token-plan.ts new file mode 100644 index 000000000..b526a9e28 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan.ts @@ -0,0 +1,53 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Alibaba Cloud Model Studio Token Plan. International and China use separate +// endpoints and API keys (sk-sp- prefix). models.dev lists them as +// "alibaba-token-plan[-cn]". +export const INTERNATIONAL_BASE_URL = "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1"; + +const COMPAT: OpenAICompletionsCompat = { + thinkingFormat: "qwen", + supportsDeveloperRole: false, + supportsStore: false, + supportsReasoningEffort: true, +}; +const FALLBACK_THINKING_LEVELS = { + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", +} as const; +const REASONING_EFFORT_FALLBACK_MODEL_IDS = new Set(["glm-5", "glm-5.1"]); +// Retired preview id — models.dev may still list it after GA ships. +const EXCLUDED_MODEL_IDS = new Set(["qwen3.8-max-preview"]); + +export function qwenTokenPlanModels(ctx: ProviderContext, source: string, baseUrl: string, modelIds?: Set) { + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + recordReasoning: false, + include: (model) => !EXCLUDED_MODEL_IDS.has(model.id) && (!modelIds || modelIds.has(model.id)), + map: (model, source): Model<"openai-completions"> => { + const thinkingLevelMap = + getEffortThinkingLevelMap(source.reasoningOptions ?? []) ?? + (REASONING_EFFORT_FALLBACK_MODEL_IDS.has(model.id) ? FALLBACK_THINKING_LEVELS : undefined); + return { + ...model, + compat: thinkingLevelMap ? COMPAT : { ...COMPAT, supportsReasoningEffort: false }, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + }; + }, + }); +} + +export const QWEN_COMPLETIONS_COMPAT = { supportsStrictMode: true } as const; + +export default defineProvider({ + id: "qwen-token-plan", + models: (ctx) => qwenTokenPlanModels(ctx, "alibaba-token-plan", INTERNATIONAL_BASE_URL), + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/radius.ts b/packages/ai/catalog/providers/radius.ts new file mode 100644 index 000000000..3c1b9deb4 --- /dev/null +++ b/packages/ai/catalog/providers/radius.ts @@ -0,0 +1,7 @@ +import { defineProvider } from "../define.ts"; + +// The unauthenticated public catalog; authenticated clients overlay it at runtime. +export default defineProvider({ + id: "radius", + models: (ctx) => structuredClone(ctx.sources.radius), +}); diff --git a/packages/ai/catalog/providers/together.ts b/packages/ai/catalog/providers/together.ts new file mode 100644 index 000000000..30cea723b --- /dev/null +++ b/packages/ai/catalog/providers/together.ts @@ -0,0 +1,58 @@ +import type { OpenAICompletionsCompat, ThinkingLevelMap } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; + +const BASE_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, +}; +const TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { ...BASE_COMPAT, thinkingFormat: "together" }; +const REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + supportsReasoningEffort: true, + thinkingFormat: "openai", +}; +const TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...TOGGLE_REASONING_COMPAT, + supportsReasoningEffort: true, +}; + +const REASONING_EFFORT_MODELS = new Set(["openai/gpt-oss-20b", "openai/gpt-oss-120b"]); +const TOGGLE_REASONING_EFFORT_MODELS = new Set(["deepseek-ai/DeepSeek-V4-Pro-0813"]); + +const REASONING_EFFORT_LEVELS: ThinkingLevelMap = { off: null, minimal: null }; +const DEEPSEEK_V4_LEVELS: ThinkingLevelMap = { minimal: null, low: null, medium: null, high: "high", xhigh: null }; +const TOGGLE_REASONING_LEVELS: ThinkingLevelMap = { minimal: null, low: null, medium: null }; + +export default defineProvider({ + id: "together", + models: (ctx) => + modelsDevModels(ctx, "togetherai", { + api: "openai-completions", + baseUrl: "https://api.together.ai/v1", + skipDeprecated: true, + map: (model) => { + if (!model.reasoning) return { ...model, compat: BASE_COMPAT }; + if (REASONING_EFFORT_MODELS.has(model.id)) { + return { ...model, compat: REASONING_EFFORT_COMPAT, thinkingLevelMap: { ...REASONING_EFFORT_LEVELS } }; + } + if (TOGGLE_REASONING_EFFORT_MODELS.has(model.id)) { + return { ...model, compat: TOGGLE_REASONING_EFFORT_COMPAT, thinkingLevelMap: { ...DEEPSEEK_V4_LEVELS } }; + } + return { ...model, compat: TOGGLE_REASONING_COMPAT, thinkingLevelMap: { ...TOGGLE_REASONING_LEVELS } }; + }, + }), + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "together", + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/typesafe.ts b/packages/ai/catalog/providers/typesafe.ts new file mode 100644 index 000000000..384c19831 --- /dev/null +++ b/packages/ai/catalog/providers/typesafe.ts @@ -0,0 +1,32 @@ +import type { ClassifierModel } from "../../src/types.ts"; +import { defineProvider } from "../define.ts"; + +export default defineProvider({ + id: "typesafe", + classifiers: (ctx): ClassifierModel<"typesafe-system-one">[] => { + const metadata = ctx.sources.modelsDevDecisions["typesafe/jev-latest"]; + if (!metadata || metadata.type !== "decision") { + if (ctx.strict) throw new Error("models.dev did not return decision model typesafe/jev-latest"); + return []; + } + return [ + { + type: "classifier", + id: "jev-latest", + name: metadata.name, + api: "typesafe-system-one", + provider: "typesafe", + baseUrl: "https://api.typesafe.ai/v1/", + // models.dev lists text input only, but the API accepts image assets as an experimental + // feature behind an opt-in header, which the typesafe-system-one API sends. + // https://github.com/typesafe-ai/typesafe-sdk-js (v0.7.0 changelog) + input: ["text", "image"], + // The canonical models.dev entry has no direct-provider pricing. System One reports token usage, + // so classify() results carry token counts but price them at zero. + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: metadata.limit?.context || 64000, + compat: { imageInput: "assets", sendImageOptInHeader: true }, + }, + ]; + }, +}); diff --git a/packages/ai/catalog/providers/vercel-ai-gateway.ts b/packages/ai/catalog/providers/vercel-ai-gateway.ts new file mode 100644 index 000000000..e414e235c --- /dev/null +++ b/packages/ai/catalog/providers/vercel-ai-gateway.ts @@ -0,0 +1,14 @@ +import { defineProvider } from "../define.ts"; +import { isKimiK3FamilyId, KIMI_K3_MAX_TOKENS } from "../families/kimi.ts"; + +export default defineProvider({ + id: "vercel-ai-gateway", + models: (ctx) => + structuredClone(ctx.sources.aiGateway.chat).map((model) => + // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. + model.id.startsWith("moonshotai/") && isKimiK3FamilyId(model.id.slice("moonshotai/".length)) + ? { ...model, maxTokens: KIMI_K3_MAX_TOKENS } + : model, + ), + classifiers: (ctx) => structuredClone(ctx.sources.aiGateway.classifiers), +}); diff --git a/packages/ai/catalog/providers/xai.ts b/packages/ai/catalog/providers/xai.ts new file mode 100644 index 000000000..1a189f82b --- /dev/null +++ b/packages/ai/catalog/providers/xai.ts @@ -0,0 +1,29 @@ +import { defineProvider, mergeThinking, modelsDevModels } from "../define.ts"; + +// Retired or unsuitable for pi's built-in catalog. +const EXCLUDED_MODEL_IDS = new Set([ + "grok-3", + "grok-3-fast", + "grok-4.20-0309-non-reasoning", + "grok-4.20-0309-reasoning", + "grok-build-0.1", + "grok-code-fast-1", +]); + +export default defineProvider({ + id: "xai", + models: (ctx) => + modelsDevModels(ctx, "xai", { + api: "openai-responses", + baseUrl: "https://api.x.ai/v1", + compat: { supportsLongCacheRetention: false }, + include: (source) => !EXCLUDED_MODEL_IDS.has(source.id), + }), + finalize: (model) => { + // Models without verified effort options must not send the undocumented + // "none"/"minimal" efforts. + if (model.api === "openai-responses" && model.thinkingLevelMap === undefined) { + mergeThinking(model, { off: null, minimal: null }); + } + }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts b/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts new file mode 100644 index 000000000..6f8c6ce12 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-ams", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-ams.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts b/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts new file mode 100644 index 000000000..a7a371860 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-cn", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-cn.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts b/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts new file mode 100644 index 000000000..d3614b3ad --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-sgp", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-sgp.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi.ts b/packages/ai/catalog/providers/xiaomi.ts new file mode 100644 index 000000000..3129bf139 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi.ts @@ -0,0 +1,46 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; + +/** + * Xiaomi MiMo. The `xiaomi` provider targets the API billing endpoint; the + * `xiaomi-token-plan-*` providers cover prepaid Token Plan endpoints per region. + * MiMo Chat Completions only documents thinking.type enabled/disabled and + * rejects some reasoning_effort values (e.g. "minimal"). Xiaomi documents all + * effort levels as identical, so expose only off and high. + */ +export function xiaomiModels(ctx: ProviderContext, baseUrl: string) { + return modelsDevModels(ctx, ctx.providerId, { + api: "openai-completions", + baseUrl, + skipDeprecated: true, + recordReasoning: false, + compat: { + supportsReasoningEffort: false, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", + }, + map: (model): Model<"openai-completions"> => + model.reasoning + ? { + ...model, + thinkingLevelMap: { + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: null, + }, + } + : model, + }); +} + +export const XIAOMI_COMPLETIONS_COMPAT = { supportsStrictMode: true } as const; + +export default defineProvider({ + id: "xiaomi", + models: (ctx) => xiaomiModels(ctx, "https://api.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/zai-coding-cn.ts b/packages/ai/catalog/providers/zai-coding-cn.ts new file mode 100644 index 000000000..2321572ec --- /dev/null +++ b/packages/ai/catalog/providers/zai-coding-cn.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { ZAI_COMPLETIONS_COMPAT, zaiCodingPlanModels } from "./zai.ts"; + +export default defineProvider({ + id: "zai-coding-cn", + models: (ctx) => zaiCodingPlanModels(ctx, "zhipuai-coding-plan", "https://open.bigmodel.cn/api/coding/paas/v4"), + compat: { "openai-completions": ZAI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/zai.ts b/packages/ai/catalog/providers/zai.ts new file mode 100644 index 000000000..540d30d79 --- /dev/null +++ b/packages/ai/catalog/providers/zai.ts @@ -0,0 +1,50 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import { modelsDevChatModels, modelsDevCost } from "../sources/models-dev.ts"; + +// Tool-call streaming (`tool_stream`) is rejected by the GLM-4.5 generation. +const TOOL_STREAM_UNSUPPORTED_MODEL_IDS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); + +/** The Z.AI coding plans. The international and China plans list the same models. */ +export function zaiCodingPlanModels(ctx: ProviderContext, source: string, baseUrl: string) { + // Coding-plan entries carry plan prices; show the pay-as-you-go API rates. + const apiModels = new Map(modelsDevChatModels(ctx.sources.modelsDev, "zai").map((model) => [model.id, model])); + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + map: (model, source): Model<"openai-completions"> => { + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + if (thinkingLevelMap && (model.id === "glm-5.2" || model.id === "glm-5.2-highspeed")) { + thinkingLevelMap.off = "none"; + } + const apiModel = apiModels.get(model.id); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + cost: modelsDevCost(apiModel?.source.cost ? apiModel.cost : source.cost), + compat: { + supportsDeveloperRole: false, + thinkingFormat: "zai", + ...(thinkingLevelMap ? { supportsReasoningEffort: true } : {}), + ...(!TOOL_STREAM_UNSUPPORTED_MODEL_IDS.has(model.id) ? { zaiToolStream: true } : {}), + }, + }; + }, + }); +} + +export const ZAI_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "zai", + supportsStrictMode: true, +} as const; + +export default defineProvider({ + id: "zai", + models: (ctx) => zaiCodingPlanModels(ctx, "zai-coding-plan", "https://api.z.ai/api/coding/paas/v4"), + compat: { "openai-completions": ZAI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/scripts/models-dev-reasoning-options.ts b/packages/ai/catalog/reasoning.ts similarity index 79% rename from packages/ai/scripts/models-dev-reasoning-options.ts rename to packages/ai/catalog/reasoning.ts index 9d0711395..389f39382 100644 --- a/packages/ai/scripts/models-dev-reasoning-options.ts +++ b/packages/ai/catalog/reasoning.ts @@ -1,12 +1,5 @@ import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; - -export type ModelsDevReasoningOption = - | { type: "toggle" } - | { - type: "effort"; - values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>; - } - | { type: "budget_tokens"; min?: number; max?: number }; +import type { ModelsDevReasoningOption } from "./sources/models-dev.ts"; const THINKING_LEVELS: readonly ThinkingLevel[] = ["minimal", "low", "medium", "high", "xhigh", "max"]; diff --git a/packages/ai/catalog/sources/index.ts b/packages/ai/catalog/sources/index.ts new file mode 100644 index 000000000..b043778b9 --- /dev/null +++ b/packages/ai/catalog/sources/index.ts @@ -0,0 +1,95 @@ +import { + DEFAULT_RADIUS_GATEWAY, + getRadiusModelsFromConfig, + loadRadiusGatewayConfig, +} from "../../src/providers/radius-config.ts"; +import type { Model } from "../../src/types.ts"; +import { + MODELS_DEV_DECISIONS_URL, + MODELS_DEV_URL, + type ModelsDevDecisionMetadata, + type ModelsDevV2Catalog, +} from "./models-dev.ts"; +import { fetchOpenRouterCatalog, type OpenRouterCatalog } from "./openrouter.ts"; +import { type AiGatewayCatalog, fetchAiGatewayCatalog } from "./vercel-ai-gateway.ts"; + +export const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; + +/** Everything fetched from upstream before the catalog is built. */ +export interface CatalogSources { + modelsDev: ModelsDevV2Catalog; + modelsDevDecisions: Record; + openRouter: OpenRouterCatalog; + aiGateway: AiGatewayCatalog; + /** Live NVIDIA NIM ids, keyed by both the raw and the normalized id. */ + nvidiaModelIds: Map; + radius: Model<"pi-messages">[]; +} + +async function fetchJson(url: string, label: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`${label} returned ${response.status}`); + return (await response.json()) as T; +} + +export function normalizeNvidiaModelId(modelId: string): string { + return modelId.toLowerCase().replaceAll("_", "."); +} + +async function fetchNvidiaModelIds(): Promise> { + const data = await fetchJson<{ data?: { id: string }[] }>(`${NVIDIA_BASE_URL}/models`, "NVIDIA NIM API"); + const modelIds = new Map(); + for (const model of data.data ?? []) { + modelIds.set(model.id, model.id); + modelIds.set(normalizeNvidiaModelId(model.id), model.id); + } + return modelIds; +} + +async function fetchRadiusModels(): Promise[]> { + const config = await loadRadiusGatewayConfig(DEFAULT_RADIUS_GATEWAY); + const models = getRadiusModelsFromConfig("radius", config); + if (models.length === 0) throw new Error("Radius API returned no models"); + return models; +} + +/** + * Fetch every upstream source. In strict mode any failure aborts generation; + * otherwise a failed source contributes nothing. + */ +export async function loadCatalogSources(strict: boolean): Promise { + async function load(label: string, fetcher: () => Promise, fallback: T): Promise { + try { + console.log(`Fetching ${label}...`); + return await fetcher(); + } catch (error) { + console.error(`Failed to fetch ${label}:`, error); + if (strict) throw error; + return fallback; + } + } + + const modelsDev = await load( + "models.dev catalog", + () => fetchJson(MODELS_DEV_URL, "models.dev API"), + {}, + ); + const modelsDevDecisions = await load( + "models.dev decision models", + () => fetchJson>(MODELS_DEV_DECISIONS_URL, "models.dev decision API"), + {}, + ); + const nvidiaModelIds = modelsDev.nvidia?.models + ? await load("NVIDIA NIM models", fetchNvidiaModelIds, new Map()) + : new Map(); + const openRouter = await load("OpenRouter models", fetchOpenRouterCatalog, { + chat: [], + images: [], + classifiers: [], + }); + if (strict && openRouter.images.length === 0) throw new Error("OpenRouter API returned no usable image models"); + const aiGateway = await load("Vercel AI Gateway models", fetchAiGatewayCatalog, { chat: [], classifiers: [] }); + const radius = await load("Radius models", fetchRadiusModels, []); + + return { modelsDev, modelsDevDecisions, openRouter, aiGateway, nvidiaModelIds, radius }; +} diff --git a/packages/ai/catalog/sources/models-dev.ts b/packages/ai/catalog/sources/models-dev.ts new file mode 100644 index 000000000..3928dc438 --- /dev/null +++ b/packages/ai/catalog/sources/models-dev.ts @@ -0,0 +1,242 @@ +import type { ModelCost } from "../../src/types.ts"; + +// models.dev v2 catalog. The schema is published as experimental, so everything +// the catalog reads goes through `normalizeModelsDevModel()` below. +// https://github.com/anomalyco/models.dev/blob/dev/packages/core/src/schema-v2.ts +export const MODELS_DEV_URL = "https://models.dev/experimental/v2.0/api.json"; +// v2 has no provider-agnostic decision metadata endpoint; v1 still serves it. +export const MODELS_DEV_DECISIONS_URL = "https://models.dev/models.json?type=decision"; + +export type ModelsDevReasoningOption = + | { type: "toggle" } + | { + type: "effort"; + values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>; + } + | { type: "budget_tokens"; min?: number; max?: number }; + +type ModelsDevEffort = Extract["values"][number]; + +interface V2CachePrices { + read?: number; + write?: number | Record; + explicit?: { read?: number; write?: number | Record }; +} + +interface V2CostOverride { + when: Record; + price?: number; + cache?: V2CachePrices; +} + +interface V2ModalityCost { + input?: { unit?: string; price: number; cache?: V2CachePrices; overrides?: V2CostOverride[] }; + output?: { unit?: string; price: number; overrides?: V2CostOverride[] }; +} + +interface V2ApiEntry { + base_url: string; + path?: string; + sdk?: { aisdk?: string }; + compatibility?: Record; + capabilities?: Record; +} + +/** The subset of a models.dev v2 model the catalog reads. */ +export interface ModelsDevV2Model { + id: string; + type: string; + name: string; + status?: string; + modalities: { input: string[]; output: string[] }; + capabilities: { + tools: { supported: boolean }; + reasoning: { + supported: boolean; + toggle?: boolean; + effort?: ModelsDevEffort[]; + budget?: { min?: number; max?: number }; + }; + structured_output?: boolean; + }; + limit?: { context?: number; input?: number; output?: number }; + cost?: { text?: V2ModalityCost }; + api: Record; +} + +export type ModelsDevV2Catalog = Record }>; + +/** Prices as models.dev lists them. Missing prices stay undefined. */ +export interface ModelsDevCost { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + /** Context-size tiers. Unlisted rates fall back to the base rates. */ + tiers?: Array<{ above: number; input?: number; output?: number; cacheRead?: number; cacheWrite?: number }>; +} + +/** One API a model is served on, in the order models.dev lists them. */ +export interface ModelsDevApi { + protocol: string; + baseUrl: string; + aisdk?: string; +} + +/** A models.dev model normalized for the catalog. */ +export interface ModelsDevModel { + id: string; + name: string; + type: string; + status?: string; + toolCall: boolean; + structuredOutput: boolean; + reasoning: boolean; + /** Undefined when the model does not reason. */ + reasoningOptions?: ModelsDevReasoningOption[]; + imageInput: boolean; + textInput: boolean; + textOutput: boolean; + contextWindow?: number; + maxTokens?: number; + cost: ModelsDevCost; + apis: ModelsDevApi[]; + /** The full v2 entry, for fields not normalized above. */ + source: ModelsDevV2Model; +} + +/** Provider-agnostic decision model metadata from models.dev. */ +export interface ModelsDevDecisionMetadata { + id: string; + type?: string; + name: string; + limit?: { context?: number }; +} + +function firstPrice(write: number | Record | undefined): number | undefined { + return typeof write === "object" ? Object.values(write)[0] : write; +} + +// models.dev lists one cache write price per TTL; the first is the host's default. +function cacheWritePrice(cache: V2CachePrices | undefined): number | undefined { + return firstPrice(cache?.write) ?? firstPrice(cache?.explicit?.write); +} + +function contextThreshold(when: Record): number | undefined { + const keys = Object.keys(when); + if (keys.length !== 1 || keys[0] !== "context") return undefined; + const context = when.context as { gt?: unknown; gte?: unknown }; + const size = context.gt ?? context.gte; + return typeof size === "number" ? size : undefined; +} + +function normalizeCost(cost: ModelsDevV2Model["cost"]): ModelsDevCost { + const input = cost?.text?.input; + const output = cost?.text?.output; + const perToken = (unit: string | undefined) => unit === undefined || unit === "token"; + if (!input || !output || !perToken(input.unit) || !perToken(output.unit)) return {}; + + // Only context-size overrides map onto pi's cost tiers. Region, service-tier, + // and time-based prices have no pi equivalent yet. + const tiers = new Map[number]>(); + const tier = (above: number) => { + let entry = tiers.get(above); + if (!entry) { + entry = { above }; + tiers.set(above, entry); + } + return entry; + }; + for (const override of input.overrides ?? []) { + const above = contextThreshold(override.when); + if (above === undefined) continue; + const entry = tier(above); + if (override.price !== undefined) entry.input = override.price; + if (override.cache?.read !== undefined) entry.cacheRead = override.cache.read; + const cacheWrite = cacheWritePrice(override.cache); + if (cacheWrite !== undefined) entry.cacheWrite = cacheWrite; + } + for (const override of output.overrides ?? []) { + const above = contextThreshold(override.when); + if (above === undefined) continue; + if (override.price !== undefined) tier(above).output = override.price; + } + + return { + input: input.price, + output: output.price, + cacheRead: input.cache?.read, + cacheWrite: cacheWritePrice(input.cache), + ...(tiers.size > 0 ? { tiers: [...tiers.values()] } : {}), + }; +} + +function reasoningOptions(reasoning: ModelsDevV2Model["capabilities"]["reasoning"]): ModelsDevReasoningOption[] { + const options: ModelsDevReasoningOption[] = []; + if (reasoning.toggle) options.push({ type: "toggle" }); + if (reasoning.effort !== undefined) options.push({ type: "effort", values: reasoning.effort }); + if (reasoning.budget !== undefined) options.push({ type: "budget_tokens", ...reasoning.budget }); + return options; +} + +export function normalizeModelsDevModel(model: ModelsDevV2Model): ModelsDevModel { + const input = model.modalities?.input ?? []; + const reasoning = model.capabilities?.reasoning; + return { + id: model.id, + name: model.name, + type: model.type, + status: model.status, + toolCall: model.capabilities?.tools?.supported === true, + structuredOutput: model.capabilities?.structured_output === true, + reasoning: reasoning?.supported === true, + reasoningOptions: reasoning?.supported ? reasoningOptions(reasoning) : undefined, + imageInput: input.some((modality) => modality === "image" || modality.startsWith("image/")), + textInput: input.includes("text"), + textOutput: (model.modalities?.output ?? []).includes("text"), + contextWindow: model.limit?.context, + maxTokens: model.limit?.output, + cost: normalizeCost(model.cost), + apis: Object.entries(model.api ?? {}).map(([protocol, entry]) => ({ + protocol, + baseUrl: entry.base_url, + aisdk: entry.sdk?.aisdk, + })), + source: model, + }; +} + +export function roundCost(value: number): number { + return Number(value.toFixed(6)); +} + +/** Converts models.dev prices to pi's cost shape. Missing rates are free. */ +export function modelsDevCost(cost: ModelsDevCost): ModelCost { + const base = { + input: cost.input || 0, + output: cost.output || 0, + cacheRead: cost.cacheRead || 0, + cacheWrite: cost.cacheWrite || 0, + }; + const tiers = cost.tiers?.map((tier) => ({ + inputTokensAbove: tier.above, + input: tier.input ?? base.input, + output: tier.output ?? base.output, + cacheRead: tier.cacheRead ?? base.cacheRead, + cacheWrite: tier.cacheWrite ?? base.cacheWrite, + })); + return tiers && tiers.length > 0 ? { ...base, tiers } : base; +} + +/** + * Chat models of one models.dev provider. Specialized types (image, speech, + * realtime, decision, ...) only appear in v2 and are left out here. + */ +export function modelsDevChatModels(catalog: ModelsDevV2Catalog, provider: string): ModelsDevModel[] { + return ( + Object.values(catalog[provider]?.models ?? {}) + // Unlike pi models, models.dev v2 always states the type. + .filter(({ type }) => type === "chat") + .map(normalizeModelsDevModel) + ); +} diff --git a/packages/ai/scripts/openrouter-catalog.ts b/packages/ai/catalog/sources/openrouter.ts similarity index 63% rename from packages/ai/scripts/openrouter-catalog.ts rename to packages/ai/catalog/sources/openrouter.ts index 8f01a14e6..a7e82a735 100644 --- a/packages/ai/scripts/openrouter-catalog.ts +++ b/packages/ai/catalog/sources/openrouter.ts @@ -1,6 +1,29 @@ -import { type OpenRouterModel, openRouterCost } from "../src/providers/openrouter-api.ts"; -import type { ClassifierModel, ImageModel, Model } from "../src/types.ts"; -import { getOpenRouterThinkingLevelMap, type OpenRouterReasoningMetadata } from "./openrouter-reasoning-options.ts"; +import { type OpenRouterModel, openRouterCost } from "../../src/providers/openrouter-api.ts"; +import type { ClassifierModel, ImageModel, Model, ThinkingLevel, ThinkingLevelMap } from "../../src/types.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"; + +export interface OpenRouterReasoningMetadata { + mandatory?: boolean; + default_enabled?: boolean; + supported_efforts?: Array; + default_effort?: ThinkingLevel | "none"; +} + +/** Convert OpenRouter's reasoning metadata into Pi model capabilities. */ +export function getOpenRouterThinkingLevelMap( + reasoning: OpenRouterReasoningMetadata | undefined, +): ThinkingLevelMap | undefined { + if (!reasoning) return undefined; + if (!reasoning.supported_efforts?.length) return reasoning.mandatory === true ? { off: null } : undefined; + + // OpenRouter's supported_efforts uses the same effort values as models.dev reasoning_options, + // so both sources can share the same Pi thinking-level conversion. + const map = getEffortThinkingLevelMap([{ type: "effort", values: reasoning.supported_efforts }]); + if (!map) return reasoning.mandatory === true ? { off: null } : undefined; + return { ...map, off: reasoning.mandatory === true ? null : "none" }; +} export interface OpenRouterModelListItem extends OpenRouterModel { name: string; @@ -45,7 +68,8 @@ export function buildOpenRouterCatalog( } const thinkingLevelMap = getOpenRouterThinkingLevelMap(model.reasoning); - const useAnthropicMessages = /^anthropic\//.test(model.id) && !model.id.endsWith(":batch"); + // Includes the `~anthropic/claude-*-latest` aliases, which OpenRouter's Messages endpoint accepts. + const useAnthropicMessages = /^~?anthropic\//.test(model.id) && !model.id.endsWith(":batch"); chat.push({ type: "chat", id: model.id, @@ -105,3 +129,20 @@ export function buildOpenRouterCatalog( return { chat, images, classifiers }; } + +async function fetchOpenRouterList(query: string): Promise { + const response = await fetch(`${OPENROUTER_MODELS_URL}${query}`); + if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); + const data = (await response.json()) as { data?: OpenRouterModelListItem[] }; + return data.data ?? []; +} + +/** Fetch the tool-capable chat, image, and decision listings. */ +export async function fetchOpenRouterCatalog(): Promise { + const [listed, imageListed, decisionListed] = await Promise.all([ + fetchOpenRouterList(""), + fetchOpenRouterList("?output_modalities=image"), + fetchOpenRouterList("?output_modalities=decisions"), + ]); + return buildOpenRouterCatalog(listed, imageListed, decisionListed); +} diff --git a/packages/ai/catalog/sources/vercel-ai-gateway.ts b/packages/ai/catalog/sources/vercel-ai-gateway.ts new file mode 100644 index 000000000..3da44b2d8 --- /dev/null +++ b/packages/ai/catalog/sources/vercel-ai-gateway.ts @@ -0,0 +1,140 @@ +import type { ClassifierModel, Model, ModelCost, ModelCostRates, ModelCostTier } from "../../src/types.ts"; + +const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models"; +const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; +// TypeSafe-compatible System One endpoint for evaluation models. +// https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe +const AI_GATEWAY_TYPESAFE_BASE_URL = "https://ai-gateway.vercel.sh/typesafe/v1"; + +/** A prompt-length bracket. `min` is inclusive and `max` exclusive, both in prompt tokens. */ +export interface AiGatewayPriceTier { + cost?: string | number; + min?: number; + max?: number; +} + +/** Vercel AI Gateway pricing in $/token. */ +export interface AiGatewayPricing { + input?: string | number; + output?: string | number; + input_cache_read?: string | number; + input_cache_write?: string | number; + input_tiers?: AiGatewayPriceTier[]; + output_tiers?: AiGatewayPriceTier[]; + input_cache_read_tiers?: AiGatewayPriceTier[]; + input_cache_write_tiers?: AiGatewayPriceTier[]; +} + +const RATE_FIELDS = [ + { rate: "input", tiers: "input_tiers" }, + { rate: "output", tiers: "output_tiers" }, + { rate: "cacheRead", tiers: "input_cache_read_tiers" }, + { rate: "cacheWrite", tiers: "input_cache_write_tiers" }, +] as const; + +function perMillion(value: string | number | undefined): number { + const parsed = typeof value === "number" ? value : parseFloat(value ?? "0"); + return Number.isFinite(parsed) ? Number((parsed * 1_000_000).toFixed(6)) : 0; +} + +/** + * Convert AI Gateway pricing to $/million tokens. Each rate lists its own prompt-length brackets; + * every bracket start above zero becomes a request-wide tier with the rates in effect there. + */ +export function getAiGatewayCost(pricing: AiGatewayPricing | undefined): ModelCost { + const base: ModelCostRates = { + input: perMillion(pricing?.input), + output: perMillion(pricing?.output), + cacheRead: perMillion(pricing?.input_cache_read), + cacheWrite: perMillion(pricing?.input_cache_write), + }; + + const starts = new Set(); + for (const field of RATE_FIELDS) { + for (const bracket of pricing?.[field.tiers] ?? []) { + if (bracket.min !== undefined && bracket.min > 0) starts.add(bracket.min); + } + } + + const tiers = [...starts] + .sort((a, b) => a - b) + .map((start): ModelCostTier => { + const tier: ModelCostTier = { inputTokensAbove: start - 1, ...base }; + for (const field of RATE_FIELDS) { + const bracket = pricing?.[field.tiers]?.find( + (candidate) => (candidate.min ?? 0) <= start && (candidate.max === undefined || start < candidate.max), + ); + if (bracket?.cost !== undefined) tier[field.rate] = perMillion(bracket.cost); + } + return tier; + }); + + return tiers.length > 0 ? { ...base, tiers } : base; +} + +interface AiGatewayModel { + id: string; + name?: string; + type?: string; + context_window?: number; + max_tokens?: number; + tags?: string[]; + pricing?: AiGatewayPricing; +} + +export interface AiGatewayCatalog { + chat: Model<"anthropic-messages">[]; + classifiers: ClassifierModel<"typesafe-system-one">[]; +} + +/** Fetch tool-capable chat models and evaluation (classifier) models. */ +export async function fetchAiGatewayCatalog(): Promise { + const response = await fetch(AI_GATEWAY_MODELS_URL); + if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); + const data = (await response.json()) as { data?: unknown }; + const items = Array.isArray(data.data) ? (data.data as AiGatewayModel[]) : []; + const chat: AiGatewayCatalog["chat"] = []; + const classifiers: AiGatewayCatalog["classifiers"] = []; + + for (const model of items) { + // Evaluation models such as TypeSafe's Jev are served through the + // TypeSafe-compatible System One endpoint. + if (model.type === "evaluation") { + classifiers.push({ + type: "classifier", + id: model.id, + name: model.name || model.id, + api: "typesafe-system-one", + provider: "vercel-ai-gateway", + baseUrl: AI_GATEWAY_TYPESAFE_BASE_URL, + input: ["text"], + cost: { + input: perMillion(model.pricing?.input), + output: perMillion(model.pricing?.output), + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: model.context_window || 4096, + }); + continue; + } + const tags = Array.isArray(model.tags) ? model.tags : []; + // Only include models that support tools + if (!tags.includes("tool-use")) continue; + + chat.push({ + id: model.id, + name: model.name || model.id, + api: "anthropic-messages", + baseUrl: AI_GATEWAY_BASE_URL, + provider: "vercel-ai-gateway", + reasoning: tags.includes("reasoning"), + input: tags.includes("vision") ? ["text", "image"] : ["text"], + compat: { allowEmptySignature: true }, + cost: getAiGatewayCost(model.pricing), + contextWindow: model.context_window || 4096, + maxTokens: model.max_tokens || 4096, + }); + } + return { chat, classifiers }; +} diff --git a/packages/ai/package.json b/packages/ai/package.json index ac6f865c6..57e8f262c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "description": "Unified LLM API with automatic model discovery and provider configuration", "type": "module", "main": "./dist/index.js", @@ -56,27 +56,27 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", - "generate-models": "node scripts/generate-models.ts --strict", - "hydrate-model-data": "node scripts/generate-models.ts --strict --data-only", - "generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog", + "clean": "node ../../scripts/package-files.mjs clean dist", + "generate-models": "node catalog/generate.ts --strict", + "hydrate-model-data": "node catalog/generate.ts --strict --data-only", + "generate-model-catalog": "node catalog/generate.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "check:model-data": "node scripts/check-model-data.ts", "build": "npm run generate-models && npm run build:offline", - "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", + "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs clean dist/providers/data && node ../../scripts/package-files.mjs copy src/providers/data dist/providers", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "keywords": [ "ai", @@ -101,6 +101,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" } } diff --git a/packages/ai/scripts/ai-gateway-pricing.ts b/packages/ai/scripts/ai-gateway-pricing.ts deleted file mode 100644 index 74ac54b84..000000000 --- a/packages/ai/scripts/ai-gateway-pricing.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { ModelCost, ModelCostRates, ModelCostTier } from "../src/types.ts"; - -/** A prompt-length bracket. `min` is inclusive and `max` exclusive, both in prompt tokens. */ -export interface AiGatewayPriceTier { - cost?: string | number; - min?: number; - max?: number; -} - -/** Vercel AI Gateway pricing in $/token. */ -export interface AiGatewayPricing { - input?: string | number; - output?: string | number; - input_cache_read?: string | number; - input_cache_write?: string | number; - input_tiers?: AiGatewayPriceTier[]; - output_tiers?: AiGatewayPriceTier[]; - input_cache_read_tiers?: AiGatewayPriceTier[]; - input_cache_write_tiers?: AiGatewayPriceTier[]; -} - -const RATE_FIELDS = [ - { rate: "input", tiers: "input_tiers" }, - { rate: "output", tiers: "output_tiers" }, - { rate: "cacheRead", tiers: "input_cache_read_tiers" }, - { rate: "cacheWrite", tiers: "input_cache_write_tiers" }, -] as const; - -function perMillion(value: string | number | undefined): number { - const parsed = typeof value === "number" ? value : parseFloat(value ?? "0"); - return Number.isFinite(parsed) ? Number((parsed * 1_000_000).toFixed(6)) : 0; -} - -/** - * Convert AI Gateway pricing to $/million tokens. Each rate lists its own prompt-length brackets; - * every bracket start above zero becomes a request-wide tier with the rates in effect there. - */ -export function getAiGatewayCost(pricing: AiGatewayPricing | undefined): ModelCost { - const base: ModelCostRates = { - input: perMillion(pricing?.input), - output: perMillion(pricing?.output), - cacheRead: perMillion(pricing?.input_cache_read), - cacheWrite: perMillion(pricing?.input_cache_write), - }; - - const starts = new Set(); - for (const field of RATE_FIELDS) { - for (const bracket of pricing?.[field.tiers] ?? []) { - if (bracket.min !== undefined && bracket.min > 0) starts.add(bracket.min); - } - } - - const tiers = [...starts] - .sort((a, b) => a - b) - .map((start): ModelCostTier => { - const tier: ModelCostTier = { inputTokensAbove: start - 1, ...base }; - for (const field of RATE_FIELDS) { - const bracket = pricing?.[field.tiers]?.find( - (candidate) => (candidate.min ?? 0) <= start && (candidate.max === undefined || start < candidate.max), - ); - if (bracket?.cost !== undefined) tier[field.rate] = perMillion(bracket.cost); - } - return tier; - }); - - return tiers.length > 0 ? { ...base, tiers } : base; -} diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts deleted file mode 100644 index f5efb94bf..000000000 --- a/packages/ai/scripts/generate-models.ts +++ /dev/null @@ -1,3632 +0,0 @@ -#!/usr/bin/env node - -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs"; -import { dirname, join, resolve } from "path"; -import { fileURLToPath } from "url"; -import { getEffortThinkingLevelMap, type ModelsDevReasoningOption } from "./models-dev-reasoning-options.ts"; -import { type AiGatewayPricing, getAiGatewayCost } from "./ai-gateway-pricing.ts"; -import { buildOpenRouterCatalog, type OpenRouterCatalog, type OpenRouterModelListItem } from "./openrouter-catalog.ts"; -import { - CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, - CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, - CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, - CLOUDFLARE_WORKERS_AI_BASE_URL, - CLOUDFLARE_WORKERS_AI_REST_BASE_URL, -} from "../src/api/cloudflare.ts"; -import type { - AnthropicMessagesCompat, - AnyModel, - Api, - ClassifierApi, - ClassifierModel, - ImageApi, - ImageModel, - KnownProvider, - Model, - ModelCost, - ModelPromptCache, - OpenAICompletionsCompat, - OpenAIResponsesCompat, -} from "../src/types.ts"; -import { - assertExactModelIds, - createModelDataManifest, - groupProviderModelData, - type ModelDataStructure, - MODEL_DATA_MANIFEST_FILE, - readModelDataProviderIds, - validateGeneratedModelData, - validateModelDataDirectory, -} from "./model-data.ts"; -import { - DEFAULT_RADIUS_GATEWAY, - getRadiusModelsFromConfig, - loadRadiusGatewayConfig, -} from "../src/providers/radius-config.ts"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -const packageRoot = join(__dirname, ".."); - -function readGeneratorOptions(args: string[]): { - strict: boolean; - dataOnly: boolean; - jsonOnly: boolean; - jsonOutputDir: string | undefined; - pretty: boolean; -} { - let strict = false; - let dataOnly = false; - let jsonOnly = false; - let jsonOutputDir: string | undefined; - let pretty = false; - - for (let index = 0; index < args.length; index++) { - const arg = args[index]; - if (arg === "--strict") { - strict = true; - continue; - } - if (arg === "--data-only") { - dataOnly = true; - continue; - } - if (arg === "--json-only") { - jsonOnly = true; - continue; - } - if (arg === "--pretty") { - pretty = true; - continue; - } - if (arg === "--json-output") { - const value = args[++index]; - if (!value) throw new Error("--json-output requires a directory"); - jsonOutputDir = resolve(value); - continue; - } - throw new Error(`Unknown argument: ${arg}`); - } - - if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); - if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output"); - return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty }; -} - -const generatorOptions = readGeneratorOptions(process.argv.slice(2)); - -interface ModelsDevModel { - id: string; - name: string; - tool_call?: boolean; - structured_output?: boolean; - reasoning?: boolean; - reasoning_options?: ModelsDevReasoningOption[]; - status?: string; - limit?: { - context?: number; - output?: number; - }; - cost?: { - input?: number; - output?: number; - cache_read?: number; - cache_write?: number; - tiers?: { - input?: number; - output?: number; - cache_read?: number; - cache_write?: number; - tier?: { - type?: string; - size?: number; - }; - }[]; - }; - modalities?: { - input?: string[]; - output?: string[]; - }; - provider?: { - npm?: string; - }; -} - -interface ModelsDevProvider { - models?: Record; -} - -interface ModelsDevMetadata { - id: string; - type?: string; - name: string; - limit?: { context?: number }; - modalities?: { input?: string[] }; -} - -type ModelsDevCatalog = Record; - -interface NvidiaNimModelListItem { - id: string; -} - -interface AiGatewayModel { - id: string; - name?: string; - type?: string; - context_window?: number; - max_tokens?: number; - tags?: string[]; - pricing?: AiGatewayPricing; -} - -const COPILOT_STATIC_HEADERS = { - "User-Agent": "GitHubCopilotChat/0.35.0", - "Editor-Version": "vscode/1.107.0", - "Editor-Plugin-Version": "copilot-chat/0.35.0", - "Copilot-Integration-Id": "vscode-chat", -} as const; - -const TOGETHER_BASE_URL = "https://api.together.ai/v1"; -const TOGETHER_BASE_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - supportsLongCacheRetention: false, -}; -const TOGETHER_TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_BASE_COMPAT, - thinkingFormat: "together", -}; -const TOGETHER_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_BASE_COMPAT, - supportsReasoningEffort: true, - thinkingFormat: "openai", -}; -const TOGETHER_TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_TOGGLE_REASONING_COMPAT, - supportsReasoningEffort: true, -}; -const TOGETHER_REASONING_EFFORT_MODELS = new Set(["openai/gpt-oss-20b", "openai/gpt-oss-120b"]); -const TOGETHER_TOGGLE_REASONING_EFFORT_MODELS = new Set(["deepseek-ai/DeepSeek-V4-Pro-0813"]); -const TOGETHER_REASONING_EFFORT_LEVEL_MAP = { - off: null, - minimal: null, -} as const; -const TOGETHER_DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, -} as const; -const TOGETHER_TOGGLE_REASONING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, -} as const; - -const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1"; -const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; -// TypeSafe-compatible System One endpoint for evaluation models. -// https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe -const AI_GATEWAY_TYPESAFE_BASE_URL = "https://ai-gateway.vercel.sh/typesafe/v1"; -const VERTEX_BASE_URL = "https://{location}-aiplatform.googleapis.com"; -const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; -const NVIDIA_HEADERS = { - "NVCF-POLL-SECONDS": "3600", -} as const; -const NVIDIA_OPENAI_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - supportsLongCacheRetention: false, -}; -const NVIDIA_NIM_UNSUPPORTED_MODELS = new Set([ - "abacusai/dracarys-llama-3.1-70b-instruct", - "bytedance/seed-oss-36b-instruct", - "deepseek-ai/deepseek-v4-flash", - "deepseek-ai/deepseek-v4-pro", - "google/gemma-2-2b-it", - "google/gemma-3n-e2b-it", - "google/gemma-3n-e4b-it", - "google/gemma-4-31b-it", - "meta/llama-3.2-1b-instruct", - "meta/llama-4-maverick-17b-128e-instruct", - "microsoft/phi-4-mini-instruct", - "minimaxai/minimax-m2.7", - "mistralai/mistral-nemotron", - "nvidia/nemotron-mini-4b-instruct", - "qwen/qwen3-next-80b-a3b-instruct", - "qwen/qwen3.5-397b-a17b", - "sarvamai/sarvam-m", - "upstage/solar-10.7b-instruct", -]); -const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); -const OPENCODE_GO_GLM52_THINKING_LEVEL_MAP = { - off: null, - minimal: null, - low: null, - medium: null, - high: "high", - max: "max", -} as const; -const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set([ - "github-copilot:claude-haiku-4.5", - "github-copilot:claude-sonnet-4", - "github-copilot:claude-sonnet-4.5", -]); -const ANTHROPIC_ALLOWED_FALLBACK_MODELS = { - "claude-fable-5": ["claude-opus-4-8", "claude-opus-5"], - "claude-opus-5": ["claude-opus-4-8"], -} satisfies Record; - -const DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - max: "max", -} as const; -const DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP = { - ...DEEPSEEK_V4_THINKING_LEVEL_MAP, - low: "low", -} as const; -// Azure Foundry rejects DeepSeek's own max effort. -const AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: null, - max: null, -} as const; -// Verified against Fireworks Messages raw_output on 2026-09-10 (#9323). -// Fall back to verified support when models.dev omits effort metadata; this is -// not an allowlist. Any Fireworks Messages model advertising effort uses adaptive thinking. -const FIREWORKS_ADAPTIVE_THINKING_FALLBACK_MODELS = new Set([ - "accounts/fireworks/models/deepseek-v4-flash-0731", - "accounts/fireworks/models/deepseek-v4-flash-vision-exp", - "accounts/fireworks/models/deepseek-v4-pro-0813", - "accounts/fireworks/models/qwen3p8-max", - "accounts/fireworks/models/qwen3p8-2p4t-a95b", -]); -const QWEN_TOKEN_PLAN_FALLBACK_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: "max", -} as const; -const QWEN_TOKEN_PLAN_REASONING_EFFORT_FALLBACK_MODEL_IDS = new Set(["glm-5", "glm-5.1"]); -// Retired preview id — models.dev may still list it after GA ships. -const QWEN_TOKEN_PLAN_EXCLUDED_MODEL_IDS = new Set(["qwen3.8-max-preview"]); -const QWEN_TOKEN_PLAN_PROVIDER_IDS = new Set([ - "qwen-token-plan", - "qwen-token-plan-cn", - "qwen-token-plan-individual", -]); -// QwenCloud Token Plan Individual text-model allowlist, verified 2026-09-03. -// Retired models remain excluded above even if the public catalog lags. -// https://docs.qwencloud.com/token-plan/personal/token-plan-personal-overview -const QWEN_TOKEN_PLAN_INDIVIDUAL_MODEL_IDS = new Set([ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "deepseek-v4-pro-0813", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-flash", - "qwen3.8-max", -]); - -const KIMI_K3_MAX_TOKENS = 131072; -const KIMI_K3_COST = { - input: 3, - output: 15, - cacheRead: 0.3, - cacheWrite: 0, -} as const; -// Kimi Chat Completions settings shared by Moonshot and Kimi Coding. -const KIMI_OPENAI_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - thinkingFormat: "deepseek", -}; -// Kimi K3-style models take reasoning_effort and need earlier reasoning sent back. -const KIMI_K3_OPENAI_COMPAT: OpenAICompletionsCompat = { - ...KIMI_OPENAI_COMPAT, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "openai", - supportsReasoningEffort: true, -}; -const OPENROUTER_KIMI_K3_MODEL_IDS = new Set(["moonshotai/kimi-k3", "~moonshotai/kimi-latest"]); - -const ANT_LING_RING_THINKING_LEVEL_MAP = { - off: null, - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: "xhigh", -} as const; - -const BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); -const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]); -const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-pro", - "gpt-5.5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", -]); -const OPENAI_ADDITIONAL_TOOLS_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; -const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; -const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", -]); -// OpenAI documents configuration_update for the GPT-6 family. -// https://developers.openai.com/api/docs/guides/reasoning -const OPENAI_MID_CONVO_EFFORT_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol"]); -const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; -const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", -]); -const OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.4-pro", - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", -]); - -// Keep the generated default no less restrictive than coding-agent's historical -// image preprocessing. Provider limits can narrow this profile, but unknown -// providers retain the cache-safe 2000px / 4.5 MiB behavior. -const DEFAULT_IMAGE_RESIZE = { - maxWidth: 2000, - maxHeight: 2000, - maxBytes: 4.5 * 1024 * 1024, - jpegQuality: 80, -} as const; - -function withOpenAiLongContextPricing(cost: Model["cost"]): Model["cost"] { - return { - ...cost, - tiers: [ - { - inputTokensAbove: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - input: roundCost(cost.input * 2), - output: roundCost(cost.output * 1.5), - cacheRead: roundCost(cost.cacheRead * 2), - cacheWrite: roundCost(cost.cacheWrite * 2), - }, - ], - }; -} - -// Keep current OpenAI prices authoritative until models.dev and passthrough -// catalogs catch up. -// https://developers.openai.com/api/docs/pricing -const OPENAI_STANDARD_COSTS: Record = { - "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, - "gpt-5.6-sol": { input: 4, output: 20, cacheRead: 0.4, cacheWrite: 5 }, - "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, - "gpt-6-astra": { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }, - "gpt-6-luna": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, - "gpt-6-sol": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, - "gpt-6.1-sol": { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, -}; - -const OPENAI_RESPONSES_NONE_REASONING_MODELS = new Set([ - "gpt-5.1", - "gpt-5.2", - "gpt-5.3-codex", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-sol", - "gpt-6-luna", -]); -const XAI_BUILTIN_EXCLUDED_MODEL_IDS = new Set([ - "grok-3", - "grok-3-fast", - "grok-4.20-0309-non-reasoning", - "grok-4.20-0309-reasoning", - "grok-build-0.1", - "grok-code-fast-1", -]); -const XAI_RESPONSES_COMPAT: OpenAIResponsesCompat = { - supportsLongCacheRetention: false, -}; - -const OPENCODE_OPENAI_COMPLETIONS_LONG_CACHE_RETENTION_UNSUPPORTED_MODELS = new Set([ - "opencode:deepseek-v4-flash", - "opencode:deepseek-v4-pro", - "opencode:kimi-k2.5", - "opencode:kimi-k2.6", - "opencode:minimax-m2.7", - "opencode-go:kimi-k2.6", -]); - -// GitHub's "Models with extended capabilities" table lists these Copilot models as supporting -// the extended 1 million token context window. -const GITHUB_COPILOT_EXTENDED_CONTEXT_MODELS = new Set([ - "claude-fable-5", - "claude-opus-4.8", - "claude-opus-5", - "claude-opus-5.5", - "claude-sonnet-4.6", - "claude-sonnet-5", - "gpt-5.3-codex", - "gpt-6-astra", - "gpt-6-luna", - "gpt-6-sol", -]); - -// Checked manually against the authenticated GitHub Copilot /models endpoint on 2026-06-15. -// Keep this to narrow corrections over models.dev metadata instead of snapshotting Copilot's catalog. -const GITHUB_COPILOT_THINKING_LEVEL_OVERRIDES = { - "claude-opus-4.8": { minimal: "low" }, - "claude-opus-5": { minimal: "low" }, - "claude-sonnet-4.6": { minimal: "low", max: "max" }, -} satisfies Record["thinkingLevelMap"]>>; - -function mergeThinkingLevelMap(model: Model, map: NonNullable["thinkingLevelMap"]>): void { - model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map }; -} - -const modelsDevReasoningOptions = new Map(); - -function getModelKey(model: Pick, "provider" | "id">): string { - return `${model.provider}:${model.id}`; -} - -function recordModelsDevReasoningOptions(provider: string, id: string, sourceModel: ModelsDevModel): void { - if (sourceModel.reasoning_options !== undefined) { - modelsDevReasoningOptions.set(`${provider}:${id}`, sourceModel.reasoning_options); - } -} - -function supportsDirectReasoningEffort(model: Model): boolean { - if (model.api === "anthropic-messages") return model.compat?.forceAdaptiveThinking === true; - if ( - model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses" - ) { - return true; - } - if (model.api !== "openai-completions") return false; - - const compat = { - ...detectOpenAICompletionsCompat(model as Model<"openai-completions">), - ...(model.compat as OpenAICompletionsCompat | undefined), - }; - return compat.thinkingFormat === "openai" && compat.supportsReasoningEffort; -} - -function applyModelsDevReasoningOptionMetadata(model: Model): void { - const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); - if (!reasoningOptions || !supportsDirectReasoningEffort(model)) return; - const thinkingLevelMap = getEffortThinkingLevelMap(reasoningOptions); - if (thinkingLevelMap) mergeThinkingLevelMap(model, thinkingLevelMap); -} - -function getTogetherCompat(modelId: string, reasoning: boolean): OpenAICompletionsCompat { - if (!reasoning) return TOGETHER_BASE_COMPAT; - if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_REASONING_EFFORT_COMPAT; - if (TOGETHER_TOGGLE_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_TOGGLE_REASONING_EFFORT_COMPAT; - return TOGETHER_TOGGLE_REASONING_COMPAT; -} - -function getTogetherThinkingLevelMap( - modelId: string, - reasoning: boolean, -): NonNullable["thinkingLevelMap"]> | undefined { - if (!reasoning) return undefined; - if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return { ...TOGETHER_REASONING_EFFORT_LEVEL_MAP }; - if (TOGETHER_TOGGLE_REASONING_EFFORT_MODELS.has(modelId)) return { ...TOGETHER_DEEPSEEK_V4_THINKING_LEVEL_MAP }; - return { ...TOGETHER_TOGGLE_REASONING_LEVEL_MAP }; -} - -function supportsOpenAiXhigh(modelId: string): boolean { - return ( - modelId.includes("gpt-5.2") || - modelId.includes("gpt-5.3") || - modelId.includes("gpt-5.4") || - modelId.includes("gpt-5.5") || - modelId.includes("gpt-5.6") || - modelId.includes("gpt-6") - ); -} - -function supportsOpenAiMax(model: Model): boolean { - return ( - (model.id.includes("gpt-5.6") || model.id.includes("gpt-6")) && - (model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses" || - model.api === "openai-completions") - ); -} - -const VERIFIED_ANTHROPIC_MID_CONVO_EFFORT_PROVIDERS = new Set(["anthropic", "openrouter"]); -// OpenRouter rejects `configuration_update` system messages on Opus 5 ("Mid-conversation -// reasoning effort (configuration_update) is not supported on anthropic/claude-opus-5-20260723") -// while accepting them on Fable 5.1, so gate that model there. -const MID_CONVO_EFFORT_UNSUPPORTED_ANTHROPIC_MODELS = new Set(["openrouter:anthropic/claude-opus-5"]); - -function supportsAnthropicMidConvoEffort(modelId: string): boolean { - const id = modelId.toLowerCase().replace(/^~?anthropic\//, ""); - return ( - /^claude-opus-(?:5|5[.-]5)(?:-\d{8})?$/.test(id) || - /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(id) || - /^claude-(?:fable|mythos)-5(?:[.-]1)(?:-\d{8})?$/.test(id) - ); -} - -// Claude models that accept `thinking: {type: "disabled"}` alongside per-message effort. -// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 reject it. -// https://platform.claude.com/docs/en/build-with-claude/thinking#configuring-thinking -const ANTHROPIC_DISABLED_THINKING_MODELS = new Set(["claude-opus-5", "claude-haiku-5-5", "claude-haiku-5.5"]); - -function supportsAnthropicDisabledThinkingWithMidConvoEffort(modelId: string): boolean { - // Strip OpenRouter's "anthropic/" or "~anthropic/" prefix and any dated snapshot suffix. - const id = modelId - .toLowerCase() - .replace(/^~?anthropic\//, "") - .replace(/-\d{8}$/, ""); - return ANTHROPIC_DISABLED_THINKING_MODELS.has(id); -} - -function supportsAnthropicMidConvoSystemMessages(modelId: string): boolean { - return ( - /^claude-opus-(?:4[.-]8|5(?:[.-]5)?)(?:-\d{8})?$/.test(modelId) || - /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(modelId) || - /^claude-(?:fable|mythos)-5(?:[.-]1)?(?:-\d{8})?$/.test(modelId) - ); -} - -function isAnthropicAdaptiveThinkingModel(modelId: string): boolean { - return ( - modelId.includes("opus-4-6") || - modelId.includes("opus-4.6") || - modelId.includes("opus-4-7") || - modelId.includes("opus-4.7") || - modelId.includes("opus-4-8") || - modelId.includes("opus-4.8") || - modelId.includes("opus-5") || - modelId.includes("opus.5") || - modelId.includes("sonnet-4-6") || - modelId.includes("sonnet-4.6") || - modelId.includes("sonnet-5") || - modelId.includes("sonnet.5") || - modelId.includes("haiku-5") || - modelId.includes("haiku.5") || - modelId.includes("fable-5") || - modelId.includes("mythos-5") - ); -} - -function isAnthropicTemperatureUnsupportedModel(modelId: string): boolean { - const id = modelId.toLowerCase(); - return ( - id.includes("opus-4-7") || - id.includes("opus-4.7") || - id.includes("opus-4-8") || - id.includes("opus-4.8") || - id.includes("opus-5") || - id.includes("opus.5") || - id.includes("sonnet-5-5") || - id.includes("sonnet-5.5") || - id.includes("haiku-5-5") || - id.includes("haiku-5.5") - ); -} - -const OPENAI_COMPLETIONS_DEFAULT_COMPAT = { - supportsStore: true, - supportsDeveloperRole: true, - supportsReasoningEffort: true, - supportsUsageInStreaming: true, - supportsFinishReason: true, - maxTokensField: "max_completion_tokens", - requiresToolResultName: false, - requiresAssistantAfterToolResult: false, - requiresThinkingAsText: false, - requiresReasoningContentOnAssistantMessages: false, - thinkingFormat: "openai", - openRouterRouting: {}, - vercelGatewayRouting: {}, - chatTemplateKwargs: {}, - chatTemplateArgs: {}, - zaiToolStream: false, - supportsStrictMode: false, - supportsOpenAIGrammarTools: false, - supportsMidConvoSystemMessages: false, - supportsMidConvoToolAdditions: false, - sendSessionAffinityHeaders: false, - supportsLongCacheRetention: true, -} satisfies Required< - Omit< - OpenAICompletionsCompat, - "cacheControlFormat" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField" - > -> & { - cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; -}; - -type OpenAICompletionsResolvedCompat = typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT & { - cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; -}; - -function mergeAnthropicMessagesCompat(model: Model, compat: AnthropicMessagesCompat): void { - model.compat = { ...(model.compat as AnthropicMessagesCompat | undefined), ...compat }; -} - -function detectOpenAICompletionsCompat(model: Model<"openai-completions">): OpenAICompletionsResolvedCompat { - const provider = model.provider; - const baseUrl = model.baseUrl; - - const isZai = - provider === "zai" || - provider === "zai-coding-cn" || - baseUrl.includes("api.z.ai") || - baseUrl.includes("open.bigmodel.cn"); - const isTogether = - provider === "together" || baseUrl.includes("api.together.ai") || baseUrl.includes("api.together.xyz"); - const isMoonshot = provider === "moonshotai" || provider === "moonshotai-cn" || baseUrl.includes("api.moonshot."); - const isOpenRouter = provider === "openrouter" || baseUrl.includes("openrouter.ai"); - const isCloudflareWorkersAI = provider === "cloudflare-workers-ai" || baseUrl.includes("api.cloudflare.com"); - const isCloudflareAiGateway = provider === "cloudflare-ai-gateway" || baseUrl.includes("gateway.ai.cloudflare.com"); - const isNvidia = provider === "nvidia" || baseUrl.includes("integrate.api.nvidia.com"); - const isAntLing = provider === "ant-ling" || baseUrl.includes("api.ant-ling.com"); - const isCerebras = provider === "cerebras" || baseUrl.includes("cerebras.ai"); - const isDeepSeek = provider === "deepseek" || baseUrl.toLowerCase().includes("deepseek.com"); - - const isNonStandard = - isNvidia || - isCerebras || - provider === "xai" || - baseUrl.includes("api.x.ai") || - isTogether || - baseUrl.includes("chutes.ai") || - isDeepSeek || - isZai || - isMoonshot || - provider === "opencode" || - baseUrl.includes("opencode.ai") || - isCloudflareWorkersAI || - isCloudflareAiGateway || - isAntLing; - - const useMaxTokens = - baseUrl.includes("chutes.ai") || - isDeepSeek || - isMoonshot || - isCloudflareAiGateway || - isTogether || - isNvidia || - isAntLing || - isZai; - - const isGrok = provider === "xai" || baseUrl.includes("api.x.ai"); - const isOpenRouterDeveloperRoleModel = - isOpenRouter && (model.id.startsWith("anthropic/") || model.id.startsWith("openai/")); - const cacheControlFormat = - provider === "openrouter" && /^~?anthropic\//.test(model.id) ? "anthropic" : undefined; - - return { - supportsStore: !isNonStandard, - supportsDeveloperRole: isOpenRouterDeveloperRoleModel || (!isNonStandard && !isOpenRouter), - supportsReasoningEffort: - !isGrok && !isZai && !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isAntLing, - supportsUsageInStreaming: true, - supportsFinishReason: true, - maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens", - requiresToolResultName: false, - requiresAssistantAfterToolResult: false, - requiresThinkingAsText: false, - requiresReasoningContentOnAssistantMessages: isDeepSeek, - thinkingFormat: isDeepSeek - ? "deepseek" - : isZai - ? "zai" - : isTogether - ? "together" - : isAntLing - ? "ant-ling" - : isOpenRouter - ? "openrouter" - : "openai", - openRouterRouting: {}, - vercelGatewayRouting: {}, - chatTemplateKwargs: {}, - chatTemplateArgs: {}, - zaiToolStream: false, - // Preserve built-in behavior as explicit metadata against the conservative runtime default. - supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isCerebras, - supportsOpenAIGrammarTools: false, - supportsMidConvoSystemMessages: false, - supportsMidConvoToolAdditions: false, - ...(cacheControlFormat ? { cacheControlFormat } : {}), - sendSessionAffinityHeaders: isOpenRouter, - supportsLongCacheRetention: !( - isTogether || - isCloudflareWorkersAI || - isCloudflareAiGateway || - isNvidia || - isAntLing - ), - }; -} - -function isPlainEmptyObject(value: unknown): boolean { - return typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 0; -} - -function openAICompletionsCompatDelta(compat: OpenAICompletionsResolvedCompat): OpenAICompletionsCompat { - const delta: OpenAICompletionsCompat = {}; - for (const [key, value] of Object.entries(compat)) { - const defaultValue = OPENAI_COMPLETIONS_DEFAULT_COMPAT[key as keyof typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT]; - if (isPlainEmptyObject(value) && isPlainEmptyObject(defaultValue)) continue; - if (value !== defaultValue) { - (delta as Record)[key] = value; - } - } - return delta; -} - -function mergeOpenAICompletionsCompat(model: Model, compat: OpenAICompletionsCompat): void { - model.compat = { ...(model.compat as OpenAICompletionsCompat | undefined), ...compat }; -} - -function applyOpenAICompletionsCompatMetadata(model: Model): void { - if (model.api !== "openai-completions") return; - const detected = openAICompletionsCompatDelta(detectOpenAICompletionsCompat(model as Model<"openai-completions">)); - model.compat = { ...detected, ...(model.compat as OpenAICompletionsCompat | undefined) }; - if (Object.keys(model.compat).length === 0) { - delete model.compat; - } -} - -function applyAnthropicMessagesCompatMetadata(model: Model): void { - if (model.api !== "anthropic-messages") return; - const compat = getAnthropicMessagesCompat(model.provider, model.id); - if (compat) { - mergeAnthropicMessagesCompat(model, compat); - if (compat.supportsMidConvoEffort && !supportsAnthropicDisabledThinkingWithMidConvoEffort(model.id)) { - mergeThinkingLevelMap(model, { off: null }); - } - } -} - -function isAnthropicFallbackMetadataModel(model: Model): model is Model<"anthropic-messages"> { - if (model.provider !== "anthropic" || model.api !== "anthropic-messages") return false; - return ( - model.id in ANTHROPIC_ALLOWED_FALLBACK_MODELS || - Object.values(ANTHROPIC_ALLOWED_FALLBACK_MODELS).some((fallbackModelIds) => fallbackModelIds.includes(model.id)) - ); -} - -function applyAnthropicAllowedFallbackModelMetadata(models: readonly Model<"anthropic-messages">[]): void { - const modelsById = new Map(models.map((model) => [model.id, model])); - for (const [modelId, fallbackModelIds] of Object.entries(ANTHROPIC_ALLOWED_FALLBACK_MODELS)) { - const model = modelsById.get(modelId); - if (!model) continue; - - const compatibleFallbackModelIds = model.compat?.supportsMidConvoEffort - ? fallbackModelIds.filter(supportsAnthropicMidConvoEffort) - : fallbackModelIds; - const allowedFallbackModels = compatibleFallbackModelIds.flatMap((fallbackModelId) => { - const fallbackModel = modelsById.get(fallbackModelId); - return fallbackModel - ? [{ provider: fallbackModel.provider, model: fallbackModel.id, cost: fallbackModel.cost }] - : []; - }); - if (allowedFallbackModels.length > 0) { - mergeAnthropicMessagesCompat(model, { allowedFallbackModels }); - } - } -} - -function applyStrictToolCompatMetadata(model: Model): void { - if ( - (model.provider === "openai" || model.provider === "cloudflare-ai-gateway") && - model.api === "openai-responses" - ) { - model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsStrictMode: true }; - } else if (model.provider === "anthropic" && model.api === "anthropic-messages") { - mergeAnthropicMessagesCompat(model, { supportsStrictTools: true }); - } -} - -// Responses endpoints verified (OpenAI, ChatGPT Codex backend, GitHub Copilot, -// opencode zen) or documented (Azure OpenAI, Cloudflare AI Gateway) to pass -// OpenAI custom grammar tools through. OpenAI rejects `type: "custom"` tools -// for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series). -const OPENAI_GRAMMAR_TOOL_PROVIDERS = new Set([ - "openai", - "openai-codex", - "azure", - "github-copilot", - "opencode", - "cloudflare-ai-gateway", -]); -const OPENAI_GRAMMAR_TOOL_APIS = new Set([ - "openai-responses", - "azure-openai-responses", - "openai-codex-responses", -]); - -function applyOpenAIGrammarToolCompatMetadata(model: Model): void { - if (!OPENAI_GRAMMAR_TOOL_APIS.has(model.api) || !OPENAI_GRAMMAR_TOOL_PROVIDERS.has(model.provider)) return; - const match = /^gpt-(\d+)/.exec(model.id); - if (!match || Number(match[1]) < 5) return; - model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsOpenAIGrammarTools: true }; -} - -function applyOpenAIToolSearchMetadata(model: Model): void { - const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses"; - const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses"; - if (!(isOpenAIResponses || isOpenAICodex) || !OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) return; - const supportsAdditionalTools = - (isOpenAIResponses && OPENAI_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id)) || - (isOpenAICodex && OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id)); - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - ...(supportsAdditionalTools ? { supportsAdditionalTools: true } : {}), - supportsToolSearch: true, - }; -} - -// Moonshot Kimi K2.6/K2.7 accept system text after the conversation starts but reject -// tool-bearing system messages. Kimi K3 accepts both forms; Fireworks and OpenCode pass -// its tool-bearing form through. GitHub Copilot forwards K3 text but silently drops its -// tool-bearing message. DeepSeek V4 Pro and OpenAI models behind OpenRouter also accept -// plain system text in place. Kimi Coding sets these flags in createKimiCodingModels(). -function applyOpenAICompletionsTranscriptMetadata(model: Model): void { - if (model.api !== "openai-completions") return; - const isKimiK3 = - (model.provider.startsWith("moonshot") && model.id === "kimi-k3") || - (model.provider === "fireworks" && model.id.includes("kimi-k3")) || - ((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "kimi-k3"); - const isMoonshotKimiK2 = - model.provider.startsWith("moonshot") && - (model.id === "kimi-k2.6" || model.id === "kimi-k2.7-code" || model.id === "kimi-k2.7-code-highspeed"); - const isTextOnly = - isMoonshotKimiK2 || - (model.provider === "github-copilot" && model.id === "kimi-k3") || - (model.provider === "deepseek" && model.id === "deepseek-v4-pro") || - (model.provider === "openrouter" && - model.id.startsWith("openai/") && - OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(model.id.slice("openai/".length))); - if (!isKimiK3 && !isTextOnly) return; - model.compat = { - ...(model.compat as OpenAICompletionsCompat | undefined), - supportsMidConvoSystemMessages: true, - ...(isKimiK3 ? { supportsMidConvoToolAdditions: true } : {}), - }; -} - -// Newer OpenAI Responses models accept developer messages after the conversation has started. -// OpenCode Zen, OpenCode Go, and GitHub Copilot pass both those messages and -// `additional_tools` items through to OpenAI unchanged; tool search is not verified -// through those proxies. -const OPENAI_RESPONSES_PROXY_PROVIDERS = new Set(["opencode", "opencode-go", "github-copilot"]); - -function applyOpenAIResponsesTranscriptMetadata(model: Model): void { - const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses"; - const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses"; - const isProxiedResponses = - OPENAI_RESPONSES_PROXY_PROVIDERS.has(model.provider) && model.api === "openai-responses"; - if ( - !(isOpenAIResponses || isOpenAICodex || isProxiedResponses) || - !OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(model.id) - ) { - return; - } - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsMidConvoSystemMessages: true, - ...(isProxiedResponses ? { supportsAdditionalTools: true } : {}), - }; -} - -// OpenAI charges prompt-cache writes starting with the GPT-5.6 family, and exactly -// those models accept `prompt_cache_options`; older models reject the parameter. -// https://developers.openai.com/api/docs/guides/prompt-caching -function applyOpenAIExplicitPromptCacheMetadata(model: Model): void { - if (model.provider !== "openai" || model.api !== "openai-responses") return; - if (!(model.cost.cacheWrite > 0)) return; - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsExplicitPromptCacheMode: true, - }; -} - -// GPT-6 models accept `configuration_update` input items that change the reasoning effort -// without changing the cached prompt prefix. Not enabled for openai-codex (Codex CLI keeps it -// behind a flag that is off by default) or proxies. -function applyOpenAIMidConvoEffortMetadata(model: Model): void { - if (model.provider !== "openai" || model.api !== "openai-responses") return; - if (!OPENAI_MID_CONVO_EFFORT_MODEL_IDS.has(model.id)) return; - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsMidConvoEffort: true, - }; -} - -// Anthropic ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` -// extends it to one hour. Only direct Anthropic is annotated so cache warming -// does not assume equivalent behavior through proxies. -// https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching -const ANTHROPIC_PROMPT_CACHE: ModelPromptCache = { short: 300, long: 3600 }; - -function applyPromptCacheMetadata(model: Model): void { - if (model.provider === "anthropic" && model.api === "anthropic-messages") { - model.promptCache = ANTHROPIC_PROMPT_CACHE; - } - // Do not add OpenAI lifetimes yet. Before enabling warming for explicit - // OpenAI caches, re-evaluate it using observed expiry, replay, and billing - // behavior; a documented TTL alone does not establish full cache loss. -} - -function applyImageInputMetadata(model: AnyModel): void { - if (!model.input.includes("image")) return; - - const providerLimits: AnyModel["inputLimits"] = - model.provider === "anthropic" - ? { - maxRequestBytes: 32 * 1024 * 1024, - images: { maxPerRequest: model.type !== "image" && model.contextWindow === 200000 ? 100 : 600 }, - } - : model.provider === "amazon-bedrock" - ? { images: { maxPerMessage: 20 } } - : model.provider === "openai" - ? { maxRequestBytes: 512 * 1024 * 1024, images: { maxPerRequest: 1500 } } - : model.provider === "google" - ? { maxRequestBytes: 20 * 1024 * 1024, images: { maxPerRequest: 3600 } } - : undefined; - const configuredImages = model.inputLimits?.images; - model.inputLimits = { - ...providerLimits, - ...model.inputLimits, - images: { - ...providerLimits?.images, - ...configuredImages, - resize: { ...DEFAULT_IMAGE_RESIZE, ...configuredImages?.resize }, - }, - }; -} - -function isGemma4Model(modelId: string): boolean { - return /gemma-?4/.test(modelId.toLowerCase()); -} - -function getGoogleThinkingLevelMap( - modelId: string, - reasoningOptions: readonly ModelsDevReasoningOption[], -): NonNullable["thinkingLevelMap"]> | undefined { - const effortMap = getEffortThinkingLevelMap(reasoningOptions); - if (effortMap) return effortMap; - if (isGemma4Model(modelId)) { - return { off: null, minimal: "MINIMAL", low: null, medium: null, high: "HIGH" }; - } - return undefined; -} - -function applyThinkingLevelMetadata(model: Model): void { - if ( - (model.api === "openai-responses" || model.api === "azure-openai-responses") && - model.id.startsWith("gpt-5") - ) { - mergeThinkingLevelMap(model, { off: null }); - } - if ( - (model.id === "gpt-6-astra" || - model.id === "gpt-6-sol" || - model.id === "gpt-6-luna" || - model.id === "gpt-6.1-sol") && - (model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses") - ) { - mergeThinkingLevelMap(model, { - // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none". - off: model.id === "gpt-6-astra" || model.id === "gpt-6.1-sol" ? null : "none", - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }); - } - if (model.provider === "github-copilot" && model.id.startsWith("gpt-5")) { - mergeThinkingLevelMap(model, { minimal: "low" }); - } - if ( - model.api === "openai-responses" && - model.provider === "openai" && - OPENAI_RESPONSES_NONE_REASONING_MODELS.has(model.id) - ) { - mergeThinkingLevelMap(model, { off: "none" }); - } - // xAI models without verified effort options must not send the undocumented - // "none"/"minimal" efforts. - if (model.provider === "xai" && model.api === "openai-responses" && model.thinkingLevelMap === undefined) { - mergeThinkingLevelMap(model, { off: null, minimal: null }); - } - if (supportsOpenAiXhigh(model.id)) { - mergeThinkingLevelMap(model, { xhigh: "xhigh" }); - } - if (supportsOpenAiMax(model)) { - mergeThinkingLevelMap(model, { max: "max" }); - } - if (model.provider === "openai" && model.id === "gpt-5.5") { - mergeThinkingLevelMap(model, { minimal: null }); - } - if (model.id.endsWith("gpt-5.5-pro")) { - mergeThinkingLevelMap(model, { off: null, minimal: null, low: null }); - } - // Anthropic adaptive-thinking effort support (per Anthropic adaptive thinking docs): - // - "max" is available on all adaptive-thinking Claude models. - // - "xhigh" is only available on Opus 4.7/4.8/5, Sonnet 5, Haiku 5.5, and Fable 5. - if ( - model.id.includes("opus-4-6") || - model.id.includes("opus-4.6") || - model.id.includes("sonnet-4-6") || - model.id.includes("sonnet-4.6") - ) { - mergeThinkingLevelMap(model, { max: "max" }); - } - if ( - model.id.includes("opus-4-7") || - model.id.includes("opus-4.7") || - model.id.includes("opus-4-8") || - model.id.includes("opus-4.8") || - model.id.includes("opus-5") || - model.id.includes("opus.5") || - model.id.includes("sonnet-5") || - model.id.includes("sonnet.5") || - model.id.includes("haiku-5") || - model.id.includes("haiku.5") - ) { - mergeThinkingLevelMap(model, { xhigh: "xhigh", max: "max" }); - } - if (model.id.includes("fable-5")) { - mergeThinkingLevelMap(model, { off: null, xhigh: "xhigh", max: "max" }); - } - if (model.api === "anthropic-messages" && isAnthropicAdaptiveThinkingModel(model.id)) { - mergeAnthropicMessagesCompat(model, { forceAdaptiveThinking: true }); - } - if (model.api === "anthropic-messages" && isAnthropicTemperatureUnsupportedModel(model.id)) { - mergeAnthropicMessagesCompat(model, { supportsTemperature: false }); - } - if ( - model.api === "openai-completions" && - model.id.includes("deepseek-v4") && - model.thinkingLevelMap === undefined - ) { - mergeThinkingLevelMap( - model, - model.provider === "openrouter" - ? { ...DEEPSEEK_V4_THINKING_LEVEL_MAP, xhigh: "xhigh", max: null } - : model.provider === "azure" - ? AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP - : (model.provider === "deepseek" || model.provider === "opencode" || model.provider === "opencode-go") && - model.id.includes("deepseek-v4-flash") - ? DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP - : DEEPSEEK_V4_THINKING_LEVEL_MAP, - ); - } - if (model.provider === "groq" && model.id === "qwen/qwen3.6-27b") { - mergeThinkingLevelMap(model, { minimal: null, low: null, medium: null, high: "default" }); - } - if (model.provider === "openai-codex" && supportsOpenAiXhigh(model.id)) { - mergeThinkingLevelMap(model, { minimal: "low" }); - } - if ( - (model.provider === "moonshotai" || model.provider === "moonshotai-cn") && - (model.id === "kimi-k2.7-code" || model.id === "kimi-k2.7-code-highspeed") - ) { - // Kimi K2.7 Code is always-thinking. Official docs say - // `thinking: { type: "disabled" }` is rejected, and callers can omit - // the thinking parameter to use the enabled default. - mergeThinkingLevelMap(model, { off: null }); - } - if (model.provider === "openrouter" && model.id.startsWith("inception/mercury-2")) { - // Mercury 2 in instant mode (reasoning_effort: "none") disables tool calling. - // Mark "off" unsupported so the openai-completions provider omits the reasoning param - // instead of defaulting to {reasoning:{effort:"none"}} (see openai-completions.ts:575). - // Pi's low/medium/high pass through verbatim; OpenRouter normalizes to Mercury's vocabulary. - mergeThinkingLevelMap(model, { off: null }); - } - if (model.provider === "openrouter" && model.id === "z-ai/glm-5.2") { - mergeThinkingLevelMap(model, { xhigh: "xhigh" }); - } - if (model.provider === "fireworks") { - if (model.api === "anthropic-messages" && model.compat?.forceAdaptiveThinking) { - // Qwen Max currently advertises only a toggle. Prefer upstream effort - // metadata once available instead of replacing it with this fallback. - if (model.id === "accounts/fireworks/models/qwen3p8-max" && !model.thinkingLevelMap) { - model.thinkingLevelMap = getEffortThinkingLevelMap([ - { type: "effort", values: ["low", "medium", "xhigh"] }, - ]); - } - const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); - if ( - reasoningOptions?.some((option) => option.type === "toggle") || - // The 2.4T alias omits the verified toggle in models.dev. - model.id === "accounts/fireworks/models/qwen3p8-2p4t-a95b" - ) { - mergeThinkingLevelMap(model, { off: "none" }); - } - if (model.id === "accounts/fireworks/models/deepseek-v4-pro-0813") { - mergeThinkingLevelMap(model, { low: "low" }); - } - } - if (model.id.includes("glm-5p2")) { - // GLM 5.2 and its fast router support off/high/max. Fireworks maps low - // and medium to high, so do not expose those aliases as distinct levels. - mergeThinkingLevelMap(model, { off: "none", minimal: null, low: null, medium: null, max: "max" }); - } - if (model.id.includes("kimi-k3")) { - // Fireworks maps medium to high on both APIs; do not expose it as a distinct level. - mergeThinkingLevelMap(model, { medium: null }); - } - } - if (model.provider === "opencode-go" && model.id === "glm-5.2") { - mergeThinkingLevelMap(model, OPENCODE_GO_GLM52_THINKING_LEVEL_MAP); - } - if (model.provider === "opencode-go" && model.id === "kimi-k2.6") { - // OpenCode Go exposes Kimi K2.6 thinking as on/off, not distinct effort tiers. - mergeThinkingLevelMap(model, { minimal: null, low: null, medium: null }); - } - if (model.provider === "opencode" && model.id === "grok-build-0.1") { - // OpenCode Zen Grok Build reasons by default but rejects explicit reasoningEffort. - mergeThinkingLevelMap(model, { off: null, minimal: null, low: null, medium: null }); - } - if (model.provider === "ant-ling" && model.reasoning) { - // Ring reasons by default. Only high/xhigh have documented explicit effort controls. - mergeThinkingLevelMap(model, ANT_LING_RING_THINKING_LEVEL_MAP); - } - if (model.provider === "github-copilot") { - const override = GITHUB_COPILOT_THINKING_LEVEL_OVERRIDES[model.id]; - if (override) { - mergeThinkingLevelMap(model, override); - } - } -} - -function getAnthropicMessagesCompat(provider: string, modelId: string): AnthropicMessagesCompat | undefined { - const compat: AnthropicMessagesCompat = {}; - if ( - VERIFIED_ANTHROPIC_MID_CONVO_EFFORT_PROVIDERS.has(provider) && - supportsAnthropicMidConvoEffort(modelId) && - !MID_CONVO_EFFORT_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`) - ) { - compat.supportsMidConvoEffort = true; - } - if (provider === "anthropic" && supportsAnthropicMidConvoSystemMessages(modelId)) { - compat.supportsMidConvoSystemMessages = true; - compat.supportsMidConvoToolChanges = true; - } - // OpenCode Zen and GitHub Copilot forward mid-conversation system messages but reject - // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level there. - if ((provider === "opencode" || provider === "github-copilot") && supportsAnthropicMidConvoSystemMessages(modelId)) { - compat.supportsMidConvoSystemMessages = true; - } - if (EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`)) { - compat.supportsEagerToolInputStreaming = false; - } - if (provider === "xiaomi" || provider.startsWith("xiaomi-token-plan-")) { - compat.allowEmptySignature = true; - } - // OpenCode Qwen 3.8 Flash emits and accepts thinking blocks with empty signatures. - if ((provider === "opencode" || provider === "opencode-go") && modelId === "qwen3.8-flash") { - compat.allowEmptySignature = true; - } - return Object.keys(compat).length > 0 ? compat : undefined; -} - -function getBedrockBaseUrl(modelId: string): string { - return modelId.startsWith("eu.") - ? "https://bedrock-runtime.eu-central-1.amazonaws.com" - : "https://bedrock-runtime.us-east-1.amazonaws.com"; -} - -function normalizeNvidiaModelId(modelId: string): string { - return modelId.toLowerCase().replaceAll("_", "."); -} - -function roundCost(value: number): number { - return Number(value.toFixed(6)); -} - -function getModelsDevCost(cost: ModelsDevModel["cost"]): ModelCost { - const base = { - input: cost?.input || 0, - output: cost?.output || 0, - cacheRead: cost?.cache_read || 0, - cacheWrite: cost?.cache_write || 0, - }; - // Rates a tier does not list keep the base price. - const tiers = cost?.tiers?.flatMap((tier) => { - const context = tier.tier; - if (context?.type !== "context" || context.size === undefined) return []; - return [ - { - inputTokensAbove: context.size, - input: tier.input ?? base.input, - output: tier.output ?? base.output, - cacheRead: tier.cache_read ?? base.cacheRead, - cacheWrite: tier.cache_write ?? base.cacheWrite, - }, - ]; - }); - - return tiers && tiers.length > 0 ? { ...base, tiers } : base; -} - -async function fetchNvidiaNimModelIds(): Promise> { - try { - console.log("Fetching models from NVIDIA NIM API..."); - const response = await fetch(`${NVIDIA_BASE_URL}/models`); - if (!response.ok) throw new Error(`NVIDIA NIM API returned ${response.status}`); - const data = (await response.json()) as { data?: NvidiaNimModelListItem[] }; - const modelIds = new Map(); - - for (const model of data.data ?? []) { - modelIds.set(model.id, model.id); - modelIds.set(normalizeNvidiaModelId(model.id), model.id); - } - - console.log(`Fetched ${data.data?.length ?? 0} model IDs from NVIDIA NIM`); - return modelIds; - } catch (error) { - console.error("Failed to fetch NVIDIA NIM models:", error); - if (generatorOptions.strict) throw error; - return new Map(); - } -} - -async function fetchOpenRouterList(query: string): Promise { - const response = await fetch(`https://openrouter.ai/api/v1/models${query}`); - if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); - const data = (await response.json()) as { data?: OpenRouterModelListItem[] }; - return data.data ?? []; -} - -async function fetchOpenRouterModels(): Promise { - try { - console.log("Fetching models from OpenRouter API..."); - const [listed, imageListed, decisionListed] = await Promise.all([ - fetchOpenRouterList(""), - fetchOpenRouterList("?output_modalities=image"), - fetchOpenRouterList("?output_modalities=decisions"), - ]); - const catalog = buildOpenRouterCatalog(listed, imageListed, decisionListed); - console.log( - `Fetched ${catalog.chat.length} tool-capable, ${catalog.images.length} image, and ${catalog.classifiers.length} classifier models from OpenRouter`, - ); - if (generatorOptions.strict && catalog.images.length === 0) { - throw new Error("OpenRouter API returned no usable image models"); - } - return catalog; - } catch (error) { - console.error("Failed to fetch OpenRouter models:", error); - if (generatorOptions.strict) throw error; - return { chat: [], images: [], classifiers: [] }; - } -} - -async function fetchRadiusModels(): Promise[]> { - try { - console.log("Fetching models from Radius API..."); - const config = await loadRadiusGatewayConfig(DEFAULT_RADIUS_GATEWAY); - const models = getRadiusModelsFromConfig("radius", config); - if (models.length === 0) throw new Error("Radius API returned no models"); - console.log(`Fetched ${models.length} models from Radius`); - return models; - } catch (error) { - console.error("Failed to fetch Radius models:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -async function fetchAiGatewayModels(): Promise<{ - chat: Model[]; - classifiers: ClassifierModel<"typesafe-system-one">[]; -}> { - try { - console.log("Fetching models from Vercel AI Gateway API..."); - const response = await fetch(`${AI_GATEWAY_MODELS_URL}/models`); - if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); - const data = await response.json(); - const models: Model[] = []; - const classifiers: ClassifierModel<"typesafe-system-one">[] = []; - - const toNumber = (value: string | number | undefined): number => { - if (typeof value === "number") { - return Number.isFinite(value) ? value : 0; - } - const parsed = parseFloat(value ?? "0"); - return Number.isFinite(parsed) ? parsed : 0; - }; - - const items = Array.isArray(data.data) ? (data.data as AiGatewayModel[]) : []; - for (const model of items) { - // Evaluation models such as TypeSafe's Jev are served through the - // TypeSafe-compatible System One endpoint. - if (model.type === "evaluation") { - classifiers.push({ - type: "classifier", - id: model.id, - name: model.name || model.id, - api: "typesafe-system-one", - provider: "vercel-ai-gateway", - baseUrl: AI_GATEWAY_TYPESAFE_BASE_URL, - input: ["text"], - cost: { - input: roundCost(toNumber(model.pricing?.input) * 1_000_000), - output: roundCost(toNumber(model.pricing?.output) * 1_000_000), - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: model.context_window || 4096, - }); - continue; - } - const tags = Array.isArray(model.tags) ? model.tags : []; - // Only include models that support tools - if (!tags.includes("tool-use")) continue; - - const input: ("text" | "image")[] = ["text"]; - if (tags.includes("vision")) { - input.push("image"); - } - - models.push({ - id: model.id, - name: model.name || model.id, - api: "anthropic-messages", - baseUrl: AI_GATEWAY_BASE_URL, - provider: "vercel-ai-gateway", - reasoning: tags.includes("reasoning"), - input, - compat: { allowEmptySignature: true }, - cost: getAiGatewayCost(model.pricing), - contextWindow: model.context_window || 4096, - maxTokens: model.max_tokens || 4096, - }); - } - - console.log( - `Fetched ${models.length} tool-capable and ${classifiers.length} classifier models from Vercel AI Gateway`, - ); - return { chat: models, classifiers }; - } catch (error) { - console.error("Failed to fetch Vercel AI Gateway models:", error); - if (generatorOptions.strict) throw error; - return { chat: [], classifiers: [] }; - } -} - -function processZaiModels(data: ModelsDevCatalog): Model[] { - const variants = [ - { - source: "zai-coding-plan", - provider: "zai", - baseUrl: "https://api.z.ai/api/coding/paas/v4", - }, - { - source: "zhipuai-coding-plan", - provider: "zai-coding-cn", - baseUrl: "https://open.bigmodel.cn/api/coding/paas/v4", - }, - ] as const; - const models: Model[] = []; - - for (const { source, provider, baseUrl } of variants) { - for (const [modelId, model] of Object.entries(data[source]?.models ?? {})) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - const supportsImage = m.modalities?.input?.includes("image"); - - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - const isGlm52 = modelId === "glm-5.2" || modelId === "glm-5.2-highspeed"; - if (thinkingLevelMap && isGlm52) { - thinkingLevelMap.off = "none"; - } - const supportsReasoningEffort = thinkingLevelMap !== undefined; - const referenceCost = data.zai?.models[modelId]?.cost ?? m.cost; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: supportsImage ? ["text", "image"] : ["text"], - cost: getModelsDevCost(referenceCost), - compat: { - supportsDeveloperRole: false, - thinkingFormat: "zai", - ...(supportsReasoningEffort ? { supportsReasoningEffort: true } : {}), - ...(!ZAI_TOOL_STREAM_UNSUPPORTED_MODELS.has(modelId) ? { zaiToolStream: true } : {}), - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions(provider, modelId, m); - } - } - - return models; -} - -function processBasetenModels(provider: ModelsDevProvider | undefined): Model[] { - if (!provider?.models) return []; - - const baseUrl = "https://inference.baseten.co/v1"; - const baseCompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - supportsUsageInStreaming: true, - maxTokensField: "max_tokens", - supportsStrictMode: true, - // Baseten automatic prompt caching needs session affinity so related - // requests land on the same replica. See: - // https://docs.baseten.co/inference/model-apis/pricing-and-limits - sendSessionAffinityHeaders: true, - supportsLongCacheRetention: false, - }; - const reasoningEffortCompat: OpenAICompletionsCompat = { - ...baseCompat, - supportsReasoningEffort: true, - thinkingFormat: "openai", - }; - const toggleReasoningCompat: OpenAICompletionsCompat = { - ...baseCompat, - thinkingFormat: "baseten", - chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, - }; - const toggleReasoningEffortCompat: OpenAICompletionsCompat = { - ...reasoningEffortCompat, - thinkingFormat: "baseten", - chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, - }; - const toggleThinkingLevelMap = { - off: "off", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: null, - } as const; - const glm52ThinkingLevelMap = { - off: "none", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: "max", - } as const; - const models: Model[] = []; - - for (const [modelId, model] of Object.entries(provider.models)) { - if (model.status === "deprecated") continue; - - const reasoning = model.reasoning === true; - const reasoningOptions = model.reasoning_options ?? []; - const isGlm52 = modelId === "zai-org/GLM-5.2" || modelId === "zai-org/GLM-5.2-Fast"; - const supportsToggle = reasoningOptions.some((option) => option.type === "toggle") || isGlm52; - const supportsEffort = reasoningOptions.some((option) => option.type === "effort") || isGlm52; - const compat = - supportsToggle && supportsEffort - ? toggleReasoningEffortCompat - : supportsToggle - ? toggleReasoningCompat - : supportsEffort - ? reasoningEffortCompat - : baseCompat; - const thinkingLevelMap = isGlm52 - ? glm52ThinkingLevelMap - : supportsToggle - ? toggleThinkingLevelMap - : getEffortThinkingLevelMap(reasoningOptions); - // Baseten's GLM-5.2 endpoints are text-only despite models.dev reporting image input. - const supportsImageInput = !isGlm52 && model.modalities?.input?.includes("image"); - - models.push({ - id: modelId, - name: model.name || modelId, - api: "openai-completions", - provider: "baseten", - baseUrl, - reasoning, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: supportsImageInput ? ["text", "image"] : ["text"], - cost: getModelsDevCost(model.cost), - compat, - contextWindow: model.limit?.context || 4096, - maxTokens: model.limit?.output || 4096, - }); - } - - return models; -} - -function processGoogleModels(data: ModelsDevCatalog): Model[] { - const models: Model[] = []; - const googleModels = data.google?.models; - if (googleModels) { - for (const [modelId, model] of Object.entries(googleModels)) { - if (model.tool_call !== true) continue; - const source = - modelId === "gemini-flash-latest" - ? (googleModels["gemini-3.5-flash"] ?? model) - : modelId === "gemini-flash-lite-latest" - ? (googleModels["gemini-3.1-flash-lite"] ?? model) - : model; - const thinkingLevelMap = getGoogleThinkingLevelMap(modelId, source.reasoning_options ?? []); - - models.push({ - id: modelId, - name: model.name || modelId, - api: "google-generative-ai", - provider: "google", - baseUrl: "https://generativelanguage.googleapis.com/v1beta", - reasoning: source.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: source.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(source.cost), - contextWindow: source.limit?.context || 4096, - maxTokens: source.limit?.output || 4096, - }); - } - } - - // The google-vertex models.dev catalog also includes Claude, OpenAI, and other - // MaaS models that do not use the @google/genai Gemini streaming path. - const vertexModels = data["google-vertex"]?.models; - if (vertexModels) { - for (const [modelId, model] of Object.entries(vertexModels)) { - if (model.tool_call !== true || !modelId.startsWith("gemini-")) continue; - if (modelId === "gemini-3.1-flash-lite-preview") continue; - const source = - modelId === "gemini-flash-latest" - ? (vertexModels["gemini-3.5-flash"] ?? model) - : modelId === "gemini-flash-lite-latest" - ? (vertexModels["gemini-3.1-flash-lite"] ?? model) - : model; - const thinkingLevelMap = getGoogleThinkingLevelMap(modelId, source.reasoning_options ?? []); - // models.dev reports Vertex cache_read/cache_write values for Gemini 2.5 Flash that - // do not match the official Gemini API standard pricing table. pi only accounts - // cachedContentTokenCount as cacheRead. - const cacheRead = modelId === "gemini-2.5-flash" ? 0.03 : source.cost?.cache_read || 0; - - models.push({ - id: modelId, - name: model.name || modelId, - api: "google-vertex", - provider: "google-vertex", - baseUrl: VERTEX_BASE_URL, - reasoning: source.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: source.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: source.cost?.input || 0, - output: source.cost?.output || 0, - cacheRead, - cacheWrite: 0, - }, - contextWindow: source.limit?.context || 4096, - maxTokens: source.limit?.output || 4096, - }); - } - } - - return models; -} - -function processFireworksModels(provider: ModelsDevProvider | undefined): Model[] { - if (!provider?.models) return []; - - const anthropicCompat: AnthropicMessagesCompat = { - allowEmptySignature: true, - sendSessionAffinityHeaders: true, - supportsEagerToolInputStreaming: false, - supportsCacheControlOnTools: false, - supportsLongCacheRetention: false, - }; - const openAICompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - sendSessionAffinityHeaders: true, - supportsLongCacheRetention: false, - }; - const kimiK3Compat: OpenAICompletionsCompat = { - ...openAICompat, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "openai", - }; - const models: Model[] = []; - - for (const [modelId, model] of Object.entries(provider.models)) { - if (model.tool_call !== true) continue; - - const input: ("text" | "image")[] = model.modalities?.input?.includes("image") - ? ["text", "image"] - : ["text"]; - const common = { - id: modelId, - name: model.name || modelId, - provider: "fireworks", - reasoning: model.reasoning === true, - input, - cost: getModelsDevCost(model.cost), - contextWindow: model.limit?.context || 4096, - maxTokens: model.limit?.output || 4096, - }; - - if (modelId.includes("glm-")) { - models.push({ - ...common, - api: "openai-completions", - baseUrl: "https://api.fireworks.ai/inference/v1", - compat: openAICompat, - }); - } else if (modelId.includes("kimi-k3")) { - models.push({ - ...common, - api: "openai-completions", - baseUrl: "https://api.fireworks.ai/inference/v1", - compat: kimiK3Compat, - }); - } else { - models.push({ - ...common, - api: "anthropic-messages", - // Fireworks Anthropic-compatible API - SDK appends /v1/messages. - baseUrl: "https://api.fireworks.ai/inference", - // Fireworks prompt caching uses automatic prefix matching + session affinity. - // x-session-affinity routes requests to the same replica for cache hits. - // cache_control on tools and eager_input_streaming are not supported. - // See: https://docs.fireworks.ai/tools-sdks/anthropic-compatibility - // Use adaptive thinking for cataloged effort controls, with verified - // fallbacks where models.dev is incomplete. New models need no allowlist entry. - compat: { - ...anthropicCompat, - ...(model.reasoning_options?.some((option) => option.type === "effort") || - FIREWORKS_ADAPTIVE_THINKING_FALLBACK_MODELS.has(modelId) - ? { forceAdaptiveThinking: true } - : {}), - }, - }); - } - recordModelsDevReasoningOptions("fireworks", modelId, model); - } - - return models; -} - -// Kimi Coding is subscription-backed, so models.dev reports zero cost. Use the -// equivalent Moonshot API rates to estimate the value of subscription usage. -const KIMI_CODING_IMPLIED_COSTS: Record["cost"]> = { - k3: KIMI_K3_COST, - "kimi-for-coding": { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 }, - "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, -}; -// Models whose https://api.kimi.com/coding/v1/models entry reports effort levels and -// dynamic tool loading (tools declared in later system messages). Highspeed reports -// neither and rejects tool-bearing system messages with a 400. -const KIMI_CODING_K3_STYLE_MODEL_IDS = new Set(["k3", "k3-256k", "kimi-for-coding"]); - -// Kimi Coding uses Chat Completions, the same protocol as the official Kimi Code client. -function createKimiCodingModels(kimiModels: Record): Model<"openai-completions">[] { - const models: Model<"openai-completions">[] = []; - for (const [modelId, m] of Object.entries(kimiModels)) { - if (m.tool_call !== true) continue; - - const isK3Style = KIMI_CODING_K3_STYLE_MODEL_IDS.has(modelId); - const impliedCost = KIMI_CODING_IMPLIED_COSTS[modelId]; - // K3-style models send reasoning_effort, so expose the effort values models.dev lists. - const effortLevels = isK3Style && m.reasoning_options ? getEffortThinkingLevelMap(m.reasoning_options) : undefined; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "kimi-coding", - baseUrl: "https://api.kimi.com/coding/v1", - compat: { - ...(isK3Style ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT), - // All models accept system text after the conversation starts. - supportsMidConvoSystemMessages: true, - ...(isK3Style ? { supportsMidConvoToolAdditions: true } : {}), - }, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || impliedCost?.input || 0, - output: m.cost?.output || impliedCost?.output || 0, - cacheRead: m.cost?.cache_read || impliedCost?.cacheRead || 0, - cacheWrite: m.cost?.cache_write || impliedCost?.cacheWrite || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - // Kimi Code reports every model as always-thinking. - thinkingLevelMap: { ...effortLevels, off: null }, - }); - } - return models; -} - -async function loadModelsDevData(): Promise[]> { - try { - console.log("Fetching models from models.dev API..."); - const response = await fetch("https://models.dev/api.json"); - if (!response.ok) throw new Error(`models.dev API returned ${response.status}`); - const data = (await response.json()) as ModelsDevCatalog; - - const models: Model[] = []; - const nvidiaNimModelIds = data.nvidia?.models ? await fetchNvidiaNimModelIds() : new Map(); - - // Process Amazon Bedrock models - if (data["amazon-bedrock"]?.models) { - for (const [modelId, model] of Object.entries(data["amazon-bedrock"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS.has(modelId)) continue; - - let id = modelId; - - if (id.startsWith("ai21.jamba")) { - // These models doesn't support tool use in streaming mode - continue; - } - - if (id.startsWith("mistral.mistral-7b-instruct-v0")) { - // These models doesn't support system messages - continue; - } - - models.push({ - id, - name: m.name || id, - api: "bedrock-converse-stream" as const, - provider: "amazon-bedrock" as const, - baseUrl: getBedrockBaseUrl(id), - reasoning: m.reasoning === true, - input: (m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"]) as ("text" | "image")[], - // Includes models.dev pricing tiers, e.g. the long-context tier for OpenAI models (#10326). - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(m.structured_output === true && { compat: { supportsStrictMode: true } }), - }); - recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m); - } - } - - // Process Anthropic models - if (data.anthropic?.models) { - for (const [modelId, model] of Object.entries(data.anthropic.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "anthropic-messages", - provider: "anthropic", - baseUrl: "https://api.anthropic.com", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("anthropic", modelId, m); - } - } - - models.push(...processGoogleModels(data)); - - // Process OpenAI models - if (data.openai?.models) { - for (const [modelId, model] of Object.entries(data.openai.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - // models.dev lists this alias, but it is not accepted by OpenAI APIs. - if (MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS.has(modelId)) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "openai", - baseUrl: "https://api.openai.com/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("openai", modelId, m); - } - } - - // Process Groq models - if (data.groq?.models) { - for (const [modelId, model] of Object.entries(data.groq.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "groq", - baseUrl: "https://api.groq.com/openai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("groq", modelId, m); - } - } - - // Process Cerebras models - if (data.cerebras?.models) { - for (const [modelId, model] of Object.entries(data.cerebras.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "cerebras", - baseUrl: "https://api.cerebras.ai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("cerebras", modelId, m); - } - } - - // Process Cloudflare Workers AI models - if (data["cloudflare-workers-ai"]?.models) { - for (const [modelId, model] of Object.entries(data["cloudflare-workers-ai"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_BASE_URL, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat: { sendSessionAffinityHeaders: true }, - }); - recordModelsDevReasoningOptions("cloudflare-workers-ai", modelId, m); - } - } - - // Process Cloudflare AI Gateway models - const cloudflareAIGatewayModelIds = new Set(); - if (data["cloudflare-ai-gateway"]?.models) { - for (const [prefixedId, model] of Object.entries(data["cloudflare-ai-gateway"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - const slashIdx = prefixedId.indexOf("/"); - if (slashIdx === -1) continue; - const upstream = prefixedId.slice(0, slashIdx); - const nativeId = prefixedId.slice(slashIdx + 1); - - let api: "anthropic-messages" | "openai-completions" | "openai-responses"; - let baseUrl: string; - let id: string; - if (upstream === "openai") { - api = "openai-responses"; - baseUrl = CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL; - id = nativeId; - } else if (upstream === "anthropic") { - api = "anthropic-messages"; - baseUrl = CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL; - // The /anthropic passthrough forwards the model ID to Anthropic unchanged. - // models.dev lists dotted versions (claude-opus-5.5), but Anthropic only - // accepts dashed IDs (claude-opus-5-5). - id = nativeId.replaceAll(".", "-"); - } else if (upstream === "workers-ai") { - api = "openai-completions"; - baseUrl = CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL; - id = prefixedId; - } else { - continue; - } - - // Gateway passthroughs forward session affinity headers to upstreams that - // use them for cache/routing affinity. - const compat = - upstream === "anthropic" || upstream === "workers-ai" ? { sendSessionAffinityHeaders: true } : undefined; - - cloudflareAIGatewayModelIds.add(id); - models.push({ - id, - name: m.name || id, - api, - provider: "cloudflare-ai-gateway", - baseUrl, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(compat ? { compat } : {}), - }); - recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m); - } - } - - // The gateway proxies Workers AI through its OpenAI-compatible /compat endpoint, - // but models.dev may omit or intermittently drop those `workers-ai/*` entries - // from the AI Gateway catalog. Mirror the Workers AI catalog under the documented - // prefix so the gateway keeps its OpenAI-compatible models stable. - if (data["cloudflare-workers-ai"]?.models) { - for (const [modelId, model] of Object.entries(data["cloudflare-workers-ai"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - const id = `workers-ai/${modelId}`; - if (cloudflareAIGatewayModelIds.has(id)) continue; - cloudflareAIGatewayModelIds.add(id); - - models.push({ - id, - name: m.name || id, - api: "openai-completions", - provider: "cloudflare-ai-gateway", - baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat: { sendSessionAffinityHeaders: true }, - }); - recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m); - } - } - - // Process xAi models - if (data.xai?.models) { - for (const [modelId, model] of Object.entries(data.xai.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "xai", - baseUrl: "https://api.x.ai/v1", - compat: { ...XAI_RESPONSES_COMPAT }, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("xai", modelId, m); - } - } - - // Process Meta models - if (data.meta?.models) { - for (const [modelId, model] of Object.entries(data.meta.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "meta", - baseUrl: "https://api.meta.ai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("meta", modelId, m); - } - } - - models.push(...processZaiModels(data)); - - // Process Mistral models - if (data.mistral?.models) { - for (const [modelId, model] of Object.entries(data.mistral.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - // Models with effort values use `reasoning_effort` with these levels. - // Reasoning models without them (Magistral) use `prompt_mode`. - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "mistral-conversations", - provider: "mistral", - baseUrl: "https://api.mistral.ai", - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read ?? (m.cost?.input ? roundCost(m.cost.input * 0.1) : 0), - cacheWrite: m.cost?.cache_write || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - } - } - - // Process Hugging Face models - if (data.huggingface?.models) { - for (const [modelId, model] of Object.entries(data.huggingface.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "huggingface", - baseUrl: "https://router.huggingface.co/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: { - supportsDeveloperRole: false, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("huggingface", modelId, m); - } - } - - models.push(...processFireworksModels(data["fireworks-ai"])); - - // Process NVIDIA NIM models - if (data.nvidia?.models) { - for (const [modelId, model] of Object.entries(data.nvidia.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (!m.modalities?.input?.includes("text")) continue; - if (!m.modalities?.output?.includes("text")) continue; - - const liveModelId = nvidiaNimModelIds.get(modelId) ?? nvidiaNimModelIds.get(normalizeNvidiaModelId(modelId)); - if (!liveModelId) continue; - if (NVIDIA_NIM_UNSUPPORTED_MODELS.has(liveModelId)) continue; - - models.push({ - id: liveModelId, - name: m.name || liveModelId, - api: "openai-completions", - provider: "nvidia", - baseUrl: NVIDIA_BASE_URL, - headers: { ...NVIDIA_HEADERS }, - reasoning: m.reasoning === true, - input: m.modalities.input.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: NVIDIA_OPENAI_COMPAT, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("nvidia", liveModelId, m); - } - } - - // Process Together AI models - const togetherProvider = data.together ?? data.togetherai ?? data["together-ai"]; - if (togetherProvider?.models) { - for (const [modelId, model] of Object.entries(togetherProvider.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - const reasoning = m.reasoning === true; - const thinkingLevelMap = getTogetherThinkingLevelMap(modelId, reasoning); - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "together", - baseUrl: TOGETHER_BASE_URL, - reasoning, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: getTogetherCompat(modelId, reasoning), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("together", modelId, m); - } - } - - models.push(...processBasetenModels(data.baseten)); - - // Process OpenCode models (Zen and Go) - // API mapping based on provider.npm field: - // - @ai-sdk/openai → openai-responses - // - @ai-sdk/anthropic → anthropic-messages - // - @ai-sdk/google → google-generative-ai - // - null/undefined/@ai-sdk/openai-compatible → openai-completions - const opencodeVariants = [ - { key: "opencode", provider: "opencode", basePath: "https://opencode.ai/inference" }, - { key: "opencode-go", provider: "opencode-go", basePath: "https://opencode.ai/inference/go" }, - ] as const; - - for (const variant of opencodeVariants) { - if (!data[variant.key]?.models) continue; - - for (const [modelId, model] of Object.entries(data[variant.key].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - const npm = m.provider?.npm; - let api: Api; - let baseUrl: string; - let compat: OpenAICompletionsCompat | OpenAIResponsesCompat | undefined; - - if (npm === "@ai-sdk/openai") { - api = "openai-responses"; - baseUrl = `${variant.basePath}/v1`; - compat = { sessionAffinityFormat: "openai-nosession" }; - } else if (npm === "@ai-sdk/anthropic") { - api = "anthropic-messages"; - // Anthropic SDK appends /v1/messages to baseURL - baseUrl = variant.basePath; - } else if (npm === "@ai-sdk/google") { - api = "google-generative-ai"; - baseUrl = `${variant.basePath}/v1`; - } else if (npm === "@ai-sdk/alibaba") { - api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; - compat = { cacheControlFormat: "anthropic" }; - } else { - // null, undefined, or @ai-sdk/openai-compatible - api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; - } - - if (variant.provider === "opencode" && modelId === "grok-build-0.1") { - compat = { ...(compat ?? {}), supportsReasoningEffort: false }; - } - - if ((variant.provider === "opencode" || variant.provider === "opencode-go") && modelId === "kimi-k2.6") { - // OpenCode Kimi K2.6 accepts Anthropic-style thinking objects - // and rejects string thinking values or combined reasoning_effort. - compat = { ...(compat ?? {}), thinkingFormat: "deepseek", supportsReasoningEffort: false }; - } - - // Fix known mismatches between models.dev npm data and actual - // OpenCode Go endpoint behaviour. models.dev reports these models - // as @ai-sdk/anthropic, but OpenCode Go serves them through the - // OpenAI-compatible /v1/chat/completions path. - if (variant.provider === "opencode-go") { - if (modelId === "qwen3.5-plus" || modelId === "qwen3.6-plus") { - api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; - // Qwen/DashScope uses enable_thinking at the top level. - compat = { ...(compat ?? {}), thinkingFormat: "qwen" }; - } - } - - if (api === "openai-completions") { - compat = { ...(compat ?? {}), maxTokensField: "max_tokens" }; - if ( - OPENCODE_OPENAI_COMPLETIONS_LONG_CACHE_RETENTION_UNSUPPORTED_MODELS.has( - `${variant.provider}:${modelId}`, - ) - ) { - compat = { ...compat, supportsLongCacheRetention: false }; - } - } - - let thinkingLevelMap: NonNullable["thinkingLevelMap"]> | undefined; - if (api === "google-generative-ai") { - thinkingLevelMap = getGoogleThinkingLevelMap(modelId, m.reasoning_options ?? []); - } else if (variant.provider === "opencode-go" && modelId === "deepseek-v4.1-flash") { - thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - } - models.push({ - id: modelId, - name: m.name || modelId, - api, - provider: variant.provider, - baseUrl, - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - ...(compat ? { compat } : {}), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions(variant.provider, modelId, m); - } - } - - // Process GitHub Copilot models - if (data["github-copilot"]?.models) { - for (const [modelId, model] of Object.entries(data["github-copilot"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - // Claude 4.x and 5.x models route to Anthropic Messages API - const isCopilotClaude = /^claude-(haiku|sonnet|opus|fable)-[45]([.\-]|$)/.test(modelId); - // GPT, Grok, OSWE, and MAI-Code models are only served through - // the Copilot /responses endpoint. - const needsResponsesApi = - modelId.startsWith("gpt-") || - modelId.startsWith("grok-") || - modelId.startsWith("oswe") || - modelId.startsWith("mai-"); - - const api: Api = isCopilotClaude - ? "anthropic-messages" - : needsResponsesApi - ? "openai-responses" - : "openai-completions"; - - const anthropicCompat = - api === "anthropic-messages" ? getAnthropicMessagesCompat("github-copilot", modelId) : undefined; - - const copilotModel: Model = { - id: modelId, - name: m.name || modelId, - api, - provider: "github-copilot", - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 128000, - maxTokens: m.limit?.output || 8192, - headers: { ...COPILOT_STATIC_HEADERS }, - ...(anthropicCompat ? { compat: anthropicCompat } : {}), - // compat only applies to openai-completions - ...(api === "openai-completions" ? { - compat: { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - }, - } : {}), - }; - - models.push(copilotModel); - recordModelsDevReasoningOptions("github-copilot", modelId, m); - } - } - - // Process MiniMax models - const minimaxVariants = [ - { key: "minimax", provider: "minimax", baseUrl: "https://api.minimax.io/anthropic" }, - { key: "minimax-cn", provider: "minimax-cn", baseUrl: "https://api.minimaxi.com/anthropic" }, - ] as const; - // MiniMax asked to hide older models from its direct providers (#2596). - const minimaxHiddenIds = new Set(["MiniMax-M2", "MiniMax-M2.1", "MiniMax-M2.5", "MiniMax-M2.5-highspeed"]); - - for (const { key, provider, baseUrl } of minimaxVariants) { - if (data[key]?.models) { - for (const [modelId, model] of Object.entries(data[key].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true || minimaxHiddenIds.has(modelId)) continue; - // Models with effort levels (e.g. MiniMax-M3.1-Flash) take adaptive thinking with - // output_config.effort and reject thinking.type="disabled" (no "none" effort, so off is null). - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "anthropic-messages", - provider, - // MiniMax's Anthropic-compatible API - SDK appends /v1/messages - baseUrl, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(thinkingLevelMap ? { thinkingLevelMap, compat: { forceAdaptiveThinking: true } } : {}), - }); - } - } - } - - // Process Kimi For Coding models - if (data["kimi-code-plan-global"]?.models) { - models.push(...createKimiCodingModels(data["kimi-code-plan-global"].models as Record)); - } - - // Process Moonshot AI models - const moonshotVariants = [ - { key: "moonshotai", provider: "moonshotai", baseUrl: "https://api.moonshot.ai/v1" }, - { key: "moonshotai-cn", provider: "moonshotai-cn", baseUrl: "https://api.moonshot.cn/v1" }, - ] as const; - const getMoonshotProviderModels = (key: "moonshotai" | "moonshotai-cn"): Record => { - const providerModels = data[key]?.models as Record | undefined; - return providerModels ? { ...providerModels } : {}; - }; - const moonshotModels = { - moonshotai: getMoonshotProviderModels("moonshotai"), - "moonshotai-cn": getMoonshotProviderModels("moonshotai-cn"), - }; - - for (const { key, provider, baseUrl } of moonshotVariants) { - for (const [modelId, m] of Object.entries(moonshotModels[key])) { - if (m.tool_call !== true) continue; - - const isKimiK3 = modelId === "kimi-k3"; - const compat = { ...(isKimiK3 ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT) }; - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - reasoning: isKimiK3 || m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - // Moonshot does not bill cache writes; models.dev lists the input rate as cache_write for Kimi K3. - cost: isKimiK3 - ? { ...KIMI_K3_COST } - : { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: m.cost?.cache_write || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat, - }); - recordModelsDevReasoningOptions(provider, modelId, m); - } - } - - // Process Xiaomi MiMo models - // Built-in `xiaomi` targets the API billing endpoint (single stable URL, - // keys from platform.xiaomimimo.com). The three `xiaomi-token-plan-*` - // providers cover prepaid Token Plan endpoints in cn / ams / sgp. - // MiMo Chat Completions only documents thinking.type enabled/disabled and - // rejects some reasoning_effort values (e.g. "minimal"). Xiaomi documents - // all effort levels as identical, so expose only off and high. - const xiaomiCompat: OpenAICompletionsCompat = { - supportsReasoningEffort: false, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "deepseek", - }; - const xiaomiThinkingLevelMap = { - off: "off", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: null, - } as const; - const xiaomiVariants = [ - { source: "xiaomi", provider: "xiaomi", baseUrl: "https://api.xiaomimimo.com/v1" }, - { - source: "xiaomi-token-plan-cn", - provider: "xiaomi-token-plan-cn", - baseUrl: "https://token-plan-cn.xiaomimimo.com/v1", - }, - { - source: "xiaomi-token-plan-ams", - provider: "xiaomi-token-plan-ams", - baseUrl: "https://token-plan-ams.xiaomimimo.com/v1", - }, - { - source: "xiaomi-token-plan-sgp", - provider: "xiaomi-token-plan-sgp", - baseUrl: "https://token-plan-sgp.xiaomimimo.com/v1", - }, - ] as const; - - for (const { source, provider, baseUrl } of xiaomiVariants) { - const providerModels = data[source]?.models; - if (!providerModels) continue; - - for (const [modelId, model] of Object.entries(providerModels)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - compat: xiaomiCompat, - reasoning: m.reasoning === true, - ...(m.reasoning === true ? { thinkingLevelMap: xiaomiThinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - } - } - - // Process Alibaba Cloud Model Studio Token Plan models. International and - // China use separate endpoints and API keys (sk-sp- prefix). The Individual - // provider reuses the international source and endpoint with a narrower catalog. - // models.dev keys are "alibaba-token-plan[-cn]"; pi exposes them as - // "qwen-token-plan[-cn]" plus the Individual catalog view. - const qwenTokenPlanCompat: OpenAICompletionsCompat = { - thinkingFormat: "qwen", - supportsDeveloperRole: false, - supportsStore: false, - supportsReasoningEffort: true, - }; - const qwenTokenPlanVariants = [ - { - source: "alibaba-token-plan", - provider: "qwen-token-plan", - baseUrl: "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", - modelIds: undefined, - }, - { - source: "alibaba-token-plan", - provider: "qwen-token-plan-individual", - baseUrl: "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", - modelIds: QWEN_TOKEN_PLAN_INDIVIDUAL_MODEL_IDS, - }, - { - source: "alibaba-token-plan-cn", - provider: "qwen-token-plan-cn", - baseUrl: "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", - modelIds: undefined, - }, - ] as const; - - for (const { source, provider, baseUrl, modelIds } of qwenTokenPlanVariants) { - const providerModels = data[source]?.models; - const emittedModelIds = modelIds ? new Set() : undefined; - - for (const [modelId, model] of Object.entries(providerModels ?? {})) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (QWEN_TOKEN_PLAN_EXCLUDED_MODEL_IDS.has(modelId)) continue; - if (modelIds && !modelIds.has(modelId)) continue; - const thinkingLevelMap = - getEffortThinkingLevelMap(m.reasoning_options ?? []) ?? - (QWEN_TOKEN_PLAN_REASONING_EFFORT_FALLBACK_MODEL_IDS.has(modelId) - ? QWEN_TOKEN_PLAN_FALLBACK_THINKING_LEVEL_MAP - : undefined); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - compat: thinkingLevelMap - ? qwenTokenPlanCompat - : { ...qwenTokenPlanCompat, supportsReasoningEffort: false }, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - emittedModelIds?.add(modelId); - } - - if (modelIds && emittedModelIds && generatorOptions.strict) { - assertExactModelIds(provider, modelIds, emittedModelIds); - } - } - - console.log(`Loaded ${models.length} tool-capable models from models.dev`); - return models; - } catch (error) { - console.error("Failed to load models.dev data:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -async function loadModelsDevClassifierModels(): Promise[]> { - try { - console.log("Fetching classifier models from models.dev API..."); - const response = await fetch("https://models.dev/models.json?type=decision"); - if (!response.ok) throw new Error(`models.dev classifier API returned ${response.status}`); - const data = (await response.json()) as Record; - const metadata = data["typesafe/jev-latest"]; - if (!metadata || metadata.type !== "decision") { - throw new Error("models.dev did not return decision model typesafe/jev-latest"); - } - return [ - { - type: "classifier", - id: "jev-latest", - name: metadata.name, - api: "typesafe-system-one", - provider: "typesafe", - baseUrl: "https://api.typesafe.ai/v1/", - // models.dev lists text input only, but the API accepts image assets as an experimental - // feature behind an opt-in header, which the typesafe-system-one API sends. - // https://github.com/typesafe-ai/typesafe-sdk-js (v0.7.0 changelog) - input: ["text", "image"], - // The canonical models.dev entry has no direct-provider pricing. System One reports token usage, - // so classify() results carry token counts but price them at zero. - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: metadata.limit?.context || 64000, - compat: { imageInput: "assets", sendImageOptInHeader: true }, - }, - ]; - } catch (error) { - console.error("Failed to load models.dev classifier data:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -// OpenCode Zen serves Jev through its TypeSafe-compatible System One endpoint. -// Neither its /inference/v1/models listing nor models.dev carries metadata for it. -// https://opencode.ai/docs/zen -const OPENCODE_CLASSIFIER_MODELS: ClassifierModel<"typesafe-system-one">[] = [ - { - type: "classifier", - id: "jev-1.13", - name: "Jev 1.13", - api: "typesafe-system-one", - provider: "opencode", - baseUrl: "https://opencode.ai/inference/v1", - input: ["text"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - }, - { - type: "classifier", - id: "jev-1.13-free", - name: "Jev 1.13 Free", - api: "typesafe-system-one", - provider: "opencode", - baseUrl: "https://opencode.ai/inference/v1", - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - }, -]; - -// Workers AI has no unauthenticated catalog, so its System One models are listed here from -// Cloudflare's model pages. -const CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS: ClassifierModel<"cloudflare-workers-ai-system-one">[] = [ - // Cloudflare-hosted Clef decision models. They take up to four PNG, JPEG, or WebP images as data URLs. - // Clef Omni also takes audio and video, which classifier contexts do not carry. - // Pricing and context: https://developers.cloudflare.com/workers-ai/models/clef/, - // https://developers.cloudflare.com/workers-ai/models/clef-flash/, - // and https://developers.cloudflare.com/workers-ai/models/clef-omni/ - { - type: "classifier", - id: "@cf/cloudflare/clef", - name: "Clef", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.24, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 65536, - compat: { imageInput: "images" }, - }, - { - type: "classifier", - id: "@cf/cloudflare/clef-flash", - name: "Clef Flash", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.038, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 24576, - compat: { imageInput: "images" }, - }, - { - type: "classifier", - id: "@cf/cloudflare/clef-omni", - name: "Clef Omni", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.15, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 64000, - compat: { imageInput: "images" }, - }, - // TypeSafe's Jev models take PNG and JPEG images as System One assets. - // https://developers.cloudflare.com/ai/models/typesafe/jev/ (jev-latest) - // and https://developers.cloudflare.com/ai/models/typesafe/jev-1.14.0/ - { - type: "classifier", - id: "typesafe/jev", - name: "Jev", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - compat: { imageInput: "assets" }, - }, - { - type: "classifier", - id: "typesafe/jev-1.14.0", - name: "Jev 1.14", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - compat: { imageInput: "assets" }, - }, -]; - -// OpenAI Decisions API (public beta): gpt-6-luna is its only model. It bills input tokens only, -// with the same long-context multiplier as chat requests. Only API keys work: Sign in with ChatGPT -// tokens are rejected on /v1/decisions, and the Codex backend has no Decisions route. -// The endpoint rejects inputs above 922K tokens (the model's documented maximum input), but -// requests running longer than about five seconds, currently above roughly 600K input tokens, -// fail with a gateway timeout. -// https://developers.openai.com/api/docs/guides/decisions -const OPENAI_CLASSIFIER_MODELS: ClassifierModel<"openai-decisions">[] = [ - { - type: "classifier", - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-decisions", - provider: "openai", - baseUrl: "https://api.openai.com/v1", - input: ["text", "image"], - cost: withOpenAiLongContextPricing({ input: 0.1, output: 0, cacheRead: 0, cacheWrite: 0 }), - contextWindow: 922000, - }, -]; - -async function generateModels() { - // Fetch models from all upstream catalogs. - // models.dev: Anthropic, Google, OpenAI, Groq, Cerebras, and others - // OpenRouter: its tool-capable routed catalog - // AI Gateway: OpenAI-compatible catalog with tool-capable models - // Radius: its unauthenticated public catalog; authenticated clients overlay it at runtime - const modelsDevModels = await loadModelsDevData(); - const modelsDevClassifierModels = await loadModelsDevClassifierModels(); - const openRouterCatalog = await fetchOpenRouterModels(); - const aiGatewayCatalog = await fetchAiGatewayModels(); - const radiusModels = await fetchRadiusModels(); - - // Combine chat models (models.dev has priority where sources overlap). - const allModels = [...modelsDevModels, ...openRouterCatalog.chat, ...aiGatewayCatalog.chat, ...radiusModels].filter( - (model) => - !(model.provider === "xai" && XAI_BUILTIN_EXCLUDED_MODEL_IDS.has(model.id)) && - !((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "gpt-5.3-codex-spark"), - ); - - // The authenticated Copilot catalog advertised these models on 2026-09-22, - // but models.dev did not include them yet. - const missingCopilotModels: Model[] = [ - { - id: "claude-opus-5.5", - name: "Claude Opus 5.5", - api: "anthropic-messages", - provider: "github-copilot", - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: true, - thinkingLevelMap: { - off: null, - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }, - input: ["text", "image"], - cost: { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 }, - contextWindow: 1000000, - maxTokens: 128000, - headers: { ...COPILOT_STATIC_HEADERS }, - }, - ...(["gpt-6-sol", "gpt-6-luna"] as const).map((id) => ({ - id, - name: id === "gpt-6-sol" ? "GPT-6 Sol" : "GPT-6 Luna", - api: "openai-responses" as const, - provider: "github-copilot" as const, - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: true, - input: ["text", "image"] as ("text" | "image")[], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[id]), - contextWindow: 1000000, - maxTokens: 128000, - headers: { ...COPILOT_STATIC_HEADERS }, - })), - ]; - for (const model of missingCopilotModels) { - if (!allModels.some((candidate) => candidate.provider === model.provider && candidate.id === model.id)) { - allModels.push(model); - } - } - - // Temporary overrides until upstream model metadata is corrected. - for (const candidate of allModels) { - if (candidate.provider === "github-copilot" && GITHUB_COPILOT_EXTENDED_CONTEXT_MODELS.has(candidate.id)) { - candidate.contextWindow = 1000000; - } - - // models.dev may list Opus 5.5, Sonnet 5.5, and Haiku 5.5 before their effort metadata is complete. - if ( - (candidate.provider === "anthropic" && - (candidate.id === "claude-opus-5-5" || - candidate.id === "claude-sonnet-5-5" || - candidate.id === "claude-haiku-5-5")) || - (candidate.provider === "github-copilot" && candidate.id === "claude-opus-5.5") - ) { - mergeThinkingLevelMap(candidate, { - // Haiku 5.5 accepts disabled thinking; Opus 5.5 and Sonnet 5.5 reject it. - ...(candidate.id === "claude-haiku-5-5" ? {} : { off: null }), - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }); - } - - if ( - (candidate.provider === "anthropic" || - candidate.provider === "opencode" || - candidate.provider === "opencode-go") && - (candidate.id === "claude-opus-4-6" || - candidate.id === "claude-sonnet-4-6" || - candidate.id === "claude-opus-4.6" || - candidate.id === "claude-sonnet-4.6") - ) { - candidate.contextWindow = 1000000; - } - - // OpenCode variants list Claude Sonnet 4/4.5 with 1M context, actual limit is 200K - if ( - (candidate.provider === "opencode" || candidate.provider === "opencode-go") && - (candidate.id === "claude-sonnet-4-5" || candidate.id === "claude-sonnet-4") - ) { - candidate.contextWindow = 200000; - } - if ((candidate.provider === "opencode" || candidate.provider === "opencode-go") && candidate.id === "gpt-5.4") { - candidate.contextWindow = 272000; - candidate.maxTokens = 128000; - } - // Keep direct OpenAI requests in the short-context pricing tier by default. Users can opt into the - // larger context through model overrides, so retain long-context cost metadata on the capped models. - if (candidate.provider === "openai" && OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS.has(candidate.id)) { - candidate.contextWindow = OPENAI_LONG_CONTEXT_INPUT_THRESHOLD; - candidate.maxTokens = 128000; - } - if (candidate.provider === "openai" && OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS.has(candidate.id)) { - const standardCost = OPENAI_STANDARD_COSTS[candidate.id]; - candidate.cost = withOpenAiLongContextPricing(standardCost ?? candidate.cost); - } - // Cloudflare AI Gateway passes OpenAI usage through at OpenAI list prices. - if (candidate.provider === "cloudflare-ai-gateway") { - const standardCost = OPENAI_STANDARD_COSTS[candidate.id]; - if (standardCost) candidate.cost = withOpenAiLongContextPricing(standardCost); - } - // models.dev reports gpt-5-pro output as 272000 (a duplicate of the input sub-limit); - // the actual max output is 128000. Also propagates to the derived Azure clone. - if (candidate.provider === "openai" && candidate.id === "gpt-5-pro") { - candidate.maxTokens = 128000; - } - // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. - if ( - (candidate.provider === "openrouter" && OPENROUTER_KIMI_K3_MODEL_IDS.has(candidate.id)) || - (candidate.provider === "vercel-ai-gateway" && candidate.id === "moonshotai/kimi-k3") - ) { - candidate.maxTokens = KIMI_K3_MAX_TOKENS; - } - // Keep selected OpenRouter model metadata stable until upstream settles. - if (candidate.provider === "openrouter" && candidate.id === "moonshotai/kimi-k2.5") { - candidate.cost.input = 0.41; - candidate.cost.output = 2.06; - candidate.cost.cacheRead = 0.07; - candidate.maxTokens = 4096; - } - if (candidate.provider === "openrouter" && candidate.id.startsWith("moonshotai/kimi-k2.6")) { - candidate.compat = { - ...candidate.compat, - supportsDeveloperRole: false, - requiresReasoningContentOnAssistantMessages: true, - }; - } - if (candidate.provider === "openrouter" && candidate.id === "z-ai/glm-5") { - candidate.cost.input = 0.6; - candidate.cost.output = 1.9; - candidate.cost.cacheRead = 0.119; - } - } - - // Add missing gpt models - const missingOpenAiModels: Model<"openai-responses">[] = [ - { - id: "gpt-6.1-sol", - name: "GPT-6.1 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-astra", - name: "GPT-6 Astra", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-astra"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-sol", - name: "GPT-6 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-luna"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-sol", - name: "GPT-5.6 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-terra", - name: "GPT-5.6 Terra", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-terra"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-luna", - name: "GPT-5.6 Luna", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-luna"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5-chat-latest", - name: "GPT-5 Chat Latest", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: false, - input: ["text", "image"], - cost: { - input: 1.25, - output: 10, - cacheRead: 0.125, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 16384, - }, - ]; - for (const model of missingOpenAiModels) { - if (!allModels.some((m) => m.provider === model.provider && m.id === model.id)) { - allModels.push(model); - } - } - - const deepseekCompat: OpenAICompletionsCompat = { - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "deepseek", - }; - const deepseekModels: Model<"openai-completions">[] = [ - { - id: "deepseek-flash", - name: "DeepSeek V4.1 Flash", - api: "openai-completions", - baseUrl: "https://api.deepseek.com", - provider: "deepseek", - reasoning: true, - thinkingLevelMap: DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP, - input: ["text", "image"], - cost: { - // DeepSeek also offers time-based off-peak rates, which the cost schema cannot represent yet. - input: 0.3, - output: 1.2, - cacheRead: 0.006, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 384000, - compat: deepseekCompat, - }, - { - id: "deepseek-v4-pro", - name: "DeepSeek V4 Pro", - api: "openai-completions", - baseUrl: "https://api.deepseek.com", - provider: "deepseek", - reasoning: true, - input: ["text"], - cost: { - // DeepSeek also offers time-based off-peak rates, which the cost schema cannot represent yet. - input: 1.32, - output: 3.96, - cacheRead: 0.044, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 384000, - compat: deepseekCompat, - }, - ]; - allModels.push(...deepseekModels); - - const antLingCompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsLongCacheRetention: false, - }; - const antLingModels: Model<"openai-completions">[] = [ - { - id: "Ling-2.6-flash", - name: "Ling 2.6 Flash", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: false, - input: ["text"], - cost: { input: 0.01, output: 0.02, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: antLingCompat, - }, - { - id: "Ling-2.6-1T", - name: "Ling 2.6 1T", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: false, - input: ["text"], - cost: { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: antLingCompat, - }, - { - id: "Ring-2.6-1T", - name: "Ring 2.6 1T", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: true, - input: ["text"], - cost: { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: { ...antLingCompat, thinkingFormat: "ant-ling" }, - }, - ]; - allModels.push(...antLingModels); - - for (const candidate of allModels) { - if ( - candidate.api === "openai-completions" && - candidate.id.includes("deepseek-v4") && - !QWEN_TOKEN_PLAN_PROVIDER_IDS.has(candidate.provider) - ) { - const preservesNativeReasoningEffort = candidate.provider === "openrouter" || candidate.provider === "opencode"; - candidate.compat = { - ...candidate.compat, - ...(preservesNativeReasoningEffort - ? { - requiresReasoningContentOnAssistantMessages: - deepseekCompat.requiresReasoningContentOnAssistantMessages, - } - : deepseekCompat), - }; - } - } - - // OpenAI Codex (ChatGPT OAuth) models - // NOTE: These are not fetched from models.dev; we keep a small, explicit list to avoid aliases. - // Only list models the Codex backend accepts for ChatGPT accounts. GPT-5.5 and GPT-5.3 Codex Spark are rejected - // with "not supported when using Codex with a ChatGPT account". - // GPT-5.6 and GPT-6 use Codex's 272k default catalog limit. - // GPT-6.1 Sol uses the 500k default returned by /backend-api/codex/models. - const CODEX_BASE_URL = "https://chatgpt.com/backend-api"; - const CODEX_CONTEXT = 272000; - const CODEX_GPT_61_SOL_CONTEXT = 500000; - const CODEX_GPT_56_CONTEXT = 272000; - const CODEX_MAX_TOKENS = 128000; - const codexModels: Model<"openai-codex-responses">[] = [ - { - id: "gpt-6.1-sol", - name: "GPT-6.1 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-sol"]), - contextWindow: CODEX_GPT_61_SOL_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-astra", - name: "GPT-6 Astra", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-astra"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-sol", - name: "GPT-6 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-sol"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-luna"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-luna", - name: "GPT-5.6 Luna", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-luna"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-sol", - name: "GPT-5.6 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-sol"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-terra", - name: "GPT-5.6 Terra", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-terra"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - ]; - allModels.push(...codexModels); - - // Add missing Mistral Medium 3.5 model until models.dev includes it - if (!allModels.some(m => m.provider === "mistral" && m.id === "mistral-medium-3.5")) { - allModels.push({ - id: "mistral-medium-3.5", - name: "Mistral Medium 3.5", - api: "mistral-conversations", - provider: "mistral", - baseUrl: "https://api.mistral.ai", - reasoning: true, - thinkingLevelMap: getEffortThinkingLevelMap([{ type: "effort", values: ["none", "high"] }]), - input: ["text", "image"], - cost: { - input: 1.5, - output: 7.5, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 262144, // 256k tokens - maxTokens: 262144, - }); - } - - // Add "auto" alias for openrouter/auto - if (!allModels.some(m => m.provider === "openrouter" && m.id === "auto")) { - allModels.push({ - id: "auto", - name: "Auto", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text", "image"], - cost: { - // we dont know about the costs because OpenRouter auto routes to different models - // and then charges you for the underlying used model - input:0, - output:0, - cacheRead:0, - cacheWrite:0, - }, - contextWindow: 2000000, - maxTokens: 30000, - }); - } - - // Add "fusion" alias for openrouter/fusion. OpenRouter exposes Fusion as a - // router alias/plugin entry point; its model metadata does not advertise - // tools, but the alias resolves to a concrete model that can invoke caller - // tools and has the openrouter:fusion server tool auto-injected. - if (!allModels.some(m => m.provider === "openrouter" && m.id === "openrouter/fusion")) { - allModels.push({ - id: "openrouter/fusion", - name: "OpenRouter: Fusion", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text"], - cost: { - // we dont know about the costs because Fusion routes to multiple models - // and then charges you for the underlying used models - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 30000, - }); - } - - // Azure Foundry deploys these with larger context windows than OpenAI's own short-tier defaults. - // See models-sold-directly-by-azure docs. - const AZURE_CONTEXT_WINDOW_OVERRIDES: Record = { - "gpt-5.4": 1050000, - "gpt-5.5": 1050000, - "gpt-5.6-luna": 1050000, - "gpt-5.6-sol": 1050000, - "gpt-5.6-terra": 1050000, - }; - const azureOpenAiModels: Model[] = allModels - .filter((model) => model.provider === "openai" && model.api === "openai-responses") - .map((model) => ({ - ...model, - api: "azure-openai-responses", - provider: "azure", - baseUrl: "", - cost: { - input: model.cost.input, - output: model.cost.output, - cacheRead: model.cost.cacheRead, - cacheWrite: model.cost.cacheWrite, - }, - contextWindow: AZURE_CONTEXT_WINDOW_OVERRIDES[model.id] ?? model.contextWindow, - })); - allModels.push(...azureOpenAiModels); - - // Azure resells DeepSeek at its own rates. US data zone, checked 2026-09-16. - // https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/ - const AZURE_DEEPSEEK_V4_PRO_COST: ModelCost = { input: 1.925, output: 3.828, cacheRead: 0.165, cacheWrite: 0 }; - // Azure 400s on DeepSeek's `thinking` field and on every prompt cache parameter, discards a - // `developer` system message unbilled once reasoning_effort is set, and honours mid-convo ones (#9645). - const azureDeepSeekModels: Model[] = allModels - .filter((model) => model.provider === "deepseek" && model.id === "deepseek-v4-pro") - .map((model) => ({ - ...model, - provider: "azure", - baseUrl: "", - cost: AZURE_DEEPSEEK_V4_PRO_COST, - compat: { - ...(model.compat as OpenAICompletionsCompat), - supportsDeveloperRole: false, - supportsMidConvoSystemMessages: true, - thinkingFormat: "openai", - supportsLongCacheRetention: false, - }, - })); - allModels.push(...azureDeepSeekModels); - - for (const model of allModels) { - applyOpenAICompletionsCompatMetadata(model); - applyAnthropicMessagesCompatMetadata(model); - applyModelsDevReasoningOptionMetadata(model); - applyThinkingLevelMetadata(model); - applyStrictToolCompatMetadata(model); - applyOpenAIGrammarToolCompatMetadata(model); - applyOpenAIToolSearchMetadata(model); - applyOpenAICompletionsTranscriptMetadata(model); - applyOpenAIResponsesTranscriptMetadata(model); - applyOpenAIExplicitPromptCacheMetadata(model); - applyOpenAIMidConvoEffortMetadata(model); - applyPromptCacheMetadata(model); - applyImageInputMetadata(model); - } - applyAnthropicAllowedFallbackModelMetadata(allModels.filter(isAnthropicFallbackMetadataModel)); - - // Keep chat and image catalogs separate so one upstream ID can expose both - // operations with different API implementations. - type ProviderCatalog = { - chat: Record>; - image: Record>; - classifier: Record>; - }; - const providers: Record = {}; - for (const model of allModels) { - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - // Only add if not already present (models.dev takes priority over OpenRouter). - providers[model.provider].chat[model.id] ??= { ...model, type: "chat" }; - } - for (const model of openRouterCatalog.images) { - applyImageInputMetadata(model); - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - providers[model.provider].image[model.id] ??= model; - } - const classifierModels: ClassifierModel[] = [ - ...modelsDevClassifierModels, - ...openRouterCatalog.classifiers, - ...aiGatewayCatalog.classifiers, - ...OPENCODE_CLASSIFIER_MODELS, - ...CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, - ...OPENAI_CLASSIFIER_MODELS, - ]; - for (const model of classifierModels) { - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - providers[model.provider].classifier[model.id] ??= model; - } - - const sortedProviderIds = Object.keys(providers).sort(); - const jsonChatProviders: Record>> = {}; - const jsonImageProviders: Record>> = {}; - const jsonClassifierProviders: Record>> = {}; - const jsonAllProviders: Record = {}; - for (const providerId of sortedProviderIds) { - jsonChatProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].chat).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonImageProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].image).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonClassifierProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].classifier).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonAllProviders[providerId] = [ - ...Object.values(jsonChatProviders[providerId]), - ...Object.values(jsonImageProviders[providerId]), - ...Object.values(jsonClassifierProviders[providerId]), - ]; - } - - const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`; - const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value)); - const generatedDataProviderIds = generatorOptions.dataOnly - ? readModelDataProviderIds(packageRoot) - : sortedProviderIds; - const missingProviderIds = generatedDataProviderIds.filter((providerId) => !jsonAllProviders[providerId]); - if (missingProviderIds.length > 0) { - throw new Error(`Cannot hydrate missing providers: ${missingProviderIds.join(", ")}`); - } - - // Only the ignored internal data is grouped by API for type derivation. - const generatedDataProviders: Record>> = {}; - const modelDataStructure: ModelDataStructure = {}; - for (const providerId of generatedDataProviderIds) { - const { groups, structure } = groupProviderModelData(providerId, jsonAllProviders[providerId]); - generatedDataProviders[providerId] = groups; - modelDataStructure[providerId] = structure; - } - - const generatedAt = new Date().toISOString(); - - if (!generatorOptions.jsonOnly) { - // Stage and validate all provider values before replacing the current generated data. - const providersDir = join(packageRoot, "src/providers"); - const dataDir = join(providersDir, "data"); - const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-")); - const stagedDataDir = join(stagingRoot, "data"); - const previousDataDir = join(stagingRoot, "previous-data"); - let restoreGeneratedCatalog: (() => void) | undefined; - try { - mkdirSync(stagedDataDir, { recursive: true }); - const fileContents: Record = {}; - for (const providerId of generatedDataProviderIds) { - const filename = `${providerId}.json`; - const content = serializeJson(generatedDataProviders[providerId]); - fileContents[filename] = content; - writeFileSync(join(stagedDataDir, filename), content); - } - writeJson( - join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), - createModelDataManifest(modelDataStructure, fileContents, generatedAt), - ); - validateModelDataDirectory(modelDataStructure, stagedDataDir); - - if (!generatorOptions.dataOnly) { - const previousShardContents = new Map( - readdirSync(providersDir) - .filter((entry) => entry.endsWith(".models.ts")) - .map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const), - ); - const aggregatorPath = join(packageRoot, "src/models.generated.ts"); - const previousAggregator = readFileSync(aggregatorPath, "utf8"); - restoreGeneratedCatalog = () => { - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry)); - } - for (const [entry, content] of previousShardContents) { - writeFileSync(join(providersDir, entry), content); - } - writeFileSync(aggregatorPath, previousAggregator); - }; - - const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts -// Do not edit manually - run 'npm run generate-models' to update - -`; - const catalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; - const imageCatalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_IMAGE_MODELS`; - const classifierCatalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_CLASSIFIER_MODELS`; - const generatedShardFiles = new Set(); - for (const providerId of sortedProviderIds) { - let output = generatedHeader; - output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; - output += `import { flattenChatModelCatalog, flattenClassifierModelCatalog, flattenImageModelCatalog, type ChatModelCatalog, type ClassifierModelCatalog, type ImageModelCatalog } from "../model-catalog.ts";\n\n`; - output += `export const ${catalogConstName(providerId)}: ChatModelCatalog =\n`; - output += `\tflattenChatModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; - output += `export const ${imageCatalogConstName(providerId)}: ImageModelCatalog =\n`; - output += `\tflattenImageModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; - output += `export const ${classifierCatalogConstName(providerId)}: ClassifierModelCatalog =\n`; - output += `\tflattenClassifierModelCatalog(${JSON.stringify(providerId)}, values);\n`; - const filename = `${providerId}.models.ts`; - generatedShardFiles.add(filename); - writeFileSync(join(providersDir, filename), output); - } - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry)); - } - - let output = generatedHeader; - for (const providerId of sortedProviderIds) { - output += `import { ${classifierCatalogConstName(providerId)}, ${imageCatalogConstName(providerId)}, ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; - } - output += `\nexport const MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${catalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; - } - output += `};\n\nexport const IMAGE_MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${imageCatalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${imageCatalogConstName(providerId)},\n`; - } - output += `};\n\nexport const CLASSIFIER_MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${classifierCatalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${classifierCatalogConstName(providerId)},\n`; - } - output += `};\n`; - writeFileSync(aggregatorPath, output); - console.log("Generated provider catalogs and src/models.generated.ts"); - } - - const hadPreviousData = existsSync(dataDir); - if (hadPreviousData) renameSync(dataDir, previousDataDir); - try { - renameSync(stagedDataDir, dataDir); - validateGeneratedModelData(packageRoot); - } catch (error) { - rmSync(dataDir, { recursive: true, force: true }); - if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir); - throw error; - } - restoreGeneratedCatalog = undefined; - console.log( - generatorOptions.dataOnly - ? "Hydrated JSON model values under src/providers/data/" - : "Generated JSON model values under src/providers/data/", - ); - } catch (error) { - restoreGeneratedCatalog?.(); - throw error; - } finally { - rmSync(stagingRoot, { recursive: true, force: true }); - } - } - - if (generatorOptions.jsonOutputDir) { - // `models.json` and `providers/{id}.json` retain the legacy keyed chat catalog. - // The `.all` variants are arrays so the same upstream id can appear once per type. - const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); - rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); - mkdirSync(providerOutputDir, { recursive: true }); - writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonChatProviders); - writeJson(join(generatorOptions.jsonOutputDir, "models.all.json"), jsonAllProviders); - writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); - for (const providerId of sortedProviderIds) { - writeJson(join(providerOutputDir, `${providerId}.json`), jsonChatProviders[providerId]); - writeJson(join(providerOutputDir, `${providerId}.all.json`), jsonAllProviders[providerId]); - } - console.log(`Generated JSON model catalog under ${generatorOptions.jsonOutputDir}`); - } - - // Print statistics - const totalModels = allModels.length; - const reasoningModels = allModels.filter((model) => model.reasoning).length; - - console.log(`\nModel Statistics:`); - console.log(` Total tool-capable models: ${totalModels}`); - console.log(` Reasoning-capable models: ${reasoningModels}`); - - for (const [provider, models] of Object.entries(providers)) { - console.log( - ` ${provider}: ${Object.keys(models.chat).length} chat models, ${Object.keys(models.image).length} image models, ${Object.keys(models.classifier).length} classifier models`, - ); - } -} - -// Run the generator -generateModels().catch((error) => { - console.error(error); - process.exitCode = 1; -}); diff --git a/packages/ai/scripts/openrouter-reasoning-options.ts b/packages/ai/scripts/openrouter-reasoning-options.ts deleted file mode 100644 index 6fc4a5c6c..000000000 --- a/packages/ai/scripts/openrouter-reasoning-options.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; -import { getEffortThinkingLevelMap } from "./models-dev-reasoning-options.ts"; - -export interface OpenRouterReasoningMetadata { - mandatory?: boolean; - default_enabled?: boolean; - supported_efforts?: Array; - default_effort?: ThinkingLevel | "none"; -} - -/** Convert OpenRouter's reasoning metadata into Pi model capabilities. */ -export function getOpenRouterThinkingLevelMap( - reasoning: OpenRouterReasoningMetadata | undefined, -): ThinkingLevelMap | undefined { - if (!reasoning) return undefined; - if (!reasoning.supported_efforts?.length) return reasoning.mandatory === true ? { off: null } : undefined; - - // OpenRouter's supported_efforts uses the same effort values as models.dev reasoning_options, - // so both sources can share the same Pi thinking-level conversion. - const map = getEffortThinkingLevelMap([{ type: "effort", values: reasoning.supported_efforts }]); - if (!map) return reasoning.mandatory === true ? { off: null } : undefined; - return { ...map, off: reasoning.mandatory === true ? null : "none" }; -} diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 53764b406..4a81dc44e 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -59,6 +59,8 @@ import { import { getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, resolveJsonSchemaStrictSampling, type UnsupportedStrictSchemaKeywordCheck, } from "./constrained-sampling.ts"; @@ -674,11 +676,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( !(error instanceof Anthropic.APIError) || error.status !== 400 || error.type !== "invalid_request_error" || - ![ - "The compiled grammar is too large", - "Too many strict tools (", - "Schemas contains too many parameters with union types (", - ].some((message) => error.message.includes(message)) + !isStrictToolLimitErrorMessage(error.message) ) { throw error; } @@ -1185,10 +1183,9 @@ function buildParams( disablePreferredStrictTools, ); if ( - definitions.some((definition, index) => { - const config = tools[index].constrainedSampling; - return definition.strict === true && config && config.type === "json_schema" && config.strict === "prefer"; - }) + definitions.some( + (definition, index) => definition.strict === true && prefersJsonSchemaStrictSampling(tools[index]), + ) ) { hasPreferredStrictTools = true; } @@ -1630,17 +1627,25 @@ function convertTools( if (!tools) return []; return tools.map((tool, index) => { - const config = tool.constrainedSampling; const strict = - disablePreferredStrictTools && config && config.type === "json_schema" && config.strict === "prefer" + disablePreferredStrictTools && prefersJsonSchemaStrictSampling(tool) ? undefined : resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); - const schema = parameters as { properties?: unknown; required?: string[] }; + const schema = parameters as { + properties?: unknown; + required?: string[]; + $defs?: unknown; + definitions?: unknown; + }; + // Keep shared definitions so property $refs still resolve. Other root keywords stay out: + // Anthropic rejects root anyOf/oneOf/allOf with a 400. const legacyInputSchema = { type: "object" as const, properties: schema.properties ?? {}, required: schema.required ?? [], + ...(schema.$defs === undefined ? {} : { $defs: schema.$defs }), + ...(schema.definitions === undefined ? {} : { definitions: schema.definitions }), }; const inputSchema = strict === true diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index 7a7740f06..0ed88be24 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -13,6 +13,7 @@ import { type ContentBlockStopEvent, ConversationRole, ConverseStreamCommand, + type ConverseStreamCommandOutput, type ConverseStreamMetadataEvent, ImageFormat, type Message, @@ -64,7 +65,12 @@ import { type TranscriptContext, withoutInitialSystemMessage, } from "../utils/transcript.ts"; -import { getJsonSchemaToolParameters, resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; +import { + getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, + resolveJsonSchemaStrictSampling, +} from "./constrained-sampling.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, @@ -269,29 +275,55 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> = const inferenceMaxTokens = options.maxTokens ?? (isAnthropicClaudeModel(model) ? model.maxTokens : undefined); const initialSystemMessage = getInitialSystemMessage(normalizedContext.messages); const initialSystemPrompt = initialSystemMessage ? getSystemMessageText(initialSystemMessage) : undefined; - let commandInput = { - modelId: model.id, - messages: convertMessages(normalizedContext, model, cacheRetention, options.env), - system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), - inferenceConfig: { - ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), - ...(options.temperature !== undefined && { temperature: options.temperature }), - }, - toolConfig: convertToolConfig( + let disablePreferredStrictTools = false; + let response: ConverseStreamCommandOutput; + for (;;) { + const tools = convertToolConfig( getCurrentTools(normalizedContext.messages), options.toolChoice, supportsStrictMode, - ), - additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), - ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), - }; - const nextCommandInput = await options?.onPayload?.(commandInput, model); - if (nextCommandInput !== undefined) { - commandInput = nextCommandInput as typeof commandInput; + disablePreferredStrictTools, + ); + let commandInput = { + modelId: model.id, + messages: convertMessages(normalizedContext, model, cacheRetention, options.env), + system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), + inferenceConfig: { + ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), + ...(options.temperature !== undefined && { temperature: options.temperature }), + }, + toolConfig: tools.toolConfig, + additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), + ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), + }; + const nextCommandInput = await options?.onPayload?.(commandInput, model); + if (nextCommandInput !== undefined) { + commandInput = nextCommandInput as typeof commandInput; + } + try { + response = await client.send(new ConverseStreamCommand(commandInput), { abortSignal: options.signal }); + break; + } catch (error) { + // Bedrock checks the tool schemas before it starts the stream, so a retry never repeats streamed output. + if ( + options.signal?.aborted || + disablePreferredStrictTools || + !tools.hasPreferredStrictTools || + !(error instanceof BedrockRuntimeServiceException) || + error.name !== "ValidationException" || + !isStrictToolLimitErrorMessage(error.message) + ) { + throw error; + } + disablePreferredStrictTools = true; + appendAssistantMessageDiagnostic(output, { + type: "bedrock_strict_tools_fallback", + timestamp: Date.now(), + error: { message: error.message }, + details: { reason: "strict_tool_limits_exceeded" }, + }); + } } - const command = new ConverseStreamCommand(commandInput); - - const response = await client.send(command, { abortSignal: options.signal }); responseRequestId = normalizeDiagnosticValue(response.$metadata.requestId); if (!observedRawResponse && response.$metadata.httpStatusCode !== undefined) { const responseHeaders: Record = {}; @@ -1155,12 +1187,18 @@ function convertToolConfig( tools: Tool[] | undefined, toolChoice: BedrockOptions["toolChoice"], supportsStrictMode: boolean, -): ToolConfiguration | undefined { - if (!tools?.length) return undefined; - if (toolChoice === "none") return undefined; + disablePreferredStrictTools: boolean, +): { toolConfig: ToolConfiguration | undefined; hasPreferredStrictTools: boolean } { + if (!tools?.length || toolChoice === "none") return { toolConfig: undefined, hasPreferredStrictTools: false }; + let hasPreferredStrictTools = false; const bedrockTools: BedrockTool[] = tools.map((tool) => { - const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + const preferred = prefersJsonSchemaStrictSampling(tool); + const strict = + disablePreferredStrictTools && preferred + ? undefined + : resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + if (strict === true && preferred) hasPreferredStrictTools = true; return { toolSpec: { name: tool.name, @@ -1185,7 +1223,7 @@ function convertToolConfig( } } - return { tools: bedrockTools, toolChoice: bedrockToolChoice }; + return { toolConfig: { tools: bedrockTools, toolChoice: bedrockToolChoice }, hasPreferredStrictTools }; } function mapStopReason(reason: string | undefined): { stopReason: StopReason; errorMessage?: string } { diff --git a/packages/ai/src/api/constrained-sampling.ts b/packages/ai/src/api/constrained-sampling.ts index 62e03a105..adb729432 100644 --- a/packages/ai/src/api/constrained-sampling.ts +++ b/packages/ai/src/api/constrained-sampling.ts @@ -139,6 +139,27 @@ export function makeStrictJsonSchema( return cloned; } +/** Returns true for a tool that asks for strict JSON-schema sampling only when possible (`strict: "prefer"`). */ +export function prefersJsonSchemaStrictSampling(tool: Tool): boolean { + const config = tool.constrainedSampling; + return !!config && config.type === "json_schema" && config.strict === "prefer"; +} + +/** + * Claude rejects (on the Anthropic API and on Bedrock) a request whose strict tools exceed its limits with one of these + * messages. Observed limits: 20 strict tools, and 16 optional or union-typed parameters across all strict tools. + */ +const STRICT_TOOL_LIMIT_ERROR_MESSAGES = [ + "The compiled grammar is too large", + "Too many strict tools (", + "Schemas contains too many parameters with union types (", +]; + +/** Returns true when a provider error message says the request's strict tools exceed the provider's limits. */ +export function isStrictToolLimitErrorMessage(message: string): boolean { + return STRICT_TOOL_LIMIT_ERROR_MESSAGES.some((limitMessage) => message.includes(limitMessage)); +} + export function getJsonSchemaToolParameters(tool: Tool, strict: boolean | undefined): Tool["parameters"] { return (strict === true ? makeStrictJsonSchema(tool.parameters) : tool.parameters) as Tool["parameters"]; } diff --git a/packages/ai/src/api/pi-messages.ts b/packages/ai/src/api/pi-messages.ts index 59b54567e..3316a56ae 100644 --- a/packages/ai/src/api/pi-messages.ts +++ b/packages/ai/src/api/pi-messages.ts @@ -195,6 +195,15 @@ function createEventConverter(model: Model<"pi-messages">) { const toolArguments = new Map(); return (event: PiMessagesEvent): AssistantMessageEvent => { + if ("contentIndex" in event) { + // The index comes from the wire and keys the content array: `__proto__` would reach Array.prototype, and a huge + // index would make a sparse array of billions of holes. A start adds the next block; other events need one. + const index: unknown = event.contentIndex; + const limit = event.type.endsWith("_start") ? partial.content.length : partial.content.length - 1; + if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index > limit) { + throw new Error(`Invalid contentIndex ${JSON.stringify(index)} in pi-messages ${event.type} event`); + } + } switch (event.type) { case "done": Object.assign(partial, { diff --git a/packages/ai/src/auth/oauth/github-copilot.ts b/packages/ai/src/auth/oauth/github-copilot.ts index 5a17d07f0..5620bba9e 100644 --- a/packages/ai/src/auth/oauth/github-copilot.ts +++ b/packages/ai/src/auth/oauth/github-copilot.ts @@ -6,6 +6,7 @@ import { GITHUB_COPILOT_MODELS } from "../../providers/github-copilot.models.ts" import { sleep } from "../../utils/sleep.ts"; import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; const decode = (s: string) => atob(s); const CLIENT_ID = decode("SXYxLmI1MDdhMDhjODdlY2ZlOTg="); @@ -239,22 +240,15 @@ async function startDeviceFlow(domain: string, signal: AbortSignal): Promise | n } } -/** The verification URI is opened in the user's browser; only http(s) URLs are trusted. */ -function trustedHttpUrl(value: unknown): string | null { - if (typeof value !== "string" || !value) return null; - try { - const url = new URL(value); - if (url.protocol !== "https:" && url.protocol !== "http:") return null; - return url.href; - } catch { - return null; - } -} - async function startDeviceAuthorization(oauthHost: string, signal: AbortSignal): Promise { const response = await fetch(`${oauthHost}/api/oauth/device_authorization`, { method: "POST", @@ -86,16 +75,9 @@ async function startDeviceAuthorization(oauthHost: string, signal: AbortSignal): const json = await readJson(response); const deviceCode = json?.device_code; const userCode = json?.user_code; - const verificationUri = json?.verification_uri; - const verificationUriComplete = json?.verification_uri_complete; - if ( - typeof deviceCode !== "string" || - typeof userCode !== "string" || - typeof verificationUri !== "string" || - typeof verificationUriComplete !== "string" || - !trustedHttpUrl(verificationUriComplete) || - !trustedHttpUrl(verificationUri) - ) { + const verificationUri = trustedHttpUrl(json?.verification_uri); + const verificationUriComplete = trustedHttpUrl(json?.verification_uri_complete); + if (typeof deviceCode !== "string" || typeof userCode !== "string" || !verificationUri || !verificationUriComplete) { throw new Error(`Invalid Kimi Code device authorization response: ${JSON.stringify(json)}`); } diff --git a/packages/ai/src/auth/oauth/meta.ts b/packages/ai/src/auth/oauth/meta.ts index b9f331cc8..24a692f8f 100644 --- a/packages/ai/src/auth/oauth/meta.ts +++ b/packages/ai/src/auth/oauth/meta.ts @@ -15,6 +15,7 @@ import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; // Muse Code CLI client id. const CLIENT_ID = "1031625952748946"; @@ -54,18 +55,6 @@ function errorDetail(json: Record | null): string { return ""; } -/** The verification URI is opened in the user's browser; only http(s) URLs are trusted. */ -function trustedHttpUrl(value: unknown): string | null { - if (typeof value !== "string" || !value) return null; - try { - const url = new URL(value); - if (url.protocol !== "https:" && url.protocol !== "http:") return null; - return url.href; - } catch { - return null; - } -} - function positiveNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; } diff --git a/packages/ai/src/auth/oauth/radius.ts b/packages/ai/src/auth/oauth/radius.ts index 0b82b9d78..811410d8a 100644 --- a/packages/ai/src/auth/oauth/radius.ts +++ b/packages/ai/src/auth/oauth/radius.ts @@ -14,6 +14,7 @@ import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../typ import { startOAuthCallbackServer } from "./callback-server.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; import { generatePKCE } from "./pkce.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; const CALLBACK_HOST = "127.0.0.1"; const CALLBACK_PORT = 1456; @@ -51,10 +52,11 @@ async function loadRadiusOAuthDiscovery(gateway: string, signal: AbortSignal): P } const discovery = (await response.json()) as Partial; - if (typeof discovery.authorizationEndpoint !== "string") { + const authorizationEndpoint = trustedHttpUrl(discovery.authorizationEndpoint); + if (!authorizationEndpoint) { throw new Error(`Invalid Radius OAuth config from ${gateway}`); } - return { authorizationEndpoint: discovery.authorizationEndpoint }; + return { authorizationEndpoint }; } class OAuthResponseError extends Error { @@ -210,11 +212,15 @@ async function requestDeviceAuthorization(gateway: string, signal: AbortSignal): if (!data.device_code || !data.user_code || !data.verification_uri || !data.expires_in) { throw new Error("Radius OAuth device authorization response is missing required fields"); } + const verificationUri = trustedHttpUrl(data.verification_uri); + if (!verificationUri) { + throw new Error("Untrusted verification_uri in Radius OAuth device authorization response"); + } return { device_code: data.device_code, user_code: data.user_code, - verification_uri: data.verification_uri, + verification_uri: verificationUri, expires_in: data.expires_in, interval: data.interval, }; diff --git a/packages/ai/src/auth/oauth/trusted-url.ts b/packages/ai/src/auth/oauth/trusted-url.ts new file mode 100644 index 000000000..a0a9450b7 --- /dev/null +++ b/packages/ai/src/auth/oauth/trusted-url.ts @@ -0,0 +1,10 @@ +/** + * The `http:` or `https:` URL `value` names, normalized, or null. Login flows open server-supplied URLs with the + * platform URL handler, which would also launch a `file:` URL or a custom protocol handler. Normalizing + * percent-encodes control characters, so the URL can also be embedded in terminal hyperlinks. + */ +export function trustedHttpUrl(value: unknown): string | null { + if (typeof value !== "string" || !URL.canParse(value)) return null; + const url = new URL(value); + return url.protocol === "https:" || url.protocol === "http:" ? url.href : null; +} diff --git a/packages/ai/src/cli.ts b/packages/ai/src/cli.ts index 04fdbaa25..8ad800f9f 100644 --- a/packages/ai/src/cli.ts +++ b/packages/ai/src/cli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { randomUUID } from "node:crypto"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; import { createInterface } from "node:readline"; import type { AuthPrompt, OAuthCredential, Provider } from "./index.ts"; import { builtinProviders } from "./providers/all.ts"; @@ -26,7 +26,10 @@ function loadAuth(): Record { } function saveAuth(auth: Record): void { - writeFileSync(AUTH_FILE, JSON.stringify(auth, null, 2), "utf-8"); + // OAuth tokens: readable by the user only. `mode` applies when the file is created; chmod covers a file an older + // version created, before the new tokens are in it. + if (existsSync(AUTH_FILE)) chmodSync(AUTH_FILE, 0o600); + writeFileSync(AUTH_FILE, JSON.stringify(auth, null, 2), { encoding: "utf-8", mode: 0o600 }); } async function answerPrompt(rl: ReturnType, authPrompt: AuthPrompt): Promise { diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 54a0345f1..0c9197ae6 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,6 +1,3 @@ -export type { Static, TSchema } from "typebox"; -export { Type } from "typebox"; - // Core only, side-effect free: no generated catalogs, no provider factories, // no api-registry, no OAuth implementations, no compat. Provider factories // live under "@earendil-works/pi-ai/providers/*", API implementations under @@ -42,9 +39,9 @@ export * from "./utils/json-parse.ts"; export * from "./utils/overflow.ts"; export * from "./utils/retry.ts"; export * from "./utils/streaming-json.ts"; +export * from "./utils/string-enum.ts"; export { contentText, getSystemMessageText, renderSystemMessageUpdate } from "./utils/text.ts"; export * from "./utils/transcript.ts"; -export * from "./utils/typebox-helpers.ts"; export { uuidv7 } from "./utils/uuid.ts"; export * from "./utils/validation.ts"; export { PI_AI_VERSION } from "./utils/version.ts"; diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index 0890c4cee..93f2889fd 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import { AMAZON_BEDROCK_CLASSIFIER_MODELS, AMAZON_BEDROCK_IMAGE_MODELS, AMAZON_BEDROCK_MODELS } from "./providers/amazon-bedrock.models.ts"; diff --git a/packages/ai/src/providers/amazon-bedrock.models.ts b/packages/ai/src/providers/amazon-bedrock.models.ts index 37b348ef7..0d4030b6a 100644 --- a/packages/ai/src/providers/amazon-bedrock.models.ts +++ b/packages/ai/src/providers/amazon-bedrock.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/amazon-bedrock.json" with { type: "json" }; diff --git a/packages/ai/src/providers/ant-ling.models.ts b/packages/ai/src/providers/ant-ling.models.ts index bc42a8c62..4f66c20ba 100644 --- a/packages/ai/src/providers/ant-ling.models.ts +++ b/packages/ai/src/providers/ant-ling.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/ant-ling.json" with { type: "json" }; diff --git a/packages/ai/src/providers/anthropic.models.ts b/packages/ai/src/providers/anthropic.models.ts index c70b3160c..656c9fabb 100644 --- a/packages/ai/src/providers/anthropic.models.ts +++ b/packages/ai/src/providers/anthropic.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/anthropic.json" with { type: "json" }; diff --git a/packages/ai/src/providers/azure.models.ts b/packages/ai/src/providers/azure.models.ts index f86701758..0fb15ec40 100644 --- a/packages/ai/src/providers/azure.models.ts +++ b/packages/ai/src/providers/azure.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/azure.json" with { type: "json" }; diff --git a/packages/ai/src/providers/baseten.models.ts b/packages/ai/src/providers/baseten.models.ts index 864eb1f99..d68edd3e1 100644 --- a/packages/ai/src/providers/baseten.models.ts +++ b/packages/ai/src/providers/baseten.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/baseten.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cerebras.models.ts b/packages/ai/src/providers/cerebras.models.ts index 39b1d2900..e3454f2fa 100644 --- a/packages/ai/src/providers/cerebras.models.ts +++ b/packages/ai/src/providers/cerebras.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cerebras.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cloudflare-ai-gateway.models.ts b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts index 95e656542..ff524e062 100644 --- a/packages/ai/src/providers/cloudflare-ai-gateway.models.ts +++ b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cloudflare-ai-gateway.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cloudflare-workers-ai.models.ts b/packages/ai/src/providers/cloudflare-workers-ai.models.ts index d84e49d16..0d34f6fae 100644 --- a/packages/ai/src/providers/cloudflare-workers-ai.models.ts +++ b/packages/ai/src/providers/cloudflare-workers-ai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cloudflare-workers-ai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/compat-schema.ts b/packages/ai/src/providers/compat-schema.ts index b3ba8198e..9327da45c 100644 --- a/packages/ai/src/providers/compat-schema.ts +++ b/packages/ai/src/providers/compat-schema.ts @@ -1,17 +1,35 @@ -import { type Static, type TProperties, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + bool, + nul, + num, + opt, + type Properties, + type Static, + str, + union, +} from "@earendil-works/pi-schema"; import { ModelCostSchema } from "./model-schema.ts"; +import { openObject, stringLiteral, stringRecord } from "./schema-builders.ts"; -function sessionAffinityFormat(options?: TSchemaOptions) { - return Type.Union([Type.Literal("openai"), Type.Literal("openai-nosession"), Type.Literal("openrouter")], options); +function sessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return union([stringLiteral("openai"), stringLiteral("openai-nosession"), stringLiteral("openrouter")], options); } export const SessionAffinityFormatSchema = sessionAffinityFormat({ description: "Session-affinity header format used to route related requests consistently.", }); -function thinkingTokenBudgetField(options?: TSchemaOptions) { - return Type.Union( - [Type.Literal("thinking_token_budget"), Type.Literal("thinking_budget"), Type.Literal("thinking_budget_tokens")], +function thinkingTokenBudgetField( + options?: Annotations<"thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens">, +) { + return union( + [ + stringLiteral("thinking_token_budget"), + stringLiteral("thinking_budget"), + stringLiteral("thinking_budget_tokens"), + ], options, ); } @@ -21,84 +39,78 @@ export const ThinkingTokenBudgetFieldSchema = thinkingTokenBudgetField({ 'Top-level request field name used by OpenAI-compatible endpoints to cap reasoning tokens. "thinking_token_budget" is used by vLLM, "thinking_budget" by Qwen, DashScope, or SGLang, and "thinking_budget_tokens" by llama.cpp.', }); -export const ChatTemplateKwargValueSchema = Type.Union([ - Type.String(), - Type.Number(), - Type.Boolean(), - Type.Null(), - Type.Object({ - $var: Type.Union([ - Type.Literal("thinking.enabled"), - Type.Literal("thinking.effort"), - Type.Literal("thinking.budget"), +export const ChatTemplateKwargValueSchema = union([ + str(), + num(), + bool(), + nul(), + openObject({ + $var: union([ + stringLiteral("thinking.enabled"), + stringLiteral("thinking.effort"), + stringLiteral("thinking.budget"), ]), - omitWhenOff: Type.Optional(Type.Boolean()), + omitWhenOff: opt(bool()), }), ]); function percentileCutoffs(metric: string) { - return Type.Object({ - p50: Type.Optional(Type.Number({ description: `${metric} at the 50th percentile.` })), - p75: Type.Optional(Type.Number({ description: `${metric} at the 75th percentile.` })), - p90: Type.Optional(Type.Number({ description: `${metric} at the 90th percentile.` })), - p99: Type.Optional(Type.Number({ description: `${metric} at the 99th percentile.` })), + return openObject({ + p50: opt(num({ description: `${metric} at the 50th percentile.` })), + p75: opt(num({ description: `${metric} at the 75th percentile.` })), + p90: opt(num({ description: `${metric} at the 90th percentile.` })), + p99: opt(num({ description: `${metric} at the 99th percentile.` })), }); } -export const OpenRouterRoutingSchema = Type.Object( +export const OpenRouterRoutingSchema = openObject( { - allow_fallbacks: Type.Optional( - Type.Boolean({ description: "Whether to allow backup providers to serve requests.", default: true }), + allow_fallbacks: opt( + bool({ description: "Whether to allow backup providers to serve requests.", default: true }), ), - require_parameters: Type.Optional( - Type.Boolean({ + require_parameters: opt( + bool({ description: "Whether to filter providers to only those that support all parameters in the request.", default: false, }), ), - data_collection: Type.Optional( - Type.Union([Type.Literal("deny"), Type.Literal("allow")], { + data_collection: opt( + union([stringLiteral("deny"), stringLiteral("allow")], { description: 'Data collection setting. "allow": allow providers that may store or train on data. "deny": only use providers that do not collect user data.', default: "allow", }), ), - zdr: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." }), + zdr: opt(bool({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." })), + enforce_distillable_text: opt( + bool({ description: "Whether to restrict routing to only models that allow text distillation." }), ), - enforce_distillable_text: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only models that allow text distillation." }), - ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: "An ordered list of provider names or slugs to try in sequence, falling back to the next if unavailable.", }), ), - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: "List of provider names or slugs to exclusively allow for this request.", }), ), - ignore: Type.Optional( - Type.Array(Type.String(), { description: "List of provider names or slugs to skip for this request." }), - ), - quantizations: Type.Optional( - Type.Array(Type.String(), { + ignore: opt(arr(str(), { description: "List of provider names or slugs to skip for this request." })), + quantizations: opt( + arr(str(), { description: 'A list of quantization levels to filter providers by, for example ["fp16", "bf16", "fp8", "fp6", "int8", "int4", "fp4", "fp32"].', }), ), - sort: Type.Optional( - Type.Union( + sort: opt( + union( [ - Type.String(), - Type.Object({ - by: Type.Optional( - Type.String({ description: 'The sorting metric, such as "price", "throughput", or "latency".' }), - ), - partition: Type.Optional( - Type.Union([Type.String(), Type.Null()], { + str(), + openObject({ + by: opt(str({ description: 'The sorting metric, such as "price", "throughput", or "latency".' })), + partition: opt( + union([str(), nul()], { description: 'Partitioning strategy: "model" or "none".', default: "model", }), @@ -111,33 +123,25 @@ export const OpenRouterRoutingSchema = Type.Object( }, ), ), - max_price: Type.Optional( - Type.Object( + max_price: opt( + openObject( { - prompt: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million prompt tokens." }), - ), - completion: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million completion tokens." }), - ), - image: Type.Optional(Type.Union([Type.Number(), Type.String()], { description: "Price per image." })), - audio: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per audio unit." }), - ), - request: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per request." }), - ), + prompt: opt(union([num(), str()], { description: "Price per million prompt tokens." })), + completion: opt(union([num(), str()], { description: "Price per million completion tokens." })), + image: opt(union([num(), str()], { description: "Price per image." })), + audio: opt(union([num(), str()], { description: "Price per audio unit." })), + request: opt(union([num(), str()], { description: "Price per request." })), }, { description: "Maximum price per million tokens in USD." }, ), ), - preferred_min_throughput: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Minimum tokens per second")], { + preferred_min_throughput: opt( + union([num(), percentileCutoffs("Minimum tokens per second")], { description: "Preferred minimum throughput in tokens per second. A number applies to p50.", }), ), - preferred_max_latency: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Maximum latency in seconds")], { + preferred_max_latency: opt( + union([num(), percentileCutoffs("Maximum latency in seconds")], { description: "Preferred maximum latency in seconds. A number applies to p50.", }), ), @@ -148,16 +152,16 @@ export const OpenRouterRoutingSchema = Type.Object( }, ); -export const VercelGatewayRoutingSchema = Type.Object( +export const VercelGatewayRoutingSchema = openObject( { - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: 'List of provider slugs to exclusively use for this request, for example ["bedrock", "anthropic"].', }), ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: 'List of provider slugs to try in order, for example ["anthropic", "openai"].', }), ), @@ -168,374 +172,382 @@ export const VercelGatewayRoutingSchema = Type.Object( }, ); -export const AnthropicAllowedFallbackModelSchema = Type.Object( +export const AnthropicAllowedFallbackModelSchema = openObject( { - provider: Type.String({ minLength: 1 }), - model: Type.String({ minLength: 1 }), + provider: str({ minLength: 1 }), + model: str({ minLength: 1 }), cost: ModelCostSchema, }, { description: "An Anthropic server-side refusal fallback model with local pricing metadata." }, ); -function optionalCompatBoolean(options?: TSchemaOptions) { - return Type.Optional(Type.Boolean(options)); +function optionalCompatBoolean(options?: Annotations) { + return opt(bool(options)); } -function optionalSessionAffinityFormat(options?: TSchemaOptions) { - return Type.Optional(sessionAffinityFormat(options)); +function optionalSessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return opt(sessionAffinityFormat(options)); } -export const OpenAICompletionsCompatSchema = Type.Object( - { - supportsStore: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports the store field. Default: auto-detected from URL.", - }), - ), - supportsDeveloperRole: optionalCompatBoolean({ - description: - "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", +const OpenAICompletionsCompatProperties = { + supportsStore: opt( + bool({ + description: "Whether the provider supports the store field. Default: auto-detected from URL.", }), - supportsReasoningEffort: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", - }), - ), - supportsUsageInStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", - default: true, - }), - ), - supportsFinishReason: Type.Optional( - Type.Boolean({ - description: - "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", - default: true, - }), - ), - maxTokensField: Type.Optional( - Type.Union([Type.Literal("max_completion_tokens"), Type.Literal("max_tokens")], { - description: "Which field to use for max tokens. Default: auto-detected from URL.", - }), - ), - requiresToolResultName: Type.Optional( - Type.Boolean({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), - ), - requiresAssistantAfterToolResult: Type.Optional( - Type.Boolean({ - description: - "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", - }), - ), - requiresThinkingAsText: Type.Optional( - Type.Boolean({ - description: - "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", - }), - ), - requiresReasoningContentOnAssistantMessages: Type.Optional( - Type.Boolean({ - description: - "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", - }), - ), - thinkingFormat: Type.Optional( - Type.Union( - [ - Type.Literal("openai"), - Type.Literal("openrouter"), - Type.Literal("deepseek"), - Type.Literal("together"), - Type.Literal("baseten"), - Type.Literal("zai"), - Type.Literal("qwen"), - Type.Literal("chat-template"), - Type.Literal("qwen-chat-template"), - Type.Literal("string-thinking"), - Type.Literal("ant-ling"), - ], - { - description: - 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', - }, - ), - ), - chatTemplateKwargs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - chatTemplateArgs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - openRouterRouting: Type.Optional(OpenRouterRoutingSchema), - vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), - zaiToolStream: Type.Optional( - Type.Boolean({ - description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", - default: false, - }), - ), - thinkingTokenBudgetField: Type.Optional( - thinkingTokenBudgetField({ - description: - 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', - }), - ), - supportsThinkingTokenBudget: Type.Optional( - Type.Boolean({ - description: - 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', - default: false, - }), - ), - supportsOpenAIGrammarTools: optionalCompatBoolean({ - description: - "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", - default: false, - }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ - description: - "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - supportsMidConvoToolAdditions: Type.Optional( - Type.Boolean({ - description: - "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", - default: false, - }), - ), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", - default: false, - }), - cacheControlFormat: Type.Optional( - Type.Literal("anthropic", { - description: - "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", - }), - ), - sendSessionAffinityHeaders: optionalCompatBoolean({ - description: - "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', - }), - vllmPriority: Type.Optional( - Type.Number({ - description: - "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", - }), - ), - }, - { + ), + supportsDeveloperRole: optionalCompatBoolean({ description: - "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", - additionalProperties: true, - }, -); - -export const OpenAIResponsesCompatSchema = Type.Object( - { - supportsDeveloperRole: optionalCompatBoolean({ - description: "Whether the provider supports the developer role instead of system.", + "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", + }), + supportsReasoningEffort: opt( + bool({ + description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", + }), + ), + supportsUsageInStreaming: opt( + bool({ + description: + "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", default: true, }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsFinishReason: opt( + bool({ description: - "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", default: true, }), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + ), + maxTokensField: opt( + union([stringLiteral("max_completion_tokens"), stringLiteral("max_tokens")], { + description: "Which field to use for max tokens. Default: auto-detected from URL.", }), - supportsOpenAIGrammarTools: optionalCompatBoolean({ + ), + requiresToolResultName: opt( + bool({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), + ), + requiresAssistantAfterToolResult: opt( + bool({ description: - "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", + }), + ), + requiresThinkingAsText: opt( + bool({ + description: + "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", + }), + ), + requiresReasoningContentOnAssistantMessages: opt( + bool({ + description: + "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", + }), + ), + thinkingFormat: opt( + union( + [ + stringLiteral("openai"), + stringLiteral("openrouter"), + stringLiteral("deepseek"), + stringLiteral("together"), + stringLiteral("baseten"), + stringLiteral("zai"), + stringLiteral("qwen"), + stringLiteral("chat-template"), + stringLiteral("qwen-chat-template"), + stringLiteral("string-thinking"), + stringLiteral("ant-ling"), + ], + { + description: + 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', + }, + ), + ), + chatTemplateKwargs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + chatTemplateArgs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + openRouterRouting: opt(OpenRouterRoutingSchema), + vercelGatewayRouting: opt(VercelGatewayRoutingSchema), + zaiToolStream: opt( + bool({ + description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", default: false, }), - supportsAdditionalTools: Type.Optional( - Type.Boolean({ - description: "Whether the model supports message-anchored additional_tools input items.", - default: false, - }), - ), - supportsToolSearch: Type.Optional( - Type.Boolean({ - description: "Whether the model supports client-executed tool search for transcript-anchored additions.", - default: false, - }), - ), - supportsExplicitPromptCacheMode: Type.Optional( - Type.Boolean({ - description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", - default: false, - }), - ), - supportsMaxOutputTokens: Type.Optional( - Type.Boolean({ - description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", - default: true, - }), - ), - }, - { description: "Compatibility settings for OpenAI Responses APIs.", additionalProperties: true }, -); + ), + thinkingTokenBudgetField: opt( + thinkingTokenBudgetField({ + description: + 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', + }), + ), + supportsThinkingTokenBudget: opt( + bool({ + description: + 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', + default: false, + }), + ), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + supportsMidConvoToolAdditions: opt( + bool({ + description: + "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", + default: false, + }), + ), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", + default: false, + }), + cacheControlFormat: opt( + stringLiteral("anthropic", { + description: + "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", + }), + ), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', + }), + vllmPriority: opt( + num({ + description: + "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", + }), + ), +}; -export const AnthropicMessagesCompatSchema = Type.Object( - { - supportsEagerToolInputStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", - default: true, - }), - ), - supportsLongCacheRetention: optionalCompatBoolean({ - description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", +export const OpenAICompletionsCompatSchema = openObject(OpenAICompletionsCompatProperties, { + description: + "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", + additionalProperties: true, +}); + +const OpenAIResponsesCompatProperties = { + supportsDeveloperRole: optionalCompatBoolean({ + description: "Whether the provider supports the developer role instead of system.", + default: true, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + default: true, + }), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + }), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsAdditionalTools: opt( + bool({ + description: "Whether the model supports message-anchored additional_tools input items.", + default: false, + }), + ), + supportsToolSearch: opt( + bool({ + description: "Whether the model supports client-executed tool search for transcript-anchored additions.", + default: false, + }), + ), + supportsExplicitPromptCacheMode: opt( + bool({ + description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", + default: false, + }), + ), + supportsMidConvoEffort: opt( + bool({ + description: + "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", + default: false, + }), + ), + supportsMaxOutputTokens: opt( + bool({ + description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", default: true, }), - sendSessionAffinityHeaders: optionalCompatBoolean({ + ), +}; + +export const OpenAIResponsesCompatSchema = openObject(OpenAIResponsesCompatProperties, { + description: "Compatibility settings for OpenAI Responses APIs.", + additionalProperties: true, +}); + +const AnthropicMessagesCompatProperties = { + supportsEagerToolInputStreaming: opt( + bool({ description: - "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", + default: true, }), - sessionAffinityFormat: Type.Optional( - Type.Literal("openrouter", { - description: - "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", - }), - ), - supportsCacheControlOnTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", - default: true, - }), - ), - supportsTemperature: Type.Optional( - Type.Boolean({ - description: - "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", - default: true, - }), - ), - forceAdaptiveThinking: Type.Optional( - Type.Boolean({ - description: - "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", - default: false, - }), - ), - allowEmptySignature: Type.Optional( - Type.Boolean({ - description: "Whether to replay empty thinking signatures instead of converting thinking to text.", - default: false, - }), - ), - supportsStrictTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model transport supports effort-only system messages and thinking binding controls.", - default: false, - }), - ), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsLongCacheRetention: optionalCompatBoolean({ + description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", + default: true, + }), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: opt( + stringLiteral("openrouter", { + description: "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", + }), + ), + supportsCacheControlOnTools: opt( + bool({ description: - "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", + default: true, + }), + ), + supportsTemperature: opt( + bool({ + description: + "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", + default: true, + }), + ), + forceAdaptiveThinking: opt( + bool({ + description: + "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", default: false, }), - supportsMidConvoToolChanges: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", - default: false, - }), - ), - allowedFallbackModels: Type.Optional( - Type.Array(AnthropicAllowedFallbackModelSchema, { - maxItems: 3, - description: - "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", - }), - ), - }, - { description: "Compatibility settings for Anthropic Messages-compatible APIs.", additionalProperties: true }, -); - -export const BedrockCompatSchema = Type.Object( - { - supportsStrictMode: optionalCompatBoolean({ - description: "Whether the model supports Bedrock strict tool schemas.", + ), + allowEmptySignature: opt( + bool({ + description: "Whether to replay empty thinking signatures instead of converting thinking to text.", default: false, }), - }, - { description: "Compatibility settings for Amazon Bedrock models.", additionalProperties: true }, -); - -export const MistralConversationsCompatSchema = Type.Object( - { - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsStrictTools: opt( + bool({ description: - "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", default: false, }), - }, - { description: "Compatibility settings for the Mistral chat API.", additionalProperties: true }, -); - -export const SystemOneCompatSchema = Type.Object( - { - imageInput: Type.Optional( - Type.Union([Type.Literal("assets"), Type.Literal("images"), Type.Literal("state-image-urls")], { - description: - 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', - }), - ), - sendImageOptInHeader: optionalCompatBoolean({ + ), + supportsMidConvoEffort: opt( + bool({ description: - "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + "Whether the exact model transport supports effort-only system messages and thinking binding controls.", + default: false, }), - }, - { description: "Compatibility settings for System One classifier APIs.", additionalProperties: true }, -); + ), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + default: false, + }), + supportsMidConvoToolChanges: opt( + bool({ + description: + "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", + default: false, + }), + ), + allowedFallbackModels: opt( + arr(AnthropicAllowedFallbackModelSchema, { + maxItems: 3, + description: + "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", + }), + ), +}; + +export const AnthropicMessagesCompatSchema = openObject(AnthropicMessagesCompatProperties, { + description: "Compatibility settings for Anthropic Messages-compatible APIs.", + additionalProperties: true, +}); + +const BedrockCompatProperties = { + supportsStrictMode: optionalCompatBoolean({ + description: "Whether the model supports Bedrock strict tool schemas.", + default: false, + }), +}; + +export const BedrockCompatSchema = openObject(BedrockCompatProperties, { + description: "Compatibility settings for Amazon Bedrock models.", + additionalProperties: true, +}); + +const MistralConversationsCompatProperties = { + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + default: false, + }), +}; + +export const MistralConversationsCompatSchema = openObject(MistralConversationsCompatProperties, { + description: "Compatibility settings for the Mistral chat API.", + additionalProperties: true, +}); + +const SystemOneCompatProperties = { + imageInput: opt( + union([stringLiteral("assets"), stringLiteral("images"), stringLiteral("state-image-urls")], { + description: + 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', + }), + ), + sendImageOptInHeader: optionalCompatBoolean({ + description: + "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + }), +}; + +export const SystemOneCompatSchema = openObject(SystemOneCompatProperties, { + description: "Compatibility settings for System One classifier APIs.", + additionalProperties: true, +}); const ProviderCompatPropertyOverrides = { supportsDeveloperRole: optionalCompatBoolean({ @@ -569,7 +581,7 @@ const ProviderCompatPropertyOverrides = { sendSessionAffinityHeaders: optionalCompatBoolean({ description: "Whether to send session-affinity data from options.sessionId. Defaults are API-specific.", }), -} satisfies TProperties; +} satisfies Properties; type UnionToIntersection = (Union extends unknown ? (value: Union) => void : never) extends ( value: infer Intersection, @@ -577,7 +589,7 @@ type UnionToIntersection = (Union extends unknown ? (value: Union) => voi ? Intersection : never; -type MergedCompatProperties = Omit< +type MergedCompatProperties = Omit< UnionToIntersection, keyof Overrides > & @@ -588,11 +600,11 @@ type MergedCompatProperties(propertyGroups: PropertyGroups, overrides: Overrides): MergedCompatProperties { - const merged: TProperties = {}; +function mergeCompatProperties( + propertyGroups: PropertyGroups, + overrides: Overrides, +): MergedCompatProperties { + const merged: Record = {}; const duplicateNames = new Set(); for (const properties of propertyGroups) { @@ -630,16 +642,25 @@ export const CompatSchemasByApi = { } as const; const ProviderCompatProperties = mergeCompatProperties( - [...new Set(Object.values(CompatSchemasByApi))].map((schema) => schema.properties), + [ + OpenAICompletionsCompatProperties, + OpenAIResponsesCompatProperties, + AnthropicMessagesCompatProperties, + BedrockCompatProperties, + MistralConversationsCompatProperties, + SystemOneCompatProperties, + ], ProviderCompatPropertyOverrides, ); -export const ProviderCompatSchema = Type.Object(ProviderCompatProperties, { +export const ProviderCompatSchema = openObject(ProviderCompatProperties, { description: "Provider and model compatibility overrides.", additionalProperties: true, }); export type ChatTemplateKwargValue = Static; +/** Provider and model compatibility overrides in models.json: every API's compat fields. */ +export type ProviderCompat = Static; /** Top-level request field used to cap reasoning tokens on OpenAI-compatible servers. */ export type ThinkingTokenBudgetField = Static; export type SessionAffinityFormat = Static; diff --git a/packages/ai/src/providers/deepseek.models.ts b/packages/ai/src/providers/deepseek.models.ts index 5ece8a190..9c92fb885 100644 --- a/packages/ai/src/providers/deepseek.models.ts +++ b/packages/ai/src/providers/deepseek.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/deepseek.json" with { type: "json" }; diff --git a/packages/ai/src/providers/fireworks.models.ts b/packages/ai/src/providers/fireworks.models.ts index 44cfa0856..53144268b 100644 --- a/packages/ai/src/providers/fireworks.models.ts +++ b/packages/ai/src/providers/fireworks.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/fireworks.json" with { type: "json" }; diff --git a/packages/ai/src/providers/github-copilot.models.ts b/packages/ai/src/providers/github-copilot.models.ts index 4b853d477..a71210730 100644 --- a/packages/ai/src/providers/github-copilot.models.ts +++ b/packages/ai/src/providers/github-copilot.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/github-copilot.json" with { type: "json" }; diff --git a/packages/ai/src/providers/google-vertex.models.ts b/packages/ai/src/providers/google-vertex.models.ts index a1778eecd..b081a3c71 100644 --- a/packages/ai/src/providers/google-vertex.models.ts +++ b/packages/ai/src/providers/google-vertex.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/google-vertex.json" with { type: "json" }; diff --git a/packages/ai/src/providers/google.models.ts b/packages/ai/src/providers/google.models.ts index 726a47440..ef5bec067 100644 --- a/packages/ai/src/providers/google.models.ts +++ b/packages/ai/src/providers/google.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/google.json" with { type: "json" }; diff --git a/packages/ai/src/providers/groq.models.ts b/packages/ai/src/providers/groq.models.ts index 75a84c36f..0af13338e 100644 --- a/packages/ai/src/providers/groq.models.ts +++ b/packages/ai/src/providers/groq.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/groq.json" with { type: "json" }; diff --git a/packages/ai/src/providers/huggingface.models.ts b/packages/ai/src/providers/huggingface.models.ts index 3bc971b6a..b8028a480 100644 --- a/packages/ai/src/providers/huggingface.models.ts +++ b/packages/ai/src/providers/huggingface.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/huggingface.json" with { type: "json" }; diff --git a/packages/ai/src/providers/kimi-coding.models.ts b/packages/ai/src/providers/kimi-coding.models.ts index 5b6ea5404..af36f3b47 100644 --- a/packages/ai/src/providers/kimi-coding.models.ts +++ b/packages/ai/src/providers/kimi-coding.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/kimi-coding.json" with { type: "json" }; diff --git a/packages/ai/src/providers/meta.models.ts b/packages/ai/src/providers/meta.models.ts index 81cd83917..d51b53db5 100644 --- a/packages/ai/src/providers/meta.models.ts +++ b/packages/ai/src/providers/meta.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/meta.json" with { type: "json" }; diff --git a/packages/ai/src/providers/minimax-cn.models.ts b/packages/ai/src/providers/minimax-cn.models.ts index b6cab80c6..db78342f4 100644 --- a/packages/ai/src/providers/minimax-cn.models.ts +++ b/packages/ai/src/providers/minimax-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/minimax-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/minimax.models.ts b/packages/ai/src/providers/minimax.models.ts index 4a46241a8..0adf27526 100644 --- a/packages/ai/src/providers/minimax.models.ts +++ b/packages/ai/src/providers/minimax.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/minimax.json" with { type: "json" }; diff --git a/packages/ai/src/providers/mistral.models.ts b/packages/ai/src/providers/mistral.models.ts index 0f7bed430..b59abdde8 100644 --- a/packages/ai/src/providers/mistral.models.ts +++ b/packages/ai/src/providers/mistral.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/mistral.json" with { type: "json" }; diff --git a/packages/ai/src/providers/model-schema.ts b/packages/ai/src/providers/model-schema.ts index 453ceabc7..2eb0a5a52 100644 --- a/packages/ai/src/providers/model-schema.ts +++ b/packages/ai/src/providers/model-schema.ts @@ -1,4 +1,17 @@ -import { type Static, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + int, + nul, + num, + oneOf, + opt, + type Static, + type StaticObject, + str, + union, +} from "@earendil-works/pi-schema"; +import { openObject as object } from "./schema-builders.ts"; export const THINKING_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const; export const MODEL_THINKING_LEVELS = ["off", ...THINKING_LEVELS] as const; @@ -6,19 +19,31 @@ export const MODEL_INPUT_MODALITIES = ["text", "image"] as const; export const CACHE_RETENTIONS = ["none", "short", "long"] as const; export const MODEL_PROMPT_CACHE_RETENTIONS = ["short", "long"] as const; -export const ThinkingLevelSchema = Type.Enum(THINKING_LEVELS); -export const ModelThinkingLevelSchema = Type.Enum(MODEL_THINKING_LEVELS); -export const ModelInputModalitySchema = Type.Enum(MODEL_INPUT_MODALITIES); -export const CacheRetentionSchema = Type.Enum(CACHE_RETENTIONS); +export const ThinkingLevelSchema = oneOf(THINKING_LEVELS); +export const ModelThinkingLevelSchema = oneOf(MODEL_THINKING_LEVELS); +export const ModelInputModalitySchema = oneOf(MODEL_INPUT_MODALITIES); +export const CacheRetentionSchema = oneOf(CACHE_RETENTIONS); -const ThinkingLevelMapValueSchema = Type.Union([Type.String(), Type.Null()]); -export const ThinkingLevelMapSchema = Type.Partial(Type.Record(ModelThinkingLevelSchema, ThinkingLevelMapValueSchema), { - description: - "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", -}); +const ThinkingLevelMapValueSchema = union([str(), nul()]); +export const ThinkingLevelMapSchema = object( + { + off: opt(ThinkingLevelMapValueSchema), + minimal: opt(ThinkingLevelMapValueSchema), + low: opt(ThinkingLevelMapValueSchema), + medium: opt(ThinkingLevelMapValueSchema), + high: opt(ThinkingLevelMapValueSchema), + xhigh: opt(ThinkingLevelMapValueSchema), + max: opt(ThinkingLevelMapValueSchema), + }, + { + description: + "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", + }, +); -export const ModelPromptCacheSchema = Type.Partial( - Type.Record(Type.Enum(MODEL_PROMPT_CACHE_RETENTIONS), Type.Number({ exclusiveMinimum: 0 })), +const PromptCacheLifetimeSchema = num({ exclusiveMinimum: 0 }); +export const ModelPromptCacheSchema = object( + { short: opt(PromptCacheLifetimeSchema), long: opt(PromptCacheLifetimeSchema) }, { description: "Best-effort prompt cache lifetime in seconds for the short and long retention tiers. Pi warms only tiers with a known lifetime.", @@ -26,63 +51,54 @@ export const ModelPromptCacheSchema = Type.Partial( ); const ModelCostRatesProperties = { - input: Type.Number({ description: "Input cost in USD per million tokens." }), - output: Type.Number({ description: "Output cost in USD per million tokens." }), - cacheRead: Type.Number({ description: "Cache-read cost in USD per million tokens." }), - cacheWrite: Type.Number({ description: "Cache-write cost in USD per million tokens." }), + input: num({ description: "Input cost in USD per million tokens." }), + output: num({ description: "Output cost in USD per million tokens." }), + cacheRead: num({ description: "Cache-read cost in USD per million tokens." }), + cacheWrite: num({ description: "Cache-write cost in USD per million tokens." }), }; -export const ModelCostRatesSchema = Type.Object(ModelCostRatesProperties); -export const ModelCostTierSchema = Type.Object({ - inputTokensAbove: Type.Number({ +export const ModelCostRatesSchema = object(ModelCostRatesProperties); +export const ModelCostTierSchema = object({ + inputTokensAbove: num({ description: "Use this tier when total request input exceeds this token count.", }), ...ModelCostRatesProperties, }); -export const ModelCostSchema = Type.Object({ +export const ModelCostSchema = object({ ...ModelCostRatesProperties, - tiers: Type.Optional( - Type.Array(ModelCostTierSchema, { + tiers: opt( + arr(ModelCostTierSchema, { description: "Request-wide pricing tiers. The highest matching input threshold applies to the full request.", }), ), }); -function modelImageResizeOptions(options?: TSchemaOptions) { - return Type.Object( - { - maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), - maxHeight: Type.Optional(Type.Integer({ minimum: 1 })), - maxBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." }), - ), - jpegQuality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), - }, - options, - ); +const ModelImageResizeProperties = { + maxWidth: opt(int({ minimum: 1 })), + maxHeight: opt(int({ minimum: 1 })), + maxBytes: opt(int({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." })), + jpegQuality: opt(int({ minimum: 1, maximum: 100 })), +}; + +function modelImageResizeOptions(options?: Annotations>) { + return object(ModelImageResizeProperties, options); } export const ModelImageResizeOptionsSchema = modelImageResizeOptions(); -export const ModelImageInputLimitsSchema = Type.Object({ - resize: Type.Optional( +export const ModelImageInputLimitsSchema = object({ + resize: opt( modelImageResizeOptions({ description: "Cache-safe resize profile applied before a new image enters conversation history.", }), ), - maxPerMessage: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted in one provider message." }), - ), - maxPerRequest: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted across one provider request." }), - ), + maxPerMessage: opt(int({ minimum: 1, description: "Maximum images accepted in one provider message." })), + maxPerRequest: opt(int({ minimum: 1, description: "Maximum images accepted across one provider request." })), }); -export const ModelInputLimitsSchema = Type.Object({ - maxRequestBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum serialized provider request size in bytes." }), - ), - images: Type.Optional(ModelImageInputLimitsSchema), +export const ModelInputLimitsSchema = object({ + maxRequestBytes: opt(int({ minimum: 1, description: "Maximum serialized provider request size in bytes." })), + images: opt(ModelImageInputLimitsSchema), }); export type ThinkingLevel = Static; diff --git a/packages/ai/src/providers/moonshotai-cn.models.ts b/packages/ai/src/providers/moonshotai-cn.models.ts index e1cde46dc..b9183981a 100644 --- a/packages/ai/src/providers/moonshotai-cn.models.ts +++ b/packages/ai/src/providers/moonshotai-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/moonshotai-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/moonshotai.models.ts b/packages/ai/src/providers/moonshotai.models.ts index 19ac3d66e..41bd93e39 100644 --- a/packages/ai/src/providers/moonshotai.models.ts +++ b/packages/ai/src/providers/moonshotai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/moonshotai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/nvidia.models.ts b/packages/ai/src/providers/nvidia.models.ts index f7dfc374f..73e3a7c18 100644 --- a/packages/ai/src/providers/nvidia.models.ts +++ b/packages/ai/src/providers/nvidia.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/nvidia.json" with { type: "json" }; diff --git a/packages/ai/src/providers/openai-codex.models.ts b/packages/ai/src/providers/openai-codex.models.ts index 996f1fd2f..b5e910527 100644 --- a/packages/ai/src/providers/openai-codex.models.ts +++ b/packages/ai/src/providers/openai-codex.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openai-codex.json" with { type: "json" }; diff --git a/packages/ai/src/providers/openai.models.ts b/packages/ai/src/providers/openai.models.ts index 06e0d4123..32faca489 100644 --- a/packages/ai/src/providers/openai.models.ts +++ b/packages/ai/src/providers/openai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/opencode-go.models.ts b/packages/ai/src/providers/opencode-go.models.ts index cb1dcc343..6231ee805 100644 --- a/packages/ai/src/providers/opencode-go.models.ts +++ b/packages/ai/src/providers/opencode-go.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/opencode-go.json" with { type: "json" }; diff --git a/packages/ai/src/providers/opencode.models.ts b/packages/ai/src/providers/opencode.models.ts index 812c584e3..c671fa275 100644 --- a/packages/ai/src/providers/opencode.models.ts +++ b/packages/ai/src/providers/opencode.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/opencode.json" with { type: "json" }; diff --git a/packages/ai/src/providers/opencode.ts b/packages/ai/src/providers/opencode.ts index 7b3aae419..4e87cedb5 100644 --- a/packages/ai/src/providers/opencode.ts +++ b/packages/ai/src/providers/opencode.ts @@ -22,7 +22,7 @@ export function opencodeProvider(): Provider { "openai-completions": withOpenCodeSessionHeader(openAICompletionsApi()), "openai-responses": withOpenCodeSessionHeader(openAIResponsesApi()), }, - // OpenCode Zen serves TypeSafe's System One protocol at /zen/v1/systemone. + // OpenCode Zen serves TypeSafe's System One protocol at /inference/systemone/v1/systemone. classifiers: { "typesafe-system-one": typesafeSystemOneApi() }, }); } diff --git a/packages/ai/src/providers/openrouter.models.ts b/packages/ai/src/providers/openrouter.models.ts index 75d808b59..80bdc1c08 100644 --- a/packages/ai/src/providers/openrouter.models.ts +++ b/packages/ai/src/providers/openrouter.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openrouter.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan-cn.models.ts b/packages/ai/src/providers/qwen-token-plan-cn.models.ts index 6b609faf7..4c6907031 100644 --- a/packages/ai/src/providers/qwen-token-plan-cn.models.ts +++ b/packages/ai/src/providers/qwen-token-plan-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan-individual.models.ts b/packages/ai/src/providers/qwen-token-plan-individual.models.ts index 8d48afd89..bb2175425 100644 --- a/packages/ai/src/providers/qwen-token-plan-individual.models.ts +++ b/packages/ai/src/providers/qwen-token-plan-individual.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan-individual.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan.models.ts b/packages/ai/src/providers/qwen-token-plan.models.ts index c546c44d3..bb035d2e3 100644 --- a/packages/ai/src/providers/qwen-token-plan.models.ts +++ b/packages/ai/src/providers/qwen-token-plan.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan.json" with { type: "json" }; diff --git a/packages/ai/src/providers/radius.models.ts b/packages/ai/src/providers/radius.models.ts index c8f3b12d7..81070bd0a 100644 --- a/packages/ai/src/providers/radius.models.ts +++ b/packages/ai/src/providers/radius.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/radius.json" with { type: "json" }; diff --git a/packages/ai/src/providers/schema-builders.ts b/packages/ai/src/providers/schema-builders.ts new file mode 100644 index 000000000..2df034ca4 --- /dev/null +++ b/packages/ai/src/providers/schema-builders.ts @@ -0,0 +1,34 @@ +// Builders for pi-ai's own configuration schemas (model metadata, compat settings): pi-schema's builders, shaped like the +// TypeBox schemas they replace, so models.json and the catalog validate exactly as before. + +import { + type Annotations, + obj, + type Properties, + type Schema, + type Static, + type StaticObject, +} from "@earendil-works/pi-schema"; + +/** An object that allows other properties; `additionalProperties: true` says so explicitly. */ +export function openObject( + properties: P, + options: Annotations> & { readonly additionalProperties?: true } = {}, +): Schema> { + const { additionalProperties, ...annotations } = options; + const schema = obj(properties, { ...annotations, open: true }); + return additionalProperties ? { ...schema, additionalProperties } : schema; +} + +/** One string: `{ type: "string", const: value }`. */ +export function stringLiteral(value: V, options: Annotations = {}): Schema { + return { type: "string", const: value, ...options }; +} + +/** An object with any string keys and values of `value`: `{ type: "object", patternProperties: { "^.*$": value } }`. */ +export function stringRecord( + value: S, + options: Annotations>> = {}, +): Schema>> { + return { type: "object", patternProperties: { "^.*$": value }, ...options }; +} diff --git a/packages/ai/src/providers/together.models.ts b/packages/ai/src/providers/together.models.ts index d8d5c919b..3e1c5d2e3 100644 --- a/packages/ai/src/providers/together.models.ts +++ b/packages/ai/src/providers/together.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/together.json" with { type: "json" }; diff --git a/packages/ai/src/providers/typesafe.models.ts b/packages/ai/src/providers/typesafe.models.ts index 05429c335..2be16f483 100644 --- a/packages/ai/src/providers/typesafe.models.ts +++ b/packages/ai/src/providers/typesafe.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/typesafe.json" with { type: "json" }; diff --git a/packages/ai/src/providers/vercel-ai-gateway.models.ts b/packages/ai/src/providers/vercel-ai-gateway.models.ts index e1c25832b..d9aff7dd9 100644 --- a/packages/ai/src/providers/vercel-ai-gateway.models.ts +++ b/packages/ai/src/providers/vercel-ai-gateway.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/vercel-ai-gateway.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xai.models.ts b/packages/ai/src/providers/xai.models.ts index b1fb5d302..654b7e524 100644 --- a/packages/ai/src/providers/xai.models.ts +++ b/packages/ai/src/providers/xai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts b/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts index 72b691d02..6c514b5d1 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-ams.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts index cae4bc12a..df833e743 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts index 13db578e9..e965bd8e4 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-sgp.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi.models.ts b/packages/ai/src/providers/xiaomi.models.ts index 0ade9200d..8fd6f6949 100644 --- a/packages/ai/src/providers/xiaomi.models.ts +++ b/packages/ai/src/providers/xiaomi.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi.json" with { type: "json" }; diff --git a/packages/ai/src/providers/zai-coding-cn.models.ts b/packages/ai/src/providers/zai-coding-cn.models.ts index 7dbd6a27c..bb30aba5d 100644 --- a/packages/ai/src/providers/zai-coding-cn.models.ts +++ b/packages/ai/src/providers/zai-coding-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/zai-coding-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/zai.models.ts b/packages/ai/src/providers/zai.models.ts index 7de6890f7..8bdb1ca58 100644 --- a/packages/ai/src/providers/zai.models.ts +++ b/packages/ai/src/providers/zai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/zai.json" with { type: "json" }; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 32b5a5f31..e435126ea 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -742,8 +742,6 @@ export interface ClassifierResult { timestamp: number; // Unix timestamp in milliseconds } -import type { TSchema } from "typebox"; - /** OpenAI grammar variants for constrained sampling. */ export type GrammarFormat = "openai_lark" | "openai_regex"; @@ -766,7 +764,11 @@ export type ConstrainedSamplingConfig = variants: GrammarVariants; }; -export interface Tool { +/** + * A tool the model may call. `parameters` is the JSON Schema of its arguments: pi-schema's builders (which also type + * the arguments, see `Static` in `@earendil-works/pi-schema`), TypeBox schemas, or any JSON Schema object. + */ +export interface Tool { name: string; description: string; parameters: TParameters; diff --git a/packages/ai/src/utils/overflow.ts b/packages/ai/src/utils/overflow.ts index 03f82f61b..36ae26b49 100644 --- a/packages/ai/src/utils/overflow.ts +++ b/packages/ai/src/utils/overflow.ts @@ -12,6 +12,7 @@ import type { AssistantMessage } from "../types.ts"; * - Anthropic: "413 {\"error\":{\"type\":\"request_too_large\",\"message\":\"Request exceeds the maximum size\"}}" * - OpenAI: "Your input exceeds the context window of this model" * - OpenAI/LiteLLM: "Requested token count exceeds the model's maximum context length of 131072 tokens" + * - LiteLLM pre-call check: "litellm.ContextWindowExceededError: ... Context Window exceeded for given call." * - OpenAI-compatible: "Input length (265330) exceeds model's maximum context length (262144)." * - Google: "The input token count (1196265) exceeds the maximum number of tokens allowed (1048575)" * - xAI: "This model's maximum prompt length is 131072 but the request contains 537812 tokens" @@ -42,6 +43,7 @@ const OVERFLOW_PATTERNS = [ /input is too long for requested model/i, // Amazon Bedrock /exceeds the context window/i, // OpenAI (Completions & Responses API) /exceeds (?:the )?(?:model'?s )?maximum context length(?: of [\d,]+ tokens?|\s*\([\d,]+\))/i, // OpenAI-compatible proxies (LiteLLM) + /context window exceeded/i, // LiteLLM pre-call check (ContextWindowExceededError) /input token count.*exceeds the maximum/i, // Google (Gemini) /maximum prompt length is \d+/i, // xAI (Grok) /reduce the length of the messages/i, // Groq diff --git a/packages/ai/src/utils/retry.ts b/packages/ai/src/utils/retry.ts index a1e984032..eabf6c7de 100644 --- a/packages/ai/src/utils/retry.ts +++ b/packages/ai/src/utils/retry.ts @@ -28,6 +28,10 @@ const NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN = buildProviderErrorPattern([ // Sign in with ChatGPT on a ChatGPT Business plan: a usage limit set by the // workspace admin. Only the admin can raise it. "subscription_sharing_workplace_individual_usage_limit_exceeded", + + // Zhipu/Z.ai (open.bigmodel.cn) error code 1308: the plan's usage limit is + // reached, sent as HTTP 429. It resets after hours, not seconds (#11537). + '"code"\\s*:\\s*"?1308(?![0-9])', ]); const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([ diff --git a/packages/ai/src/utils/typebox-helpers.ts b/packages/ai/src/utils/string-enum.ts similarity index 68% rename from packages/ai/src/utils/typebox-helpers.ts rename to packages/ai/src/utils/string-enum.ts index 85a53555c..b660178f8 100644 --- a/packages/ai/src/utils/typebox-helpers.ts +++ b/packages/ai/src/utils/string-enum.ts @@ -1,24 +1,24 @@ -import { type TUnsafe, Type } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; /** * Creates a string enum schema compatible with Google's API and other providers - * that don't support anyOf/const patterns. + * that don't support anyOf/const patterns: `{ type: "string", enum: [...] }`. * * @example * const OperationSchema = StringEnum(["add", "subtract", "multiply", "divide"], { * description: "The operation to perform" * }); * - * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" + * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" (Static from @earendil-works/pi-schema) */ export function StringEnum( values: T, options?: { description?: string; default?: T[number] }, -): TUnsafe { - return Type.Unsafe({ +): Schema { + return { type: "string", - enum: values as any, + enum: values, ...(options?.description && { description: options.description }), ...(options?.default && { default: options.default }), - }); + }; } diff --git a/packages/ai/src/utils/transcript.ts b/packages/ai/src/utils/transcript.ts index 6f8a2691f..e9bec79f8 100644 --- a/packages/ai/src/utils/transcript.ts +++ b/packages/ai/src/utils/transcript.ts @@ -133,7 +133,7 @@ export function toToolDeclaration(tool: Tool): Tool { * Whether two tools declare the same interface to the model. * * Both sides go through {@link toToolDeclaration} first: its JSON round-trip drops the - * typebox symbol keys and `undefined` fields that a structural comparison would see, and + * schema metadata keys (TypeBox's non-enumerable kind marks) and `undefined` fields that a structural comparison would see, and * builds both objects with the same key order, so comparing the serialized declarations * is exact. This avoids a deep-equal dependency in a browser-safe package. */ diff --git a/packages/ai/src/utils/validation.ts b/packages/ai/src/utils/validation.ts index c9e818834..586765bf0 100644 --- a/packages/ai/src/utils/validation.ts +++ b/packages/ai/src/utils/validation.ts @@ -1,299 +1,34 @@ -import { Compile } from "typebox/compile"; -import type { TLocalizedValidationError } from "typebox/error"; -import { Value } from "typebox/value"; +import { type CoercingValidator, compile } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import type { Tool, ToolCall } from "../types.ts"; -const validatorCache = new WeakMap>(); -const TYPEBOX_KIND = Symbol.for("TypeBox.Kind"); +// One compiled validator per schema object: tool schemas are long-lived, so the WeakMap drops entries with their tools. +const validatorCache = new WeakMap(); -interface JsonSchemaObject { - type?: string | string[]; - properties?: Record; - required?: string[]; - items?: JsonSchemaObject | JsonSchemaObject[]; - additionalProperties?: boolean | JsonSchemaObject; - allOf?: JsonSchemaObject[]; - anyOf?: JsonSchemaObject[]; - oneOf?: JsonSchemaObject[]; -} - -function getSchemaTypes(schema: JsonSchemaObject): string[] { - if (typeof schema.type === "string") { - return [schema.type]; - } - if (Array.isArray(schema.type)) { - return schema.type.filter((type): type is string => typeof type === "string"); - } - return []; -} - -function matchesJsonType(value: unknown, type: string): boolean { - switch (type) { - case "number": - return typeof value === "number"; - case "integer": - return typeof value === "number" && Number.isInteger(value); - case "boolean": - return typeof value === "boolean"; - case "string": - return typeof value === "string"; - case "null": - return value === null; - case "array": - return Array.isArray(value); - case "object": - return typeof value === "object" && value !== null && !Array.isArray(value); - default: - return false; - } -} - -function getSubSchemaValidator(schema: JsonSchemaObject): ReturnType | undefined { - try { - return getValidator(schema as Tool["parameters"]); - } catch { - return undefined; - } -} - -function coercePrimitiveByType(value: unknown, type: string): unknown { - switch (type) { - case "number": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isFinite(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "integer": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isInteger(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "boolean": { - if (value === null) { - return false; - } - if (typeof value === "string") { - if (value === "true") { - return true; - } - if (value === "false") { - return false; - } - } - if (typeof value === "number") { - if (value === 1) { - return true; - } - if (value === 0) { - return false; - } - } - return value; - } - case "string": { - if (value === null) { - return ""; - } - if (typeof value === "number" || typeof value === "boolean") { - return String(value); - } - return value; - } - case "null": { - if (value === "" || value === 0 || value === false) { - return null; - } - return value; - } - default: - return value; - } -} - -function applySchemaObjectCoercion(value: Record, schema: JsonSchemaObject): void { - const properties = schema.properties; - const definedKeys = new Set(properties ? Object.keys(properties) : []); - - if (properties) { - for (const [key, propertySchema] of Object.entries(properties)) { - if (!(key in value)) { - continue; - } - value[key] = coerceWithJsonSchema(value[key], propertySchema); - } - } - - if (schema.additionalProperties && typeof schema.additionalProperties === "object") { - for (const [key, propertyValue] of Object.entries(value)) { - if (definedKeys.has(key)) { - continue; - } - value[key] = coerceWithJsonSchema(propertyValue, schema.additionalProperties); - } - } -} - -function applySchemaArrayCoercion(value: unknown[], schema: JsonSchemaObject): void { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (!itemSchema) { - continue; - } - value[index] = coerceWithJsonSchema(value[index], itemSchema); - } - return; - } - - if (schema.items && typeof schema.items === "object") { - for (let index = 0; index < value.length; index++) { - value[index] = coerceWithJsonSchema(value[index], schema.items); - } - } -} - -function coerceWithUnionSchema(value: unknown, schemas: JsonSchemaObject[]): unknown { - for (const schema of schemas) { - const validator = getSubSchemaValidator(schema); - if (validator?.Check(value)) { - return value; - } - } - - for (const schema of schemas) { - const candidate = structuredClone(value); - const coerced = coerceWithJsonSchema(candidate, schema); - const validator = getSubSchemaValidator(schema); - if (validator?.Check(coerced)) { - return coerced; - } - } - return value; -} - -function coerceWithJsonSchema(value: unknown, schema: JsonSchemaObject): unknown { - let nextValue = value; - - if (Array.isArray(schema.allOf)) { - for (const nested of schema.allOf) { - nextValue = coerceWithJsonSchema(nextValue, nested); - } - } - - if (Array.isArray(schema.anyOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.anyOf); - } - - if (Array.isArray(schema.oneOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.oneOf); - } - - const schemaTypes = getSchemaTypes(schema); - const matchesUnionMember = - schemaTypes.length > 1 && schemaTypes.some((schemaType) => matchesJsonType(nextValue, schemaType)); - if (schemaTypes.length > 0 && !matchesUnionMember) { - for (const schemaType of schemaTypes) { - const candidate = coercePrimitiveByType(nextValue, schemaType); - if (candidate !== nextValue) { - nextValue = candidate; - break; - } - } - } - - if ( - schemaTypes.includes("object") && - typeof nextValue === "object" && - nextValue !== null && - !Array.isArray(nextValue) - ) { - applySchemaObjectCoercion(nextValue as Record, schema); - } - - if (schemaTypes.includes("array") && Array.isArray(nextValue)) { - applySchemaArrayCoercion(nextValue, schema); - } - - return nextValue; -} - -function normalizeOptionalNulls(value: unknown, schema: JsonSchemaObject): void { - if (Array.isArray(value)) { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (itemSchema) normalizeOptionalNulls(value[index], itemSchema); - } - } else if (schema.items) { - for (const item of value) normalizeOptionalNulls(item, schema.items); - } - return; - } - if (typeof value !== "object" || value === null || !schema.properties) return; - - const object = value as Record; - const required = new Set(schema.required ?? []); - for (const [key, propertySchema] of Object.entries(schema.properties)) { - if (!(key in object)) continue; - if ( - object[key] === null && - !required.has(key) && - typeof (propertySchema as { $ref?: unknown }).$ref !== "string" && - getSubSchemaValidator(propertySchema)?.Check(null) === false - ) { - delete object[key]; - } else { - normalizeOptionalNulls(object[key], propertySchema); - } - } -} - -function getValidator(schema: Tool["parameters"]): ReturnType { - const key = schema as object; - const cached = validatorCache.get(key); +function getValidator(schema: Tool["parameters"]): CoercingValidator { + const cached = validatorCache.get(schema); if (cached) { return cached; } - const validator = Compile(schema); - validatorCache.set(key, validator); + // A $ref that names nothing (a dangling definition in an MCP server's schema) rejects only values at its place, + // instead of making every call of the tool fail. + const validator = compile(schema, { formats, coerce: true, unresolvedRefs: "reject" }); + validatorCache.set(schema, validator); return validator; } -function formatValidationPath(error: TLocalizedValidationError): string { - if (error.keyword === "required") { - const requiredProperties = (error.params as { requiredProperties?: string[] }).requiredProperties; - const requiredProperty = requiredProperties?.[0]; - if (requiredProperty) { - const basePath = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return basePath ? `${basePath}.${requiredProperty}` : requiredProperty; - } - } - const path = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return path || "root"; +/** A pi-schema error path (a JSON Pointer) as the dotted path the model sees: `/edits/0/oldText` -> `edits.0.oldText`. */ +function formatValidationPath(pointer: string): string { + if (pointer === "") return "root"; + return pointer + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join("."); } /** - * Finds a tool by name and validates the tool call arguments against its TypeBox schema + * Finds a tool by name and validates the tool call arguments against its JSON Schema * @param tools Array of tool definitions * @param toolCall The tool call from the LLM * @returns The validated arguments @@ -308,8 +43,10 @@ export function validateToolCall(tools: Tool[], toolCall: ToolCall): any { } /** - * Validates tool call arguments against the tool's TypeBox schema - * @param tool The tool definition with TypeBox schema + * Validates tool call arguments against the tool's JSON Schema (any JSON Schema: pi-schema's builders, TypeBox, plain + * objects). The arguments are coerced first (pi-schema's `parse()`): models often send `"42"` for a number, a single + * value for an array, or `null` for an omitted optional property. + * @param tool The tool definition with its JSON Schema * @param toolCall The tool call from the LLM * @returns The validated (and potentially coerced) arguments * @throws Error with formatted message if validation fails @@ -318,34 +55,13 @@ export function validateToolArguments(tool: Tool, toolCall: ToolCall): any { if (toolCall.argumentsError) { throw new Error(`Invalid arguments for tool "${toolCall.name}": ${toolCall.argumentsError}`); } - const args = structuredClone(toolCall.arguments); - normalizeOptionalNulls(args, tool.parameters as JsonSchemaObject); - Value.Convert(tool.parameters, args); - - const validator = getValidator(tool.parameters); - if (!Object.getOwnPropertySymbols(tool.parameters).includes(TYPEBOX_KIND)) { - const coerced = coerceWithJsonSchema(args, tool.parameters as JsonSchemaObject); - if (coerced !== args) { - if (typeof args === "object" && args !== null && typeof coerced === "object" && coerced !== null) { - for (const key of Object.keys(args)) { - delete args[key]; - } - Object.assign(args, coerced); - } else { - return validator.Check(coerced) ? coerced : args; - } - } + // A copy, so tools that modify their arguments never modify the tool call in the transcript. + const result = getValidator(tool.parameters).parse(structuredClone(toolCall.arguments)); + if (result.ok) { + return result.value; } - if (validator.Check(args)) { - return args; - } - - const errors = - validator - .Errors(args) - .map((error) => ` - ${formatValidationPath(error)}: ${error.message}`) - .join("\n") || "Unknown validation error"; + const errors = result.errors.map((error) => ` - ${formatValidationPath(error.path)}: ${error.message}`).join("\n"); const errorMessage = `Validation failed for tool "${toolCall.name}":\n${errors}\n\nReceived arguments:\n${JSON.stringify(toolCall.arguments, null, 2)}`; diff --git a/packages/ai/src/utils/version.ts b/packages/ai/src/utils/version.ts index 18fe5a582..f1e6f3211 100644 --- a/packages/ai/src/utils/version.ts +++ b/packages/ai/src/utils/version.ts @@ -1,2 +1,2 @@ // The @earendil-works/pi-ai package version. scripts/sync-versions.js keeps it in sync with package.json. -export const PI_AI_VERSION = "1.4.0"; +export const PI_AI_VERSION = "1.5.0"; diff --git a/packages/ai/test/anthropic-adaptive-thinking-models.test.ts b/packages/ai/test/anthropic-adaptive-thinking-models.test.ts deleted file mode 100644 index a3a1c4414..000000000 --- a/packages/ai/test/anthropic-adaptive-thinking-models.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getModels, getProviders } from "../src/compat.ts"; -import type { Api, Model } from "../src/types.ts"; - -const EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS = [ - "anthropic/claude-fable-5", - "anthropic/claude-haiku-5-5", - "anthropic/claude-opus-4-8", - "anthropic/claude-opus-5", - "anthropic/claude-sonnet-5", - "cloudflare-ai-gateway/claude-fable-5", - "fireworks/accounts/fireworks/models/deepseek-v4p1-flash", - "fireworks/accounts/fireworks/models/gpt-oss-120b", - "fireworks/accounts/fireworks/models/qwen3p8-max", - "minimax-cn/MiniMax-M3.1-Flash", - "minimax/MiniMax-M3.1-Flash", - "opencode/claude-opus-4-8", - "opencode/claude-opus-5", - "vercel-ai-gateway/anthropic/claude-opus-4.8", - "vercel-ai-gateway/anthropic/claude-opus-5", - "vercel-ai-gateway/anthropic/claude-sonnet-5", -]; - -function getAllModels(): Model[] { - return getProviders().flatMap((provider) => getModels(provider) as Model[]); -} - -describe("Anthropic adaptive thinking model metadata", () => { - it("marks built-in Anthropic Messages models that use adaptive thinking", () => { - const flaggedModels = getAllModels() - .filter((model): model is Model<"anthropic-messages"> => model.api === "anthropic-messages") - .filter((model) => model.compat?.forceAdaptiveThinking === true) - .map((model) => `${model.provider}/${model.id}`) - .sort(); - - expect(flaggedModels).toEqual(expect.arrayContaining([...EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS].sort())); - expect(flaggedModels).toEqual( - flaggedModels.filter( - (modelId) => - // Regression for #9323: Fireworks uses catalog effort metadata and - // verified fallbacks, not a fixed set of adaptive model names. - modelId.startsWith("fireworks/") || - // #11121: MiniMax models with catalog effort levels require adaptive thinking. - modelId.startsWith("minimax/") || - modelId.startsWith("minimax-cn/") || - /(opus[-.](4[-.][678]|5)|sonnet[-.]4[-.]6|sonnet[-.]5|haiku[-.]5[-.]5|fable[-.]5)/.test(modelId), - ), - ); - }); -}); diff --git a/packages/ai/test/anthropic-strict-tool-schema.test.ts b/packages/ai/test/anthropic-strict-tool-schema.test.ts index 587cde17f..0e7ec623d 100644 --- a/packages/ai/test/anthropic-strict-tool-schema.test.ts +++ b/packages/ai/test/anthropic-strict-tool-schema.test.ts @@ -98,6 +98,31 @@ describe("Anthropic strict tool schemas", () => { }); }); + // https://github.com/earendil-works/pi/issues/9557 + it("keeps root definitions in non-strict schemas so property refs resolve", async () => { + const settings = { type: "object", properties: { verbose: { type: "boolean" } } }; + const cases = [ + { ref: "#/$defs/Settings", definitions: { $defs: { Settings: settings } } }, + { ref: "#/definitions/Settings", definitions: { definitions: { Settings: settings } } }, + ]; + for (const { ref, definitions } of cases) { + const parameters = { + type: "object", + properties: { settings: { $ref: ref } }, + required: ["settings"], + ...definitions, + } as unknown as Tool["parameters"]; + const tool = await captureFirstTool(createTool(parameters)); + expect(tool.strict).toBeUndefined(); + expect(tool.input_schema).toEqual({ + type: "object", + properties: { settings: { $ref: ref } }, + required: ["settings"], + ...definitions, + }); + } + }); + // https://github.com/earendil-works/pi/issues/9953 it("sends prefer tools non-strict when they use keywords Anthropic strict mode rejects", async () => { const unsupportedParameters: Tool["parameters"][] = [ diff --git a/packages/ai/test/bedrock-strict-tool-fallback.test.ts b/packages/ai/test/bedrock-strict-tool-fallback.test.ts new file mode 100644 index 000000000..c3853b461 --- /dev/null +++ b/packages/ai/test/bedrock-strict-tool-fallback.test.ts @@ -0,0 +1,173 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + inputs: [] as unknown[], + respond: undefined as ((attempt: number) => Promise) | undefined, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error { + constructor(options: { name: string; message: string }) { + super(options.message); + this.name = options.name; + } + } + + class BedrockRuntimeClient { + send(command: { input: unknown }): Promise { + bedrockMock.inputs.push(structuredClone(command.input)); + return bedrockMock.respond!(bedrockMock.inputs.length); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { END_TURN: "end_turn" }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { BedrockRuntimeServiceException } from "@aws-sdk/client-bedrock-runtime"; +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import type { Model, Tool } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const model: Model<"bedrock-converse-stream"> = { + id: "us.anthropic.claude-opus-5-5", + name: "Claude Opus 5.5 (US)", + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 100, + compat: { supportsStrictMode: true }, +}; + +// Observed from live Bedrock with 20 optional parameters across strict tools. +const unionLimitMessage = + "The model returned the following errors: Schemas contains too many parameters with union types (20 parameters with type arrays or anyOf). This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 parameters with unions)."; + +type ToolSpecInput = { toolSpec: { name: string; strict?: boolean; inputSchema: { json: unknown } } }; +type CommandInput = { toolConfig: { tools: ToolSpecInput[] }; requestMetadata?: Record }; + +function tool(name: string, strict?: "prefer" | "require"): Tool { + return { + name, + description: "Look up a value", + parameters: Type.Object({ value: Type.String(), optional: Type.Optional(Type.Number()) }), + ...(strict ? { constrainedSampling: { type: "json_schema", strict } } : {}), + }; +} + +function sdkError(name: string, message: string): Error { + return new BedrockRuntimeServiceException({ name, message, $fault: "client", $metadata: { httpStatusCode: 400 } }); +} + +function successResponse(): Promise { + return Promise.resolve({ + $metadata: { httpStatusCode: 200 }, + stream: (async function* () { + yield { messageStart: { role: "assistant" } }; + yield { messageStop: { stopReason: "end_turn" } }; + })(), + }); +} + +function run(tools: Tool[], onPayload?: (payload: unknown) => unknown) { + return streamBedrock( + model, + normalizeContext({ tools, messages: [{ role: "user", content: "Hello", timestamp: 1 }] }), + { cacheRetention: "none", onPayload }, + ).result(); +} + +// Regression tests for https://github.com/earendil-works/pi/issues/11416. +describe("Bedrock preferred strict tool fallback", () => { + beforeEach(() => { + bedrockMock.inputs = []; + }); + + it("retries a strict tool limit rejection with preferred tools non-strict and keeps required tools strict", async () => { + bedrockMock.respond = (attempt) => + attempt === 1 ? Promise.reject(sdkError("ValidationException", unionLimitMessage)) : successResponse(); + const onPayload = vi.fn((payload: unknown) => ({ ...(payload as object), requestMetadata: { team: "a" } })); + + const result = await run([tool("preferred", "prefer"), tool("required", "require"), tool("ordinary")], onPayload); + + expect(result.stopReason).toBe("stop"); + expect(onPayload).toHaveBeenCalledTimes(2); + const [first, second] = bedrockMock.inputs as CommandInput[]; + expect(first.toolConfig.tools[0].toolSpec).toMatchObject({ + strict: true, + inputSchema: { + json: { + required: ["value", "optional"], + properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }, + }, + }); + expect(second.toolConfig.tools[0].toolSpec.strict).toBeUndefined(); + expect(second.toolConfig.tools[0].toolSpec.inputSchema.json).toEqual({ + type: "object", + properties: { value: { type: "string" }, optional: { type: "number" } }, + required: ["value"], + }); + expect(second.toolConfig.tools[1]).toEqual(first.toolConfig.tools[1]); + expect(second.toolConfig.tools[1].toolSpec.strict).toBe(true); + expect(second.toolConfig.tools[2]).toEqual(first.toolConfig.tools[2]); + expect(second.requestMetadata).toEqual({ team: "a" }); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + type: "bedrock_strict_tools_fallback", + error: { message: unionLimitMessage }, + details: { reason: "strict_tool_limits_exceeded" }, + }), + ]); + }); + + it("does not retry without preferred strict tools", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("required", "require"), tool("ordinary")]); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe(`Validation error: ${unionLimitMessage}`); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("does not retry other validation errors", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", "messages: field required")); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("retries only once", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(2); + }); +}); diff --git a/packages/ai/test/cache-retention.test.ts b/packages/ai/test/cache-retention.test.ts index f943f0529..89ce3d8f6 100644 --- a/packages/ai/test/cache-retention.test.ts +++ b/packages/ai/test/cache-retention.test.ts @@ -240,12 +240,17 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }); describe("OpenAI Responses Provider", () => { - it.each(["gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-luna", "gpt-6-sol"] as const)( - "does not enable cache warming from the documented TTL alone for %s", - (modelId) => { - expect(getModel("openai", modelId).promptCache).toBeUndefined(); - }, - ); + it.each([ + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-terra", + ] as const)("does not enable cache warming from the documented TTL alone for %s", (modelId) => { + expect(getModel("openai", modelId).promptCache).toBeUndefined(); + }); it.skipIf(!process.env.OPENAI_API_KEY)( "should not set prompt_cache_retention when PI_CACHE_RETENTION is not set", @@ -505,7 +510,7 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }); it.each([ - MODELS.opencode["deepseek-v4-flash"], + MODELS.opencode["deepseek-v4.1-flash"], MODELS.opencode["deepseek-v4-pro"], MODELS.opencode["kimi-k2.5"], MODELS.opencode["kimi-k2.6"], diff --git a/packages/ai/test/catalog-generation-utils.ts b/packages/ai/test/catalog-generation-utils.ts new file mode 100644 index 000000000..4e68f390d --- /dev/null +++ b/packages/ai/test/catalog-generation-utils.ts @@ -0,0 +1,52 @@ +import { writeFileSync } from "node:fs"; +import type { ModelsDevV2Catalog } from "../catalog/sources/models-dev.ts"; + +/** + * Write a `node --import` preload that serves every upstream catalog the + * generator fetches: the given models.dev v2 catalog, and minimal responses for + * the other sources. Any other request fails. + */ +export function writeUpstreamMock(preloadPath: string, modelsDev: ModelsDevV2Catalog): void { + const responses = { + "https://models.dev/experimental/v2.0/api.json": modelsDev, + "https://models.dev/models.json?type=decision": { + "typesafe/jev-latest": { id: "typesafe/jev-latest", name: "Jev", type: "decision", limit: { context: 64000 } }, + }, + "https://openrouter.ai/api/v1/models": { data: [] }, + "https://openrouter.ai/api/v1/models?output_modalities=image": { + data: [ + { + id: "test/image", + name: "Test Image", + architecture: { input_modalities: ["text"], output_modalities: ["image"] }, + pricing: {}, + }, + ], + }, + "https://openrouter.ai/api/v1/models?output_modalities=decisions": { data: [] }, + "https://ai-gateway.vercel.sh/v1/models": { data: [] }, + "https://radius.pi.dev/v1/config": { + baseUrl: "https://radius.pi.dev", + models: [ + { + id: "test", + name: "Test", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 4096, + maxTokens: 4096, + }, + ], + }, + }; + writeFileSync( + preloadPath, + `const responses = ${JSON.stringify(responses)};\n` + + `globalThis.fetch = async (input) => {\n` + + ` const url = String(input);\n` + + ` if (url in responses) return Response.json(responses[url]);\n` + + ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + + `};\n`, + ); +} diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index 4a5592501..cae1df4a5 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -130,8 +130,8 @@ describe("Models with classifier models", () => { it.each([ ["vercel-ai-gateway", "typesafe-ai/jev", "https://ai-gateway.vercel.sh/typesafe/v1/systemone"], - ["opencode", "jev-1.13", "https://opencode.ai/inference/v1/systemone"], - ["opencode", "jev-1.13-free", "https://opencode.ai/inference/v1/systemone"], + ["opencode", "jev-1.13", "https://opencode.ai/inference/systemone/v1/systemone"], + ["opencode", "jev-1.13-free", "https://opencode.ai/inference/systemone/v1/systemone"], ])("routes %s Jev (%s) to its TypeSafe-compatible endpoint", async (provider, id, url) => { const models = builtinModels(); const jev = models.getModelOfType("classifier", provider, id); diff --git a/packages/ai/test/compat-schema.test.ts b/packages/ai/test/compat-schema.test.ts index 4ef40713e..d1a6d66ca 100644 --- a/packages/ai/test/compat-schema.test.ts +++ b/packages/ai/test/compat-schema.test.ts @@ -1,5 +1,4 @@ -import type { Static } from "typebox"; -import { Compile } from "typebox/compile"; +import type { Static } from "@earendil-works/pi-schema"; import { describe, expect, expectTypeOf, it } from "vitest"; import { AnthropicMessagesCompatSchema, @@ -11,6 +10,16 @@ import { type ProviderCompat = Static; +/** The `default` annotation of each property of an object schema. */ +function defaults(schema: { readonly [keyword: string]: unknown }): Record { + const properties = schema.properties as Record; + return Object.fromEntries( + Object.entries(properties) + .filter(([, property]) => property.default !== undefined) + .map(([name, property]) => [name, property.default]), + ); +} + describe("compatibility schemas", () => { it("preserves property types in the provider superset", () => { expectTypeOf().toEqualTypeOf(); @@ -20,21 +29,21 @@ describe("compatibility schemas", () => { }); it("preserves API-specific defaults", () => { - const completionsDefaults = Compile(OpenAICompletionsCompatSchema).Default({}); + const completionsDefaults = defaults(OpenAICompletionsCompatSchema); expect(completionsDefaults).not.toHaveProperty("thinkingFormat"); expect(completionsDefaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(Compile(OpenAIResponsesCompatSchema).Default({})).toMatchObject({ supportsDeveloperRole: true }); - const anthropicDefaults = Compile(AnthropicMessagesCompatSchema).Default({}); + expect(defaults(OpenAIResponsesCompatSchema)).toMatchObject({ supportsDeveloperRole: true }); + const anthropicDefaults = defaults(AnthropicMessagesCompatSchema); expect(anthropicDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); expect(anthropicDefaults).toMatchObject({ supportsLongCacheRetention: true }); - expect(Compile(BedrockCompatSchema).Default({})).toMatchObject({ supportsStrictMode: false }); + expect(defaults(BedrockCompatSchema)).toMatchObject({ supportsStrictMode: false }); }); it("does not assign API-specific defaults to the provider superset", () => { - const defaults = Compile(ProviderCompatSchema).Default({}); - expect(defaults).not.toHaveProperty("supportsDeveloperRole"); - expect(defaults).not.toHaveProperty("sendSessionAffinityHeaders"); - expect(defaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(defaults).not.toHaveProperty("supportsStrictMode"); + const providerDefaults = defaults(ProviderCompatSchema); + expect(providerDefaults).not.toHaveProperty("supportsDeveloperRole"); + expect(providerDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); + expect(providerDefaults).not.toHaveProperty("supportsLongCacheRetention"); + expect(providerDefaults).not.toHaveProperty("supportsStrictMode"); }); }); diff --git a/packages/ai/test/faux-provider.test.ts b/packages/ai/test/faux-provider.test.ts index ad23ed938..afdf685b6 100644 --- a/packages/ai/test/faux-provider.test.ts +++ b/packages/ai/test/faux-provider.test.ts @@ -1,3 +1,4 @@ +import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import { complete, @@ -7,7 +8,6 @@ import { fauxToolCall, registerFauxProvider, stream, - Type, } from "../src/compat.ts"; import type { AssistantMessageEvent, Context } from "../src/types.ts"; diff --git a/packages/ai/test/fireworks-model-generation.test.ts b/packages/ai/test/fireworks-model-generation.test.ts index 4b497bb77..189f80ac5 100644 --- a/packages/ai/test/fireworks-model-generation.test.ts +++ b/packages/ai/test/fireworks-model-generation.test.ts @@ -1,14 +1,15 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; -import type { ModelsDevReasoningOption } from "../scripts/models-dev-reasoning-options.ts"; +import type { ModelsDevReasoningOption, ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; import { streamSimple } from "../src/api/anthropic-messages.ts"; import { getSupportedThinkingLevels, hasApi } from "../src/models.ts"; import type { Api, Model } from "../src/types.ts"; import { normalizeContext } from "../src/utils/transcript.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); const temporaryRoots: string[] = []; @@ -24,38 +25,32 @@ function generateFireworksModels( temporaryRoots.push(root); const preloadPath = join(root, "mock-catalog.mjs"); const outputPath = join(root, "catalog"); - const catalog = { - "fireworks-ai": { - models: Object.fromEntries( - Object.entries(options).map(([id, reasoning_options]) => [ - `accounts/fireworks/models/${id}`, - { id, tool_call: true, reasoning: true, reasoning_options }, - ]), - ), - }, - }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` const url = String(input);\n` + - ` if (url === "https://models.dev/api.json") return Response.json(catalog);\n` + - ` if (url === "https://models.dev/models.json?type=decision") return Response.json({ "typesafe/jev-latest": { name: "Jev", type: "decision", limit: { context: 64000, output: 0 } } });\n` + - ` if (url.startsWith("https://openrouter.ai/api/v1/models") || url === "https://ai-gateway.vercel.sh/v1/models") return Response.json({ data: [] });\n` + - ` if (url === "https://radius.pi.dev/v1/config") return Response.json({ baseUrl: "https://radius.pi.dev", models: [{ id: "test", name: "Test", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 4096, maxTokens: 4096 }] });\n` + - ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + - `};\n`, - ); + const models = Object.entries(options).map(([id, reasoningOptions]): ModelsDevV2Model => { + const effort = reasoningOptions?.find((option) => option.type === "effort"); + const budget = reasoningOptions?.find((option) => option.type === "budget_tokens"); + return { + id: `accounts/fireworks/models/${id}`, + type: "chat", + name: id, + modalities: { input: ["text"], output: ["text"] }, + capabilities: { + tools: { supported: true }, + reasoning: { + supported: true, + ...(reasoningOptions?.some((option) => option.type === "toggle") ? { toggle: true } : {}), + ...(effort ? { effort: effort.values } : {}), + ...(budget ? { budget: { min: budget.min, max: budget.max } } : {}), + }, + }, + api: {}, + }; + }); + writeUpstreamMock(preloadPath, { + "fireworks-ai": { models: Object.fromEntries(models.map((model) => [model.id, model])) }, + }); const result = spawnSync( process.execPath, - [ - "--import", - pathToFileURL(preloadPath).href, - "scripts/generate-models.ts", - "--json-only", - "--json-output", - outputPath, - ], + ["--import", pathToFileURL(preloadPath).href, "catalog/generate.ts", "--json-only", "--json-output", outputPath], { cwd: packageRoot, encoding: "utf8", timeout: 10_000 }, ); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); diff --git a/packages/ai/test/generate-models-strict.test.ts b/packages/ai/test/generate-models-strict.test.ts index 78d27f4ec..a5f3ffa8e 100644 --- a/packages/ai/test/generate-models-strict.test.ts +++ b/packages/ai/test/generate-models-strict.test.ts @@ -1,9 +1,11 @@ import { spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; +import type { ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); const temporaryRoots: string[] = []; @@ -18,7 +20,7 @@ describe("strict model generation", () => { temporaryRoots.push(fixtureRoot); const isolatedPackageRoot = join(fixtureRoot, "package"); mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { + for (const entry of ["package.json", "catalog", "scripts", "src"]) { cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); } const preloadPath = join(fixtureRoot, "mock-models-dev.mjs"); @@ -35,26 +37,22 @@ describe("strict model generation", () => { "qwen3.8-max-preview", ]; const sourceModels = Object.fromEntries( - modelIds.map((id) => [ + modelIds.map((id): [string, ModelsDevV2Model] => [ id, { id, + type: "chat", name: id, - tool_call: id !== "deepseek-v4-flash-0731", + modalities: { input: ["text"], output: ["text"] }, + capabilities: { + tools: { supported: id !== "deepseek-v4-flash-0731" }, + reasoning: { supported: false }, + }, + api: {}, }, ]), ); - const catalog = { "alibaba-token-plan": { models: sourceModels } }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` if (String(input) === "https://models.dev/api.json") {\n` + - ` return new Response(JSON.stringify(catalog), { status: 200 });\n` + - ` }\n` + - ` throw new Error(\`Unexpected fetch: \${String(input)}\`);\n` + - `};\n`, - ); + writeUpstreamMock(preloadPath, { "alibaba-token-plan": { models: sourceModels } }); const generatedPaths = [ "src/models.generated.ts", @@ -67,7 +65,7 @@ describe("strict model generation", () => { const result = spawnSync( process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], + ["--import", pathToFileURL(preloadPath).href, "catalog/generate.ts", "--strict"], { cwd: isolatedPackageRoot, encoding: "utf8", diff --git a/packages/ai/test/image-model-data.test.ts b/packages/ai/test/image-model-data.test.ts index 21545fe8f..645fdc242 100644 --- a/packages/ai/test/image-model-data.test.ts +++ b/packages/ai/test/image-model-data.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../scripts/openrouter-catalog.ts"; +import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../catalog/sources/openrouter.ts"; const imageOnly: OpenRouterModelListItem = { id: "example/image-model", diff --git a/packages/ai/test/interleaved-thinking.test.ts b/packages/ai/test/interleaved-thinking.test.ts index b4da12835..b787a2cf7 100644 --- a/packages/ai/test/interleaved-thinking.test.ts +++ b/packages/ai/test/interleaved-thinking.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { completeSimple, getModel } from "../src/compat.ts"; import { getEnvApiKey } from "../src/env-api-keys.ts"; import type { Api, Context, Model, StopReason, Tool, ToolCall, ToolResultMessage } from "../src/types.ts"; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; const calculatorSchema = Type.Object({ diff --git a/packages/ai/test/max-thinking.test.ts b/packages/ai/test/max-thinking.test.ts index 653d9a820..bc76d0d08 100644 --- a/packages/ai/test/max-thinking.test.ts +++ b/packages/ai/test/max-thinking.test.ts @@ -67,7 +67,7 @@ describe("max thinking level", () => { expect(clampThinkingLevel(model, "xhigh")).toBe("max"); }); - it.each(["gpt-5.6-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol"] as const)( + it.each(["gpt-5.6-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"] as const)( "sends max to the Codex Responses API for %s", async (modelId) => { const model = getModel("openai-codex", modelId)!; diff --git a/packages/ai/test/model-cost-tiers.test.ts b/packages/ai/test/model-cost-tiers.test.ts index ac74ea7c6..9eade534e 100644 --- a/packages/ai/test/model-cost-tiers.test.ts +++ b/packages/ai/test/model-cost-tiers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { getAiGatewayCost } from "../scripts/ai-gateway-pricing.ts"; -import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../scripts/openrouter-catalog.ts"; +import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../catalog/sources/openrouter.ts"; +import { getAiGatewayCost } from "../catalog/sources/vercel-ai-gateway.ts"; function openRouterChatCost(pricing: OpenRouterModelListItem["pricing"]) { const catalog = buildOpenRouterCatalog( diff --git a/packages/ai/test/openai-responses-compat.test.ts b/packages/ai/test/openai-responses-compat.test.ts index 3b61686e8..ec7783313 100644 --- a/packages/ai/test/openai-responses-compat.test.ts +++ b/packages/ai/test/openai-responses-compat.test.ts @@ -119,7 +119,7 @@ describe("openai-responses provider defaults", () => { ); const stream = streamOpenAIResponses( - getModel("openai", "gpt-5.4"), + getModel("openai", "gpt-5.6-luna"), normalizeContext({ messages: [ { @@ -149,7 +149,7 @@ describe("openai-responses provider defaults", () => { if (event.type === "done" || event.type === "error") break; } - // gpt-5.4 declares its tools in additional_tools items; tool_choice applies to them. + // gpt-5.6-luna declares its tools in additional_tools items; tool_choice applies to them. expect(capturedPayload).toMatchObject({ tool_choice: "required", input: [{ type: "additional_tools", tools: [expect.objectContaining({ name: "ping" })] }, expect.anything()], diff --git a/packages/ai/test/opencode-model-generation.test.ts b/packages/ai/test/opencode-model-generation.test.ts new file mode 100644 index 000000000..6bf861697 --- /dev/null +++ b/packages/ai/test/opencode-model-generation.test.ts @@ -0,0 +1,97 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; +import type { Api, Model } from "../src/types.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; + +const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +function sourceModel(id: string, api: ModelsDevV2Model["api"]): ModelsDevV2Model { + return { + id, + type: "chat", + name: id, + modalities: { input: ["text"], output: ["text"] }, + capabilities: { tools: { supported: true }, reasoning: { supported: false } }, + api, + }; +} + +function generateOpencodeModels(models: ModelsDevV2Model[], strict = false) { + const root = mkdtempSync(join(tmpdir(), "pi-opencode-generation-")); + temporaryRoots.push(root); + const preloadPath = join(root, "mock-catalog.mjs"); + const outputPath = join(root, "catalog"); + writeUpstreamMock(preloadPath, { + opencode: { models: Object.fromEntries(models.map((model) => [model.id, model])) }, + }); + const result = spawnSync( + process.execPath, + [ + "--import", + pathToFileURL(preloadPath).href, + "catalog/generate.ts", + ...(strict ? ["--strict"] : []), + "--json-only", + "--json-output", + outputPath, + ], + { cwd: packageRoot, encoding: "utf8", timeout: 10_000 }, + ); + return { + result, + models: () => + JSON.parse(readFileSync(join(outputPath, "providers/opencode.json"), "utf8")) as Record>, + }; +} + +describe("OpenCode model generation", () => { + it("uses the protocol and base URL models.dev lists for each model", () => { + const base = "https://example.test/zen"; + const { result, models } = generateOpencodeModels([ + sourceModel("chat", { "chat-completions": { base_url: `${base}/openai/v1` } }), + sourceModel("responses", { responses: { base_url: `${base}/openai/v1` } }), + sourceModel("messages", { messages: { base_url: `${base}/anthropic/v1` } }), + sourceModel("gemini", { "generate-content": { base_url: `${base}/google/v1beta` } }), + // Native protocols win over Chat Completions regardless of listing order. + sourceModel("both", { + "chat-completions": { base_url: `${base}/openai/v1` }, + messages: { base_url: `${base}/anthropic/v1` }, + }), + sourceModel("live", { "bidi-generate-content": { base_url: `${base}/google/v1beta` } }), + ]); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stderr).toContain("models.dev lists no supported API for opencode/live (bidi-generate-content)"); + + const generated = models(); + const endpoints = Object.fromEntries( + Object.values(generated).map((model) => [model.id, { api: model.api, baseUrl: model.baseUrl }]), + ); + expect(endpoints).toEqual({ + chat: { api: "openai-completions", baseUrl: `${base}/openai/v1` }, + responses: { api: "openai-responses", baseUrl: `${base}/openai/v1` }, + // The Anthropic SDK appends /v1/messages itself. + messages: { api: "anthropic-messages", baseUrl: `${base}/anthropic` }, + gemini: { api: "google-generative-ai", baseUrl: `${base}/google/v1beta` }, + both: { api: "anthropic-messages", baseUrl: `${base}/anthropic` }, + }); + }); + + it("fails strict generation for a model without a supported protocol", () => { + const { result } = generateOpencodeModels( + [sourceModel("live", { "bidi-generate-content": { base_url: "https://example.test/google/v1beta" } })], + true, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain("models.dev lists no supported API for opencode/live (bidi-generate-content)"); + }); +}); diff --git a/packages/ai/test/openrouter-cache-control-models.test.ts b/packages/ai/test/openrouter-cache-control-models.test.ts deleted file mode 100644 index f8db1794f..000000000 --- a/packages/ai/test/openrouter-cache-control-models.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getModel } from "../src/compat.ts"; - -const OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS = [ - "~anthropic/claude-fable-latest", - "~anthropic/claude-haiku-latest", - "~anthropic/claude-opus-latest", - "~anthropic/claude-sonnet-latest", -] as const; - -describe("OpenRouter Anthropic latest alias metadata", () => { - it.each(OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS)("keeps completions cache control for %s", (modelId) => { - const model = getModel("openrouter", modelId); - expect(model.api).toBe("openai-completions"); - if (model.api !== "openai-completions") throw new Error(`Unexpected API for ${modelId}`); - expect(model.compat?.cacheControlFormat).toBe("anthropic"); - }); -}); diff --git a/packages/ai/test/openrouter-reasoning-options.test.ts b/packages/ai/test/openrouter-reasoning-options.test.ts index dbeb1140e..743f58d6c 100644 --- a/packages/ai/test/openrouter-reasoning-options.test.ts +++ b/packages/ai/test/openrouter-reasoning-options.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getOpenRouterThinkingLevelMap } from "../scripts/openrouter-reasoning-options.ts"; +import { getOpenRouterThinkingLevelMap } from "../catalog/sources/openrouter.ts"; import { streamSimple } from "../src/api/openai-completions.ts"; import type { Model, ThinkingLevelMap } from "../src/types.ts"; import { normalizeContext } from "../src/utils/transcript.ts"; diff --git a/packages/ai/test/overflow.test.ts b/packages/ai/test/overflow.test.ts index e7ae90548..1cab5e92e 100644 --- a/packages/ai/test/overflow.test.ts +++ b/packages/ai/test/overflow.test.ts @@ -91,6 +91,14 @@ describe("isContextOverflow", () => { expect(isContextOverflow(message, 131072)).toBe(true); }); + it("detects LiteLLM pre-call ContextWindowExceededError", () => { + // Regression for #11492. + const message = createErrorMessage( + '400: {"message":"litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\nModel=Qwen3.8-Flash-Next, Max Input Tokens=262144, Got=265262\nmodel=Qwen3.8-Flash-Next. context_window_fallbacks=None. fallbacks=[]."}', + ); + expect(isContextOverflow(message, 262144)).toBe(true); + }); + it("detects OpenAI-compatible parenthesized maximum context length errors", () => { const message = createErrorMessage( "Error: 400 Input length (265330) exceeds model's maximum context length (262144).", diff --git a/packages/ai/test/pi-messages.test.ts b/packages/ai/test/pi-messages.test.ts index 54b3d4d9e..78d7328bf 100644 --- a/packages/ai/test/pi-messages.test.ts +++ b/packages/ai/test/pi-messages.test.ts @@ -267,6 +267,34 @@ describe("pi-messages", () => { expect(message.errorMessage).toContain("No API key provided"); }); + it("rejects content indexes that are not the next or an existing block", async () => { + const cases: unknown[][] = [ + [{ type: "text_start", contentIndex: 4294967000 }], + [{ type: "text_start", contentIndex: "__proto__" }], + [ + { type: "text_start", contentIndex: 0 }, + { + type: "toolcall_end", + contentIndex: "__proto__", + toolCall: { polluted: true }, + }, + ], + [{ type: "text_delta", contentIndex: 0, delta: "x" }], + ]; + for (const events of cases) { + const { baseUrl } = await startServer({ + events: [{ type: "start" }, ...events, { type: "done", reason: "stop", usage }], + }); + const message = await stream(createModel(baseUrl), normalizeContext(context), { apiKey: "test-key" }).result(); + server?.close(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toContain("Invalid contentIndex"); + expect(message.content.length).toBeLessThanOrEqual(1); + } + expect(([] as unknown as Record).polluted).toBeUndefined(); + }); + it("errors when the stream ends without a terminal event", async () => { const { baseUrl } = await startServer({ events: [ diff --git a/packages/ai/test/radius-oauth.test.ts b/packages/ai/test/radius-oauth.test.ts index 793647762..73b2c0c86 100644 --- a/packages/ai/test/radius-oauth.test.ts +++ b/packages/ai/test/radius-oauth.test.ts @@ -128,6 +128,39 @@ describe("Radius OAuth", () => { expect(fetchMock).toHaveBeenCalledOnce(); }); + it("rejects an authorization endpoint that is not an http(s) URL", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => jsonResponse({ authorizationEndpoint: "file:///Users/victim/Downloads/payload.app" })), + ); + const events: AuthEvent[] = []; + + const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY }); + await expect(oauth.login(interaction("browser", events))).rejects.toThrow( + `Invalid Radius OAuth config from ${GATEWAY}`, + ); + expect(events).toEqual([]); + }); + + it("rejects a device verification URI that is not an http(s) URL", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + jsonResponse({ + device_code: "device-code", + user_code: "ABCD-1234", + verification_uri: "file:///Users/victim/Downloads/payload.app", + expires_in: 600, + }), + ), + ); + const events: AuthEvent[] = []; + + const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY }); + await expect(oauth.login(interaction("device-code", events))).rejects.toThrow("Untrusted verification_uri"); + expect(events).toEqual([]); + }); + it("exchanges the browser callback code before showing the sign-in page", async () => { let tokenStatus = 400; vi.stubGlobal( diff --git a/packages/ai/test/reasoning-options.test.ts b/packages/ai/test/reasoning-options.test.ts index fafa9fa5e..135a0b9ec 100644 --- a/packages/ai/test/reasoning-options.test.ts +++ b/packages/ai/test/reasoning-options.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getEffortThinkingLevelMap } from "../scripts/models-dev-reasoning-options.ts"; +import { getEffortThinkingLevelMap } from "../catalog/reasoning.ts"; describe("getEffortThinkingLevelMap", () => { it("exposes only verified effort values and none", () => { diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.ts index 6cac98a15..3cc02268b 100644 --- a/packages/ai/test/stream.test.ts +++ b/packages/ai/test/stream.test.ts @@ -9,7 +9,7 @@ import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResult type StreamOptionsWithExtras = StreamOptions & Record; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; diff --git a/packages/ai/test/supports-xhigh.test.ts b/packages/ai/test/supports-xhigh.test.ts index d16fdb61d..91f5c72f0 100644 --- a/packages/ai/test/supports-xhigh.test.ts +++ b/packages/ai/test/supports-xhigh.test.ts @@ -93,13 +93,14 @@ describe("getSupportedThinkingLevels", () => { "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ] as const)("includes xhigh for openai-codex %s models", (modelId) => { const model = getModel("openai-codex", modelId); expect(model).toBeDefined(); expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); }); - it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna"] as const)( + it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-terra"] as const)( "includes xhigh and max for OpenAI %s models", (modelId) => { const model = getModel("openai", modelId); @@ -123,11 +124,22 @@ describe("getSupportedThinkingLevels", () => { } }); + // GPT-6.1 Terra accepts reasoning.effort "none" but, like GPT-6.1 Sol, rejects "minimal". + it("supports off but not minimal for GPT-6.1 Terra", () => { + for (const provider of ["openai", "azure", "openai-codex"] as const) { + const model = getModel(provider, "gpt-6.1-terra"); + expect(model).toBeDefined(); + expect(model!.thinkingLevelMap?.off).toBe("none"); + expect(model!.thinkingLevelMap?.minimal).toBe(provider === "openai-codex" ? "low" : null); + } + }); + it.each([ ["gpt-6-sol", { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, 272000], ["gpt-6-luna", { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, 272000], // Codex serves GPT-6.1 Sol with a 500K default context window (#10840) ["gpt-6.1-sol", { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, 500000], + ["gpt-6.1-terra", { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, 500000], ] as const)("includes official metadata for OpenAI and Codex %s", (modelId, cost, codexContextWindow) => { for (const provider of ["openai", "openai-codex"] as const) { const model = getModel(provider, modelId); diff --git a/packages/ai/test/validation-probes.test.ts b/packages/ai/test/validation-probes.test.ts new file mode 100644 index 000000000..0ddfe8d12 --- /dev/null +++ b/packages/ai/test/validation-probes.test.ts @@ -0,0 +1,779 @@ +// The reviewers' probes of the schema switch (round 1) as regression tests: TypeBox-built and plain JSON +// (MCP-shaped) tool schemas with arguments through validateToolArguments. Each case's outcome is "ok ", "REJECT " or "THROW ". Cases marked as TypeBox-only record the documented +// differences from TypeBox (see the CHANGELOG): Refine and BigInt. + +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax. +// biome-ignore-all lint/suspicious/noThenProperty: JSON Schema's `then` keyword. +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import type { Tool, ToolCall } from "../src/types.ts"; +import { validateToolArguments } from "../src/utils/validation.ts"; + +/** The outcome of one call, in the probes' format. */ +function outcome(name: string, schema: object, args: unknown): string { + const tool = { name, description: "", parameters: schema } as Tool; + const call = { type: "toolCall", id: "1", name, arguments: args } as ToolCall; + try { + return `ok ${JSON.stringify(validateToolArguments(tool, call), (_key, value) => (typeof value === "bigint" ? `${value}n` : value))}`; + } catch (error) { + const message = (error as Error).message; + return message.startsWith("Validation failed") + ? `REJECT ${message.split("\n")[1]?.trim()}` + : `THROW ${message.split("\n")[0]}`; + } +} + +type Case = { name: string; schema: object; args: unknown[] }; + +const Node = Type.Cyclic( + { Node: Type.Object({ name: Type.String(), children: Type.Array(Type.Ref("Node")) }) }, + "Node", +); + +const cases: Case[] = [ + { + name: "cyclic", + schema: Type.Object({ tree: Node }), + args: [{ tree: { name: "a", children: [{ name: "b", children: [] }] } }, { tree: { name: 1, children: [] } }], + }, + { name: "ref-with-$id", schema: Type.Object({ a: Type.Ref("A") }), args: [{ a: { x: "1" } }] }, + { + name: "function", + schema: Type.Object({ f: Type.Optional(Type.Function([], Type.Void())) }), + args: [{}, { f: "x" }], + }, + { + name: "codec", + schema: Type.Object({ + c: Type.Codec(Type.String()) + .Decode((s) => Number(s)) + .Encode((n) => String(n)), + }), + args: [{ c: "12" }, { c: 12 }], + }, + { name: "undefined-optional", schema: Type.Object({ u: Type.Optional(Type.Undefined()) }), args: [{}, { u: null }] }, + { name: "bigint", schema: Type.Object({ n: Type.BigInt() }), args: [{ n: 1 }, { n: "12" }] }, + { + name: "refine", + schema: Type.Object({ n: Type.Refine(Type.Number(), (v) => v > 0) }), + args: [{ n: 5 }, { n: -5 }], + }, + { + name: "template-literal", + schema: Type.Object({ t: Type.TemplateLiteral("id-${number}") }), + args: [{ t: "id-12" }, { t: "id-x" }, { t: 3 }], + }, + { + name: "record-string", + schema: Type.Object({ r: Type.Record(Type.String(), Type.Number()) }), + args: [{ r: { a: "1" } }, { r: { a: 1, b: "x" } }], + }, + { + name: "record-literal-keys", + schema: Type.Object({ r: Type.Record(Type.Union([Type.Literal("a"), Type.Literal("b")]), Type.Number()) }), + args: [{ r: { a: 1, b: 2 } }, { r: { a: 1 } }, { r: { a: 1, b: 2, c: 3 } }], + }, + { + name: "intersect", + schema: Type.Intersect([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Boolean() })]), + args: [ + { a: "1", b: "true" }, + { a: 1, b: true, c: 1 }, + ], + }, + { + name: "union-objects", + schema: Type.Object({ + v: Type.Union([ + Type.Object({ k: Type.Literal("x"), n: Type.Number() }), + Type.Object({ k: Type.Literal("y"), s: Type.String() }), + ]), + }), + args: [{ v: { k: "x", n: "2" } }, { v: { k: "y", s: 2 } }], + }, + { name: "minLength-astral", schema: Type.Object({ s: Type.String({ minLength: 2 }) }), args: [{ s: "😀" }] }, + { name: "maxLength-astral", schema: Type.Object({ s: Type.String({ maxLength: 1 }) }), args: [{ s: "😀" }] }, + { name: "pattern-astral", schema: Type.Object({ s: Type.String({ pattern: "^.$" }) }), args: [{ s: "😀" }] }, + { + name: "pattern-class-dash", + schema: Type.Object({ s: Type.String({ pattern: "^[\\w-]+$" }) }), + args: [{ s: "a-b" }], + }, + { + name: "pattern-unicode-prop", + schema: Type.Object({ s: Type.String({ pattern: "^\\p{L}+$" }) }), + args: [{ s: "äb" }, { s: "p{L}" }], + }, + { + name: "multipleOf-float", + schema: Type.Object({ n: Type.Number({ multipleOf: 0.01 }) }), + args: [{ n: 0.07 }, { n: 1.1 }], + }, + { + name: "uniqueItems-keyorder", + schema: Type.Object({ a: Type.Array(Type.Unknown(), { uniqueItems: true }) }), + args: [ + { + a: [ + { x: 1, y: 2 }, + { y: 2, x: 1 }, + ], + }, + { a: [1, 1.0] }, + ], + }, + { name: "const-object", schema: Type.Object({ c: Type.Literal("a") }), args: [{ c: "a" }] }, + { + name: "formats", + schema: Type.Object({ + e: Type.Optional(Type.String({ format: "email" })), + u: Type.Optional(Type.String({ format: "uri" })), + d: Type.Optional(Type.String({ format: "date-time" })), + x: Type.Optional(Type.String({ format: "made-up" })), + h: Type.Optional(Type.String({ format: "hostname" })), + r: Type.Optional(Type.String({ format: "regex" })), + i: Type.Optional(Type.String({ format: "uuid" })), + ip: Type.Optional(Type.String({ format: "ipv4" })), + }), + args: [ + { e: "a@b.co" }, + { e: "not-an-email" }, + { u: "relative/path" }, + { u: "https://x.y/z" }, + { d: "2024-01-01T00:00:00Z" }, + { d: "2024-01-01" }, + { x: "anything" }, + { h: "under_score.example" }, + { r: "[" }, + { i: "00000000-0000-0000-0000-000000000000" }, + { i: "not" }, + { ip: "999.1.1.1" }, + ], + }, + { name: "integer-big", schema: Type.Object({ n: Type.Integer() }), args: [{ n: 1e21 }, { n: 2 ** 53 + 2 }] }, + { + name: "additional-false", + schema: Type.Object({ a: Type.String() }, { additionalProperties: false }), + args: [{ a: "x", b: 1 }], + }, + { + name: "prototype-key", + schema: Type.Object({ constructor: Type.Optional(Type.String()), toString: Type.Optional(Type.Number()) }), + args: [{}, { constructor: 1 }], + }, + { + name: "proto-key-value", + schema: Type.Object({}, { additionalProperties: Type.Array(Type.String()) }), + args: [JSON.parse('{"__proto__": "x", "a": "y"}')], + }, + // Plain JSON (MCP-shaped) + { + name: "mcp-draft7-ref-definitions", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { t: { $ref: "#/definitions/T" } }, + required: ["t"], + definitions: { T: { type: "object", properties: { path: { type: "string" } }, required: ["path"] } }, + }, + args: [{ t: { path: "a" } }, { t: { path: 42 } }], + }, + { + name: "mcp-ref-unresolved", + schema: { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }, + args: [{ t: 1 }, {}], + }, + { + name: "mcp-ref-external", + schema: { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }, + args: [{ t: 1 }], + }, + { + name: "mcp-nullable-openapi", + schema: { type: "object", properties: { s: { type: "string", nullable: true } }, required: ["s"] }, + args: [{ s: null }, { s: "x" }], + }, + { + name: "mcp-draft7-tuple", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { p: { type: "array", items: [{ type: "number" }, { type: "string" }], additionalItems: false } }, + }, + args: [{ p: [1, "a"] }, { p: [1, "a", 3] }, { p: ["1", 2] }], + }, + { + name: "mcp-draft7-ref-sibling", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { s: { $ref: "#/definitions/S", maxLength: 1 } }, + definitions: { S: { type: "string" } }, + }, + args: [{ s: "abc" }], + }, + { + name: "mcp-draft4-exclusive-bool", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + type: "object", + properties: { n: { type: "number", minimum: 0, exclusiveMinimum: true } }, + }, + args: [{ n: 0 }, { n: 1 }], + }, + { + name: "mcp-id-draft4", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + id: "http://x/y", + type: "object", + properties: { a: { type: "string" } }, + }, + args: [{ a: "x" }], + }, + { + name: "mcp-bad-pattern", + schema: { type: "object", properties: { s: { type: "string", pattern: "(? `v${i}`) } }, + }, + args: [{ e: "v4999" }, { e: "v5000" }], + }, +]; + +const o = (properties: Record, extra: Record = {}) => ({ + type: "object", + properties, + ...extra, +}); +const keywordCases: [string, object, unknown[]][] = [ + [ + "oneOf-overlap", + o({ v: { oneOf: [{ type: "string" }, { type: "string", maxLength: 5 }] } }), + [{ v: "abc" }, { v: "abcdefgh" }], + ], + ["oneOf-number-integer", o({ v: { oneOf: [{ type: "number" }, { type: "integer" }] } }), [{ v: 2 }, { v: 2.5 }]], + [ + "oneOf-objects-open", + o({ + v: { + oneOf: [ + { type: "object", properties: { a: { type: "string" } } }, + { type: "object", properties: { b: { type: "string" } } }, + ], + }, + }), + [{ v: { a: "x" } }, { v: {} }], + ], + [ + "if-then-else", + o( + { kind: { type: "string" }, n: { type: "number" } }, + { if: { properties: { kind: { const: "a" } } }, then: { required: ["n"] } }, + ), + [{ kind: "a" }, { kind: "b" }], + ], + [ + "dependentRequired", + o({ a: { type: "string" }, b: { type: "string" } }, { dependentRequired: { a: ["b"] } }), + [{ a: "x" }], + ], + [ + "dependencies-d7", + o({ a: { type: "string" }, b: { type: "string" } }, { dependencies: { a: ["b"] } }), + [{ a: "x" }], + ], + ["propertyNames", o({}, { propertyNames: { pattern: "^[a-z]+$" } }), [{ abc: 1 }, { ABC: 1 }]], + ["contains", o({ l: { type: "array", contains: { type: "number" } } }), [{ l: ["a"] }, { l: ["a", 1] }]], + ["not", o({ s: { not: { type: "null" } } }), [{ s: "x" }, { s: null }]], + ["minProperties", o({ m: { type: "object", minProperties: 1 } }), [{ m: {} }]], + [ + "unevaluatedProperties", + o({ a: { type: "string" } }, { unevaluatedProperties: false }), + [{ a: "x" }, { a: "x", b: 1 }], + ], + [ + "allOf-additional-false", + { allOf: [o({ a: { type: "string" } }, { additionalProperties: false }), o({ b: { type: "string" } })] }, + [{ a: "x" }, { a: "x", b: "y" }], + ], + ["const-object", o({ c: { const: { x: 1, y: 2 } } }), [{ c: { y: 2, x: 1 } }]], + ["enum-object", o({ e: { enum: [{ x: 1, y: 2 }] } }), [{ e: { y: 2, x: 1 } }]], + ["type-array-multi", o({ v: { type: ["string", "number"] } }), [{ v: true }, { v: null }]], + ["nullable-type-array", o({ v: { type: ["string", "null"] } }, { required: ["v"] }), [{ v: null }, { v: 5 }]], + ["exclusiveMinimum-number", o({ n: { type: "number", exclusiveMinimum: 0 } }), [{ n: 0 }, { n: "0" }]], + ["maxItems", o({ l: { type: "array", maxItems: 1 } }), [{ l: [1, 2] }, { l: "x" }]], + [ + "discriminator-openapi", + o({ + p: { + oneOf: [ + o({ kind: { type: "string", enum: ["a"] }, x: { type: "number" } }, { required: ["kind"] }), + o({ kind: { type: "string", enum: ["b"] } }, { required: ["kind"] }), + ], + discriminator: { propertyName: "kind" }, + }, + }), + [{ p: { kind: "a", x: "1" } }, { p: { kind: "b" } }], + ], + [ + "ref-root-defs", + { $ref: "#/$defs/Root", $defs: { Root: o({ a: { type: "string" } }, { required: ["a"] }) } }, + [{ a: "x" }, { a: 1 }, {}], + ], + [ + "ref-recursive", + { + $defs: { N: o({ c: { type: "array", items: { $ref: "#/$defs/N" } }, v: { type: "integer" } }) }, + ...o({ root: { $ref: "#/$defs/N" } }), + }, + [{ root: { v: "2", c: [{ v: "3", c: [] }] } }], + ], + [ + "ref-percent", + { ...o({ a: { $ref: "#/$defs/a%20b" } }), $defs: { "a b": { type: "string" } } }, + [{ a: "x" }, { a: 1 }], + ], + ["ref-tilde", { ...o({ a: { $ref: "#/$defs/a~1b" } }), $defs: { "a/b": { type: "string" } } }, [{ a: "x" }]], + [ + "defs-with-$id", + { ...o({ a: { $ref: "item.json" } }), $defs: { item: { $id: "item.json", type: "string" } } }, + [{ a: "x" }, { a: 2 }], + ], + [ + "anyOf-null-default", + o({ v: { anyOf: [{ type: "string" }, { type: "null" }], default: null } }), + [{ v: null }, {}], + ], + [ + "format-date-time-lower", + o({ d: { type: "string", format: "date-time" } }), + [{ d: "2024-01-01t00:00:00z" }, { d: "2024-01-01 00:00:00Z" }], + ], + ["format-url", o({ u: { type: "string", format: "url" } }), [{ u: "not a url" }]], + ["integer-as-float-1e0", o({ n: { type: "integer" } }), [{ n: 1e3 }]], + ["required-missing-props", { type: "object", required: ["a"] }, [{}, { a: null }]], + ["string-to-number-trim", o({ n: { type: "number" } }), [{ n: " 12 " }, { n: "0x10" }, { n: "1e3" }, { n: "" }]], +]; + +const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, +}; +const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, +}; +const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, +}; +const recursionCases: [string, object, unknown][] = [ + ["tree valid", tree, { t: ["a", ["b"]] }], + ["tree number", tree, { t: 5 }], + ["tree [5]", tree, { t: [5] }], + ["json valid", json, { data: { a: [1, "x", { b: true }] } }], + ["json null", json, { data: null }], + ["json nested null", json, { data: { a: [1, null] } }], + ["filter valid", filter, { where: { and: [{ field: "a", eq: "b" }] } }], + ["filter bad leaf", filter, { where: { and: [{ field: "a", eq: 1, x: 2 }] } }], + ["filter deep bad", filter, { where: { and: [{ and: [{ and: [{ and: [{ and: [{ field: 1 }] }] }] }] }] } }], +]; + +const expected: Record = { + 'cyclic {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}': + 'ok {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}', + 'cyclic {"tree":{"name":1,"children":[]}}': 'ok {"tree":{"name":"1","children":[]}}', + 'ref-with-$id {"a":{"x":"1"}}': "REJECT - a: unresolved reference A", + "function {}": "ok {}", + 'function {"f":"x"}': "REJECT - f: expected function, got string", + 'codec {"c":"12"}': 'ok {"c":"12"}', + 'codec {"c":12}': 'ok {"c":"12"}', + "undefined-optional {}": "ok {}", + 'undefined-optional {"u":null}': "ok {}", + 'bigint {"n":1}': "REJECT - n: expected bigint, got integer", + 'bigint {"n":"12"}': "REJECT - n: expected bigint, got string", + 'refine {"n":5}': 'ok {"n":5}', + 'refine {"n":-5}': 'ok {"n":-5}', + 'template-literal {"t":"id-12"}': 'ok {"t":"id-12"}', + 'template-literal {"t":"id-x"}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'template-literal {"t":3}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'record-string {"r":{"a":"1"}}': 'ok {"r":{"a":1}}', + 'record-string {"r":{"a":1,"b":"x"}}': "REJECT - r.b: expected number, got string", + 'record-literal-keys {"r":{"a":1,"b":2}}': 'ok {"r":{"a":1,"b":2}}', + 'record-literal-keys {"r":{"a":1}}': "REJECT - r.b: missing", + 'record-literal-keys {"r":{"a":1,"b":2,"c":3}}': 'ok {"r":{"a":1,"b":2,"c":3}}', + 'intersect {"a":"1","b":"true"}': 'ok {"a":1,"b":true}', + 'intersect {"a":1,"b":true,"c":1}': 'ok {"a":1,"b":true,"c":1}', + 'union-objects {"v":{"k":"x","n":"2"}}': 'ok {"v":{"k":"x","n":2}}', + 'union-objects {"v":{"k":"y","s":2}}': 'ok {"v":{"k":"y","s":"2"}}', + 'minLength-astral {"s":"😀"}': "REJECT - s: expected at least 2 characters", + 'maxLength-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-class-dash {"s":"a-b"}': 'ok {"s":"a-b"}', + 'pattern-unicode-prop {"s":"äb"}': 'ok {"s":"äb"}', + 'pattern-unicode-prop {"s":"p{L}"}': "REJECT - s: expected to match ^\\p{L}+$", + 'multipleOf-float {"n":0.07}': 'ok {"n":0.07}', + 'multipleOf-float {"n":1.1}': 'ok {"n":1.1}', + 'uniqueItems-keyorder {"a":[{"x":1,"y":2},{"y":2,"x":1}]}': "REJECT - a: expected unique items", + 'uniqueItems-keyorder {"a":[1,1]}': "REJECT - a: expected unique items", + 'const-object {"c":"a"}': 'ok {"c":"a"}', + 'formats {"e":"a@b.co"}': 'ok {"e":"a@b.co"}', + 'formats {"e":"not-an-email"}': "REJECT - e: expected format email", + 'formats {"u":"relative/path"}': "REJECT - u: expected format uri", + 'formats {"u":"https://x.y/z"}': 'ok {"u":"https://x.y/z"}', + 'formats {"d":"2024-01-01T00:00:00Z"}': 'ok {"d":"2024-01-01T00:00:00Z"}', + 'formats {"d":"2024-01-01"}': "REJECT - d: expected format date-time", + 'formats {"x":"anything"}': 'ok {"x":"anything"}', + 'formats {"h":"under_score.example"}': "REJECT - h: expected format hostname", + 'formats {"r":"["}': "REJECT - r: expected format regex", + 'formats {"i":"00000000-0000-0000-0000-000000000000"}': 'ok {"i":"00000000-0000-0000-0000-000000000000"}', + 'formats {"i":"not"}': "REJECT - i: expected format uuid", + 'formats {"ip":"999.1.1.1"}': "REJECT - ip: expected format ipv4", + 'integer-big {"n":1e+21}': 'ok {"n":1e+21}', + 'integer-big {"n":9007199254740994}': 'ok {"n":9007199254740994}', + 'additional-false {"a":"x","b":1}': "REJECT - b: unknown field", + "prototype-key {}": "ok {}", + 'prototype-key {"constructor":1}': 'ok {"constructor":"1"}', + 'proto-key-value {"__proto__":"x","a":"y"}': 'ok {"__proto__":["x"],"a":["y"]}', + 'mcp-draft7-ref-definitions {"t":{"path":"a"}}': 'ok {"t":{"path":"a"}}', + 'mcp-draft7-ref-definitions {"t":{"path":42}}': 'ok {"t":{"path":"42"}}', + 'mcp-ref-unresolved {"t":1}': "REJECT - t: unresolved reference #/definitions/Missing", + "mcp-ref-unresolved {}": "ok {}", + 'mcp-ref-external {"t":1}': "REJECT - t: unresolved reference https://example.com/schemas/t.json", + 'mcp-nullable-openapi {"s":null}': 'ok {"s":"null"}', + 'mcp-nullable-openapi {"s":"x"}': 'ok {"s":"x"}', + 'mcp-draft7-tuple {"p":[1,"a"]}': 'ok {"p":[1,"a"]}', + 'mcp-draft7-tuple {"p":[1,"a",3]}': "REJECT - p.2: no value is allowed here", + 'mcp-draft7-tuple {"p":["1",2]}': 'ok {"p":[1,"2"]}', + 'mcp-draft7-ref-sibling {"s":"abc"}': "REJECT - s: expected at most 1 characters", + 'mcp-draft4-exclusive-bool {"n":0}': 'ok {"n":0}', + 'mcp-draft4-exclusive-bool {"n":1}': 'ok {"n":1}', + 'mcp-id-draft4 {"a":"x"}': 'ok {"a":"x"}', + 'mcp-bad-pattern {"s":"y"}': 'ok {"s":"y"}', + 'mcp-invalid-regex {"s":"y"}': "THROW Invalid regular expression: /[/: Unterminated character class", + 'mcp-type-integer-float-0 {"n":1}': 'ok {"n":1}', + 'mcp-type-integer-float-0 {"n":1.5}': 'ok {"n":1}', + 'mcp-format-unknown {"s":"xx"}': 'ok {"s":"xx"}', + 'mcp-required-null-string {"q":null}': 'ok {"q":"null"}', + 'mcp-array-json-string {"p":"[\\"a\\",\\"b\\"]"}': 'ok {"p":["[\\"a\\",\\"b\\"]"]}', + 'mcp-integer-float {"n":3.7}': 'ok {"n":3}', + 'mcp-integer-float {"n":"3.7"}': 'ok {"n":3}', + 'mcp-empty-schema {"a":1}': 'ok {"a":1}', + 'mcp-type-object-no-props {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"b":1}': "REJECT - b: no value is allowed here", + 'mcp-huge-enum {"e":"v4999"}': 'ok {"e":"v4999"}', + 'mcp-huge-enum {"e":"v5000"}': + 'REJECT - e: expected one of ["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19", ... and 4980 more]', + 'oneOf-overlap {"v":"abc"}': "REJECT - v: matches more than one alternative", + 'oneOf-overlap {"v":"abcdefgh"}': 'ok {"v":"abcdefgh"}', + 'oneOf-number-integer {"v":2}': "REJECT - v: matches more than one alternative", + 'oneOf-number-integer {"v":2.5}': 'ok {"v":2.5}', + 'oneOf-objects-open {"v":{"a":"x"}}': "REJECT - v: matches more than one alternative", + 'oneOf-objects-open {"v":{}}': "REJECT - v: matches more than one alternative", + 'if-then-else {"kind":"a"}': "REJECT - n: missing", + 'if-then-else {"kind":"b"}': 'ok {"kind":"b"}', + 'dependentRequired {"a":"x"}': "REJECT - b: required when a is present", + 'dependencies-d7 {"a":"x"}': "REJECT - b: required when a is present", + 'propertyNames {"abc":1}': 'ok {"abc":1}', + 'propertyNames {"ABC":1}': "REJECT - ABC: invalid property name", + 'contains {"l":["a"]}': "REJECT - l: expected at least 1 matching item", + 'contains {"l":["a",1]}': 'ok {"l":["a",1]}', + 'not {"s":"x"}': 'ok {"s":"x"}', + 'not {"s":null}': "ok {}", + 'minProperties {"m":{}}': "REJECT - m: expected at least 1 properties", + 'unevaluatedProperties {"a":"x"}': 'ok {"a":"x"}', + 'unevaluatedProperties {"a":"x","b":1}': "REJECT - b: unknown field", + 'allOf-additional-false {"a":"x"}': 'ok {"a":"x"}', + 'allOf-additional-false {"a":"x","b":"y"}': "REJECT - b: unknown field", + 'const-object {"c":{"y":2,"x":1}}': 'ok {"c":{"y":2,"x":1}}', + 'enum-object {"e":{"y":2,"x":1}}': 'ok {"e":{"y":2,"x":1}}', + 'type-array-multi {"v":true}': 'ok {"v":"true"}', + 'type-array-multi {"v":null}': "ok {}", + 'nullable-type-array {"v":null}': 'ok {"v":null}', + 'nullable-type-array {"v":5}': 'ok {"v":"5"}', + 'exclusiveMinimum-number {"n":0}': "REJECT - n: expected > 0", + 'exclusiveMinimum-number {"n":"0"}': "REJECT - n: expected > 0", + 'maxItems {"l":[1,2]}': "REJECT - l: expected at most 1 items", + 'maxItems {"l":"x"}': 'ok {"l":["x"]}', + 'discriminator-openapi {"p":{"kind":"a","x":"1"}}': 'ok {"p":{"kind":"a","x":1}}', + 'discriminator-openapi {"p":{"kind":"b"}}': 'ok {"p":{"kind":"b"}}', + 'ref-root-defs {"a":"x"}': 'ok {"a":"x"}', + 'ref-root-defs {"a":1}': 'ok {"a":"1"}', + "ref-root-defs {}": "REJECT - a: missing", + 'ref-recursive {"root":{"v":"2","c":[{"v":"3","c":[]}]}}': 'ok {"root":{"v":2,"c":[{"v":3,"c":[]}]}}', + 'ref-percent {"a":"x"}': 'ok {"a":"x"}', + 'ref-percent {"a":1}': 'ok {"a":"1"}', + 'ref-tilde {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":2}': 'ok {"a":"2"}', + 'anyOf-null-default {"v":null}': 'ok {"v":null}', + "anyOf-null-default {}": "ok {}", + 'format-date-time-lower {"d":"2024-01-01t00:00:00z"}': 'ok {"d":"2024-01-01t00:00:00z"}', + 'format-date-time-lower {"d":"2024-01-01 00:00:00Z"}': "REJECT - d: expected format date-time", + 'format-url {"u":"not a url"}': "REJECT - u: expected format url", + 'integer-as-float-1e0 {"n":1000}': 'ok {"n":1000}', + "required-missing-props {}": "REJECT - a: missing", + 'required-missing-props {"a":null}': 'ok {"a":null}', + 'string-to-number-trim {"n":" 12 "}': 'ok {"n":12}', + 'string-to-number-trim {"n":"0x10"}': 'ok {"n":16}', + 'string-to-number-trim {"n":"1e3"}': 'ok {"n":1000}', + 'string-to-number-trim {"n":""}': 'ok {"n":0}', + "tree valid": 'ok {"t":["a",["b"]]}', + "tree number": 'ok {"t":"5"}', + "tree [5]": 'ok {"t":["5"]}', + "json valid": 'ok {"data":{"a":[1,"x",{"b":true}]}}', + "json null": 'ok {"data":"null"}', + "json nested null": 'ok {"data":{"a":[1,"null"]}}', + "filter valid": 'ok {"where":{"and":[{"field":"a","eq":"b"}]}}', + "filter bad leaf": "REJECT - where.and.0.eq: expected string, got integer", + "filter deep bad": "REJECT - where.and.0.and.0.and.0.and.0.and.0.eq: missing", +}; + +/** Every case with the key of its expected outcome: the name and the arguments, or the name alone for recursion. */ +const all: Array<[string, string, object, unknown]> = [ + ...cases.flatMap(({ name, schema, args }) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...keywordCases.flatMap(([name, schema, args]) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...recursionCases.map(([name, schema, arg]): [string, string, object, unknown] => [name, name, schema, arg]), +]; + +describe("reviewer probes", () => { + for (const [key, name, schema, arg] of all) { + it(key, () => { + expect(outcome(name, schema, arg)).toBe(expected[key]); + }); + } +}); + +describe("reviewer probes (Sol)", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const solCases: Array<[string, object, unknown, string]> = [ + // TypeBox-only: Type.BigInt and Type.Undefined have no JSON form; TypeBox converted to them. + [ + "BigInt from a JSON string", + Type.Object({ n: Type.BigInt() }), + { n: "42" }, + "REJECT - n: expected bigint, got string", + ], + [ + "Undefined from JSON null", + Type.Object({ n: Type.Undefined() }), + { n: null }, + "REJECT - n: expected undefined, got null", + ], + // TypeBox-only: its additionalProperties also converted declared properties; pi-schema converts only undeclared ones. + [ + "additionalProperties on declared properties", + Type.Object({ amount: Type.Number(), label: Type.String() }, { additionalProperties: Type.Integer() }), + { amount: 3.5, label: "7" }, + 'ok {"amount":3.5,"label":"7"}', + ], + // TypeBox-only: property names are names, not regular expressions. + [ + "property names are not patterns", + Type.Object({ "a.b": Type.Integer(), axb: Type.String() }), + { "a.b": "4", axb: "5.8" }, + 'ok {"a.b":4,"axb":"5.8"}', + ], + // TypeBox-only: Refine runs only in TypeBox's own validator. + [ + "Refine is not run", + Type.Object({ mode: Type.Refine(Type.String(), (value) => value === "safe") }), + { mode: "unsafe" }, + 'ok {"mode":"unsafe"}', + ], + [ + "scalar-or-array that refers to itself", + { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }, + { value: "42" }, + 'ok {"value":42}', + ], + [ + "StringEnum with an empty member from null", + Type.Object({ mode: Type.Unsafe({ type: "string", enum: ["", "x"] }) }), + { mode: null }, + 'ok {"mode":""}', + ], + [ + "Cyclic with named local definitions", + Type.Object({ + node: Type.Cyclic( + { Node: Type.Object({ n: Type.Number(), next: Type.Optional(Type.Ref("Node")) }) }, + "Node", + ), + }), + { node: { n: 1 } }, + 'ok {"node":{"n":1}}', + ], + [ + "unpaired high surrogate, minLength 2", + Type.Object({ s: Type.String({ minLength: 2 }) }), + { s: "\ud800A" }, + 'ok {"s":"\\ud800A"}', + ], + [ + "unpaired high surrogate, maxLength 1", + Type.Object({ s: Type.String({ maxLength: 1 }) }), + { s: "\ud800A" }, + "REJECT - s: expected at most 1 characters", + ], + ["integer from a string", Type.Object({ n: Type.Integer() }), { n: "42" }, 'ok {"n":42}'], + ["recursive union with a value no branch converts", tree, { t: {} }, "REJECT - t: expected string, got object"], + ]; + for (const [name, schema, arg, result] of solCases) { + it(name, () => { + expect(outcome(name, schema, arg)).toBe(result); + }); + } + + it("accepts the last member of a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + const schema = { type: "object", properties: { choice: { type: "string", enum: values } }, required: ["choice"] }; + expect(outcome("enum", schema, { choice: "member9999" })).toBe('ok {"choice":"member9999"}'); + expect(outcome("enum", schema, { choice: "x" })).toBe( + `REJECT - choice: expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 9980 more]`, + ); + }); + + it("keeps the validator of a schema object, as before, also when the schema is changed later", () => { + const parameters = Type.Object({ n: Type.Number({ minimum: 0 }) }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + Object.assign(parameters.properties.n, { minimum: 10 }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + }); +}); + +describe("reviewer probes (round 2)", () => { + const item = { type: "object", properties: { n: { type: "integer", minimum: 1 } } }; + const nested: Case[] = [ + { name: "declared", schema: { type: "object", properties: { entry: item } }, args: [{ entry: { n: null } }] }, + { name: "additional", schema: { type: "object", additionalProperties: item }, args: [{ entry: { n: null } }] }, + { + name: "pattern", + schema: { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + args: [{ entry: { n: null } }], + }, + ]; + for (const { name, schema, args } of nested) { + it(`drops an optional null inside ${name} property values`, () => { + expect(outcome(name, schema, args[0])).toBe('ok {"entry":{}}'); + }); + } + + for (const keyword of ["anyOf", "oneOf"]) { + it(`converts a ${keyword} branch together with the node's sibling properties`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + expect(outcome(keyword, schema, { a: "3", c: "2" })).toBe('ok {"a":[3],"c":2}'); + }); + } + + it("rejects a deep value every branch of a recursive object union rejects, without exponential work", () => { + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const schema = { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }; + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + expect(outcome("deep", schema, { s: value })).toBe("REJECT - s.a: missing"); + }); +}); diff --git a/packages/ai/test/validation.test.ts b/packages/ai/test/validation.test.ts index 7cebbe541..474d0b6cf 100644 --- a/packages/ai/test/validation.test.ts +++ b/packages/ai/test/validation.test.ts @@ -1,5 +1,4 @@ import { Type } from "typebox"; -import { Compile } from "typebox/compile"; import { describe, expect, it } from "vitest"; import type { JsonValue, Tool, ToolCall } from "../src/types.ts"; import { validateToolArguments } from "../src/utils/validation.ts"; @@ -61,7 +60,7 @@ describe("validateToolArguments", () => { } }); - it("coerces serialized plain JSON schemas with AJV-compatible primitive rules", () => { + it("coerces serialized plain JSON schemas like TypeBox schemas", () => { const passingCases: Array<{ schema: Tool["parameters"]; input: unknown; @@ -75,7 +74,8 @@ describe("validateToolArguments", () => { { schema: { type: "boolean" } as Tool["parameters"], input: "false", expected: false }, { schema: { type: "boolean" } as Tool["parameters"], input: 1, expected: true }, { schema: { type: "boolean" } as Tool["parameters"], input: 0, expected: false }, - { schema: { type: "string" } as Tool["parameters"], input: null, expected: "" }, + // TypeBox's conversion rules come first: null is "null" for a required string. + { schema: { type: "string" } as Tool["parameters"], input: null, expected: "null" }, { schema: { type: "string" } as Tool["parameters"], input: true, expected: "true" }, { schema: { type: "null" } as Tool["parameters"], input: "", expected: null }, { schema: { type: "null" } as Tool["parameters"], input: 0, expected: null }, @@ -90,6 +90,11 @@ describe("validateToolArguments", () => { input: "1", expected: 1, }, + { schema: { type: "boolean" } as Tool["parameters"], input: "1", expected: true }, + { schema: { type: "boolean" } as Tool["parameters"], input: "0", expected: false }, + { schema: { type: "null" } as Tool["parameters"], input: "null", expected: null }, + { schema: { type: "integer" } as Tool["parameters"], input: "42.1", expected: 42 }, + { schema: { type: "array", items: { type: "number" } } as Tool["parameters"], input: "7", expected: [7] }, ]; for (const testCase of passingCases) { @@ -184,10 +189,6 @@ describe("validateToolArguments", () => { { type: ["array", "null"], items: { type: "string" } } as Tool["parameters"], null, ); - // The CSP test above selects TypeBox's process-wide interpreted fallback, so exercise the generated validator explicitly. - const generatedCheck = new Function(Compile(tool.parameters).Code())() as (value: unknown) => boolean; - - expect(generatedCheck(toolCall.arguments)).toBe(true); expect(validateToolArguments(tool, toolCall)).toEqual({ value: null }); }); @@ -196,10 +197,10 @@ describe("validateToolArguments", () => { schema: Tool["parameters"]; input: unknown; }> = [ - { schema: { type: "boolean" } as Tool["parameters"], input: "1" }, - { schema: { type: "boolean" } as Tool["parameters"], input: "0" }, - { schema: { type: "null" } as Tool["parameters"], input: "null" }, - { schema: { type: "integer" } as Tool["parameters"], input: "42.1" }, + { schema: { type: "boolean" } as Tool["parameters"], input: "yes" }, + { schema: { type: "number" } as Tool["parameters"], input: "abc" }, + { schema: { type: "null" } as Tool["parameters"], input: "nil" }, + { schema: { type: "integer" } as Tool["parameters"], input: "4x" }, ]; for (const testCase of failingCases) { @@ -207,6 +208,25 @@ describe("validateToolArguments", () => { expect(() => validateToolArguments(tool, toolCall)).toThrow("Validation failed"); } }); + it("lists pi-schema's errors, then the arguments it received", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "number" } as Tool["parameters"], "many"); + expect(() => validateToolArguments(tool, toolCall)).toThrow( + 'Validation failed for tool "echo":\n - value: expected number, got string\n\nReceived arguments:\n{\n "value": "many"\n}', + ); + }); + + it("reports a missing required property at the property", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + expect(() => validateToolArguments(tool, { ...toolCall, arguments: {} })).toThrow(" - value: missing"); + }); + + it("returns a copy of the arguments, also when nothing is coerced", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + const args = validateToolArguments(tool, toolCall); + expect(args).toEqual({ value: "x" }); + expect(args).not.toBe(toolCall.arguments); + }); + // #10897 it("reports malformed arguments JSON instead of validating empty arguments", () => { const tool: Tool = { diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 84c74692f..397107fa1 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -13,5 +13,8 @@ export default defineConfig({ }, resolve: { alias: [{ find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }], + // Workspace packages without a build (pi-schema) resolve to their source. + conditions: ["source"], }, + ssr: { resolve: { conditions: ["source"] } }, }); \ No newline at end of file diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index bbd4759bc..39a18568b 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -2,23 +2,53 @@ ## [Unreleased] +### Added + +- Chord v2's plugins component gains the plugin model (slice 12): `createPlugins({ api: { name, versions } })` checks every import of the app's versioned API (`@acme/app/v3`, per-kind modules such as `@acme/app/v3/web`): an unlisted version, another kind's module, two versions in one facet and a direct import of Chord are build errors naming the file and line. Roots may be `{ path, required }`: a required root's plugin failing at `load` rejects it with a `PluginError` after the load settled. Files named `..` (`contract.web.tsx`) are of that kind, refused in facets of another kind with the chain of imports named. Single-file plugins (`hello.server.ts` at a root's top, grouped by name, host packages only). `installPlugin({ git, ref } | { npm, version }, root, ctx)` installs plugins from git (a shallow clone of a branch, tag or commit; a repository of several plugins too) or npm; a package lists the plugin folders it provides in its `package.json` (`"chord": { "plugins": ["./notes", "plugins/*", "!plugins/draft"] }`), used by discovery and `installPlugin` (without it: one plugin when the package's top has facet files, else its subfolders with a `package.json`). `testPlugin(dir, { kind, modules, stubs, principal }, ctx)` builds a plugin as production does and loads it on an in-memory host beside stub facets, collecting its contributions to registries nobody serves and listing the contracts it waits for. Agents' guide: `docs/next/plugin-authoring.md`. + +### Changed + +- Chord v2 plugins import each other by relative path within a root (copied into the importer's script) and across repositories by package name through the package's `exports`, pinned in `dependencies`; slice 10's imports by plugin name (`@app/notes/contract`) are removed, and a relative path leaving the root is a build error. + +## [1.5.0] - 2026-11-02 + ### Breaking Changes +- Schemas are JSON Schema checked by `@earendil-works/pi-schema` instead of TypeBox, which is no longer a dependency of Chord. Contracts, `self.state(...)` schemas and method signatures take pi-schema builders or any JSON Schema object (TypeBox-built schemas still validate, being JSON Schema, but type as `unknown`); handler and reference types come from pi-schema's `Static`. Validation, input defaults and the stripping of unknown fields work on any JSON Schema by its shape (before, defaults and stripping applied only to TypeBox-built schemas). `format` is an annotation, no longer checked: `str({ format: "email" })` accepts `"nope"` (TypeBox checked the formats it knew); check a format in the handler where it matters. An `invalid` error's message is now ` ` with pi-schema's paths and messages (a missing field at its own path, `missing`; `expected string, got number` instead of `must be string`), and its `details` are pi-schema's `ValidationError`s (`{ path, keyword, message }`) instead of TypeBox's errors. Link frames, descriptions, fingerprints and identities are unchanged for schemas with the same JSON. Migration, keeping a contract's fingerprint (it is computed from the schema's JSON): `Type.String/Number/Integer/Boolean/Null/Unknown/Array/Union/Optional(...)` become `str/num/int/bool/nul/unknown/arr/union/opt(...)` with the same options; `Type.Object(p)` becomes `obj(p, { open: true })` (pi-schema's `obj` is closed by default) and `Type.Object(p, { additionalProperties: false })` `obj(p)`; `Type.Enum(values)` becomes `oneOf(values)`; `Type.Literal(v)` becomes `{ type: typeof v, const: v }` (pi-schema's `lit(v)` has no `type`); `Type.Record(Type.String(), v)` becomes `{ type: "object", patternProperties: { "^.*$": v } }` (pi-schema's `map(v)` uses `additionalProperties`); `Type.Intersect(list)` becomes `{ allOf: list }`; `Type.Function(params, result)` and `Type.Void()` become `{ type: "function", parameters, returnType }` and `{ type: "void" }`; `Type.Unsafe(schema)` becomes `schema as Schema`; `Type.Immutable(schema)` becomes the schema (cast its `Schema` type); `opt(...)` marks only an `obj` property; `Static` and `TSchema` come from pi-schema (`Static`, `Schema`). pi-schema's own `lit`, `map` and closed `obj` give other JSON and so other fingerprints, which matters only while peers of both versions talk. - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. +- Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. +- `SqliteExecutor.runCounted(sql, ...params)` runs one statement like `run` and resolves with the rows it inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not count). The Node driver takes it from `node:sqlite`; the Durable Object driver reads `SELECT changes()` after the statement (the cursor's `rowsWritten` also counts index writes), so `run` stays as cheap as before. Migration: custom `SqliteDatabase` and `SqliteExecutor` implementations add `runCounted`. +- Retirement is a flag, not an end: `WatchEnd` loses `retired`, and `Watch.retire(context)` is removed. A source reports a retirement with `advance({ value, ops, retired: true }, context)`, whose value is what the address shows after it, and the watch goes on: the next call has `retired: true`, folded with every other pending frame, so a document created at the address afterwards arrives in the same or a later call as `[["r", value]]`. `endAfterDelivery()` tells a watch its source sends nothing more: it takes no more frames, delivers what is pending, and ends `ended` after its last call returned, at once when nothing is pending and no call runs; it stays attached until then, so the source's `end()` (such as at its close), `stop()`, cancellation, or a throwing listener still end it first. Migration: a listener that waited for `closed` to give `retired` or tested for a `null` value tests `change.retired`; a source that called `retire(ctx)` calls `advance({ value, ops, retired: true }, ctx)`, then `endAfterDelivery()` when its owner ends with the retirement. +- Watch listeners get a tagged call, and a watch stays open through a failure its source recovers from: `WatchCall` is `{ kind: "value", value, ops, retired }` or `{ kind: "failed", error }`, and `NativeWatchCall` is `{ kind: "value", value }` or `{ kind: "failed", error }` (replacing `WatchChange` and `NativeWatchChange`). After a `failed` call `watch.value` keeps the last value and the next value arrives as `[["r", value]]`; while the listener is busy, a failure drops the pending values (`value` then `failed` folds into the failure) and a value replaces a pending failure (`failed` then `value` folds into the value, as `r`). `Watch.advance(call, context)` takes a `WatchCall`, a value call delivered alone still reaches the listener as the object the source passed (no allocation), and `Watch.fail(error, context)` and `NativeWatch.fail(error, context)` send a failure. Migration: `start(async ({ value, ops }, ctx) => …)` becomes `start(async (call, ctx) => { if (call.kind === "failed") return; const { value, ops } = call; … })`, a native `start(async ({ value }, ctx) => …)` likewise; `advance({ value, ops, retired }, ctx)` becomes `advance({ kind: "value", value, ops, retired }, ctx)`. +- `WatchEnd` is tagged by `kind` instead of `reason`, and a watch whose acquisition context is aborted ends `aborted` instead of `cancelled`: `{ kind: "stopped" | "aborted" | "ended" }`, `{ kind: "listener_error", error }`, or `{ kind: "failed", error }` (a final failure only). `Watch.cancel()` is `abort()`, `observeCancellation(signal)` is `observeAbort(signal)`, and `end(failure?)` takes the end: `end({ kind: "ended" })` or `end({ kind: "failed", error })`. Migration: `end.reason === "cancelled"` becomes `end.kind === "aborted"`, every other `end.reason` becomes `end.kind`; `watch.end()` becomes `watch.end({ kind: "ended" })`, `watch.end({ error })` becomes `watch.end({ kind: "failed", error })`. ### Added +- Chord v2's store (`@earendil-works/chord/next`) gains `ChordStore.commitSeries(commits, receive, ctx)`: several commits in one backend transaction, each at its own seq with its own outbox row, the store metadata written once. Each commit's check runs on a step that sees the commits before it, before its writes; a check that throws refuses that commit and stores the ones before it, while a failed statement or write rolls the whole series back, and a failed `COMMIT` stops the store. It replaces `commitInBatch`, which is removed. Durable's `StoreStorage` stores each Session commit call through it. +- Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). +- Chord v2 (`@earendil-works/chord/next`) gains registries, writables, wake signals and root scopes (slice 8): `defineRegistry(name, { version })` tokens shared by plugins, served by the defining plugin's facet with `host.serve(R, { compose, accepts? })`; contributions from facets (`defineFacet((host, self) => self.contribute(R, value))`) and host-lifetime objects (`StoreSelf.contribute`), plain values or Chord watchables, in a table sorted by plugin name and composed lazily (once at the end of `load`'s setups, then when read or followed, in the batch that changed the table); readers through `self.registry(R)` (a declared dependency: an object activates once the registry has a value, and fails `transient` with the new reason `unavailable/undeclared` when nobody serves it) and `host.registry(R, ctx)`, a `RegistryValue` with `observe(observer, ctx)`; older contributions and readers adapted by `accepts`; the helpers `keyedEntries` and `orderedEntries`; the `onError` kinds `registry_conflict`, `registry_failed`, `contribution_failed` and `contribution_refused`. `self.writable(initial)` is an in-process value an attempt sets (identity compared); native sources (writables, registry values, `ref.state`) are derive sources. `self.retryOn(...sources)` retries a failed object when a registry value changes. `host.root(fn, context, { principal? })` runs code outside objects with a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes. +- Chord v2 gains plugins (slice 7c): `src/next-node/plugins` (Node, to be exported as `@earendil-works/chord/next/plugins`) finds plugin directories, builds each facet with esbuild (now an optional peer dependency of `@earendil-works/chord`, imported only there), evaluates it with `vm.Script`, maps plugin stack traces (`mapStack`), installs plugin dependencies, watches plugin and base code, and rolls reloads out across hosts. `host.load` and `host.reload` work per plugin: a plugin whose setup, factory or declarations fail is left out at `load` (`{ failed }`) or kept on its old code at `reload` (`failed` in the report, `reload(facets, ctx, { failed })` for build and evaluation failures), with a closure rule keeping a consumer old when its provider was kept old; an abort after a reload's commit rejects with `details.committed`. +- Chord v2 gains HTTP and workers (slice 10): `src/next-node/http` (to be exported as `@earendil-works/chord/next/http`) builds a Web-standard `fetch` handler from the routes, wraps, sockets, hosts, relays and CSP sources plugins contribute to the registry `Http`, with authenticators from the registry `Authenticators`, responses as descriptions (`json`, `html`, `bytes` with ranges, `redirect`, `error`; `fileResponse`), and a security baseline on by default (provenance by the socket and `Host`, foreign requests anonymous, `Origin` checks, default headers with a per-response CSP nonce, signed cookies, safe downloads); Chord's own Node listener as a host-lifetime object, `nodeRequest` for apps mounting it on their own Node server, the links endpoint and plugin sockets with a heartbeat, closed at once when their contribution leaves. `ws` is an optional peer dependency, imported only there. `src/next-node/workers` (`createWorkers`, `workerMain`) runs a forked worker per address of a host kind, started on demand, stopped when idle, never restarted by itself (a failure stays until `retry` or a rollout), its plugins built by the server and shipped. The plugins component defines one definition per facet kind (`kinds: { name: { platform, hostPackages } }`, modules given at `plugins.load(host, { kind, core, modules })`), splits loading into `createPluginRuntime()` (no esbuild), offers `scripts(kind)`, rolls out with one result per host (`RolloutStep` returns `{ hosts }`), and resolves cross-plugin imports by plugin name (`@app/notes/contract`). Kernel: `host.route` returns a `RouteHandle` (`retry`, `suspend`) and a route has no backoff timer (a failed `resolve` fails its holders `waiting`); `host.root(fn, ctx, { principal: undefined })` runs anonymous; a link reads `origin` only on parent and route links (a protocol error on a child link) and drops a per-origin record once it is idle. +- Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. -- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`. `WatchEnd` reasons are `stopped`, `cancelled`, `retired`, `ended`, `listener_error`, and `failed`. -- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(value, ops, context)`, `retire(context)` (last frame `[["r", null]]`, then `retired`), `end()` or `end({ error })`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. +- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. +- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(change, context)` (`retired: true` for a retirement), `end()` or `end({ error })`, `endAfterDelivery()`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. - `Watchable`: a value that changes and can be watched (`value`, `watch(context)` returning a `WatchHandle`), the common interface of documents, derives and other observable values. -- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `(value, context)` and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. +- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `({ value }, context)` (`NativeWatchChange`) and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. - `PendingOps` and `PendingTake` in `@earendil-works/chord/delta`: the pending ops of one slow consumer, kept raw and folded while they wait (appends to one path join, a primitive set replaces earlier leaf ops on its path, `r` drops everything before it); `size` counts the pending ops after folding; `take()` empties the queue and hands over the folded ops, or `replace` past 10,000 pending ops. Received batches and op tuples are never mutated. - `ownedSlice(text, start, end?)` in `@earendil-works/chord/delta`: a slice of a string that does not keep the whole string alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - `appendText(draft, key, text)` in `@earendil-works/chord/delta`: append to a string in a draft at O(text) cost, recorded as an append op without comparing the old and new strings (a plain `draft[key] += text` makes `prepare()` read the whole string, O(length) per change); `StringKeys` names the keys it accepts ([#9265](https://github.com/earendil-works/pi/issues/9265)) +### Changed + +- Chord v2 (`@earendil-works/chord/next`) checks its schemas with `@earendil-works/pi-schema` instead of TypeBox: its helpers (`str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, `unknown`) build on pi-schema's builders and keep the JSON contracts were fingerprinted with, so every fingerprint, normal form and stored record stays the same (a test records them and reads a store written before the switch). `Static` and `Schema` (was `TSchema`) come from pi-schema; `opt(...)` marks an object field or a table column and is no schema elsewhere (a top-level `opt` had no effect before); a schema with a non-enumerable key (a TypeBox schema's metadata) is refused at declaration; `map(bytes())` is accepted now (it was refused while `bytes()` carried a TypeBox refinement). `format` is an annotation, no longer checked (`{ type: "string", format: "email" }` accepts `"nope"`; TypeBox checked the formats it knew). A failed check's message comes from pi-schema (`args/title: expected string, got integer`), its `details.path` escapes `~` and `/` (JSON Pointer); reasons and codes are unchanged. Declaring and compiling are cheaper (a 20-method contract 1.07 ms to 0.86 ms, a validator's first use 172 µs to 48 µs), a check costs the same (0.25 µs), the message envelope check up to 100 ns more per message. +- The delta tracker keeps its one open change in a plain field and needs a `WeakRef` only for a second change open at the same time (one per change before). A `WeakRef` target survives the scavenges of its job, so a run of changes in one job promoted every change's overlay to old space: 200,000 small changes in one job now promote 2 MB instead of 47.5 MB and take 3.1 µs each instead of 3.8 µs (3.0 instead of 3.5 µs when they span jobs). A draft dropped without `prepare()` or `abort()` now stays referenced by its tracker until the tracker's next `beginChange()` or `adopt()`. +- `applyImmutable()`, `applyImmutableBatches()` and tracker revisions copy each container on a changed path with spread (or `Object.assign()` onto a null-prototype object) instead of one `Object.defineProperty()` per key, about 25 times faster per container ([#9265](https://github.com/earendil-works/pi/issues/9265)) + ### Fixed +- A host a route resolves is admitted even when the relayed reference that asked for it is released, its client disconnects, or its principal is refused or revoked while the route resolves. Before, the parent never read the new host's link: a spawned worker's own references to its parent (its `lifetime: "host"` objects' upstream references) waited until they failed `unavailable/ready_timeout`, reported as a possible reference cycle across hosts. +- The Node SQLite driver keeps the 256 most recently used prepared statements per connection instead of every statement it ever prepared, so callers generating SQL of unbounded shapes (a query builder) no longer grow memory without bound. - Fixed append ops from the delta tracker and `diffRevisions()` keeping the whole string they were cut from alive (V8 represents a slice as a view of its source), so a delta chain, a watch backlog or a replica built from appends to a growing text held one copy of the text per commit ([#9265](https://github.com/earendil-works/pi/issues/9265)) ## [1.4.0] - 2026-10-21 diff --git a/packages/chord/README.md b/packages/chord/README.md index 4239568bc..081e3fe21 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -8,13 +8,13 @@ Chord builds applications out of **objects** that live in several processes (a s - **State** is observable JSON. Remote viewers get a snapshot followed by deltas. - **Hot reload** swaps code while it runs. Work in flight finishes on the code it started with, state is kept, and incompatible changes migrate at a safe point. -Contracts are runtime values described with [TypeBox](https://github.com/sinclairzx81/typebox) schemas. Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. +Contracts are runtime values described with JSON Schema, written with the builders of `@earendil-works/pi-schema` (any JSON Schema object works; `format` is an annotation, never checked). Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. ## Entry points | Import | Contents | Runs in | |--------|----------|---------| -| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | +| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, `WatchCall`, `NativeWatchCall`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | | `@earendil-works/chord/node` | Child host processes: `createHostManager`, `forkDriver`, `childControl`, `processLock`; the facet module loader `nodeLoader`. | Node | | `@earendil-works/chord/context` | Explicit cancellation, telemetry, and invocation values: `BACKGROUND_CONTEXT`, `withCancel`, `withAbortSignal`, `withoutAbortSignal`, `withTelemetry`, `traceIn`, `withContextValue`, `awaitWithContext`. | Node and browsers | | `@earendil-works/chord/delta` | The transactional JSON tracker, structural diff, immutable apply, and the wire encoder/decoder for operation batches. | Node and browsers | @@ -27,18 +27,18 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, hostNamed, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), host: hostNamed("server"), // where its objects live; callers never name a host variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -46,7 +46,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, @@ -209,27 +209,59 @@ model; see [State](docs/guide.md#state) in the usage guide. `WatchHandle` is the one watch interface, implemented by watch sources such as Durable's documents, conversation views, and task graph. `watch.value` is the value at acquisition until `start(listener)` installs the sole asynchronous listener; each -later change arrives as `(value, ops, context)`, one call at a time, and `value` -advances to it. Changes made while the listener runs (or before `start`) arrive in -one call: the newest value, the ops of all of them folded by Delta's `PendingOps`, -and the newest change's context. Past 10,000 pending ops after folding the call -gets one root replacement `[["r", newest]]` instead. `stop()` is idempotent, drops -pending frames, and neither aborts nor awaits a running call; it and `closed` give -the `WatchEnd`: +later change arrives as `(call, context)`, one call at a time, with +`call: WatchCall` tagged by `kind` (plain data; the context is the second +argument): + +- `{ kind: "value", value, ops, retired }`: the new value, and `watch.value` + advances to it; +- `{ kind: "failed", error }`: the source cannot compute its value now. The watch + stays open and `watch.value` keeps the last value; the next value arrives as + `[["r", value]]`. + +```ts +watch.start(async (call) => { + if (call.kind === "failed") return showError(call.error); + render(call.value); +}); +``` + +Changes made while the listener runs (or before `start`) arrive in one call: the +newest value, the ops of all of them folded by Delta's `PendingOps`, and the newest +change's context. Past 10,000 pending ops after folding the call gets one root +replacement `[["r", newest]]` instead. A failure drops the pending values and waits +as one `failed` call (the newest failure); a value after it replaces it, as one +`[["r", value]]` call. + +`retired` says the document the watch follows was retired since the previous value +call. It does not end the watch: the call carries what the address shows after the +retirement (the source decides; a Durable document keeps the last value the watch +was given, with no ops), and a document created there later arrives as +`[["r", value]]`, folded with what is pending, so one call can report a retirement +and carry the new value. + +A watch ends only for good. `stop()` is idempotent, drops pending frames, and +neither aborts nor awaits a running call; it and `closed` give the `WatchEnd`, +tagged by `kind`: - `stopped`: `stop()` was called; -- `cancelled`: the context the watch was acquired with was cancelled; -- `retired`: the source was retired, an explicit event after a last frame `[["r", null]]`; +- `aborted`: the context the watch was acquired with was aborted; - `ended`: the source closed (its owner ended or closed); - `listener_error` with `error`: the listener threw or rejected; -- `failed` with `error`: the source failed. +- `failed` with `error`: the source failed for good. An ended watch drops every reference into its source, so a stopped watch that is still referenced keeps only its last value alive. -Sources implement handles with `Watch`, fed through `advance(value, ops, context)`, -`retire(context)`, `end()` or `end({ error })`, and `observeCancellation(signal)`. +Sources implement handles with `Watch`, fed through `advance(call, context)` +(a `WatchCall`: a value with `retired` set for a retirement, or a failure), +`fail(error, context)` (a failure the source may recover from), `end(end)` (a +`WatchEnd`, at once, dropping what is pending), `endAfterDelivery()` (take no more +frames, deliver what is pending, then end `ended` once the last call returned; the +source's `end()` still ends it first), and `observeAbort(signal)`. `NativeWatch` implements `NativeWatchHandle`, the same lifecycle for -in-process values: the listener gets `(value, context)` without ops, one call per -value in order; past 100 pending values they collapse to one, the value of the -optional `replace()` or the newest. +in-process values: the listener gets a `NativeWatchCall` +(`{ kind: "value", value }` without ops, or `{ kind: "failed", error }`), one call +per value in order; past 100 pending values they collapse to one, the value of the +optional `replace()` or the newest. A failure drops the pending values; values +after it are delivered in order. diff --git a/packages/chord/docs/guide.md b/packages/chord/docs/guide.md index 59c1414b7..c3b7b3211 100644 --- a/packages/chord/docs/guide.md +++ b/packages/chord/docs/guide.md @@ -36,17 +36,17 @@ Chord builds applications out of **objects** that live in several processes (a s ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -54,7 +54,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, @@ -82,7 +82,7 @@ Remote types can be reached from other hosts. Everything that crosses the wire h ```ts export const Chat = remote("pi.chat", { version: 1, - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // lives on `session:S1` for `{ session: "S1", … }` variants: { viewer: { @@ -91,7 +91,7 @@ export const Chat = remote("pi.chat", { participant: { state: { view: ConversationViewSchema }, methods: { - send: method(SendInputSchema, Type.Object({ submissionId: Type.Number() })), + send: method(SendInputSchema, obj({ submissionId: num() })), stop: method(), // no arguments, no result: `chat.stop(ctx)` }, }, @@ -119,10 +119,10 @@ Local types are reachable only inside their host. Their members may be anything: ```ts export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(HitSchema)), - onChange: method(Type.Function([ChangeSchema], Type.Void()), UnsubscribeSchema), + search: method(obj({ query: str() }), arr(HitSchema)), + onChange: method({ type: "function", parameters: [ChangeSchema], returnType: { type: "void" } }, UnsubscribeSchema), harness: opaque("pi.durable-harness@1"), }, }); @@ -181,7 +181,7 @@ A reload runs the factory again over the same object. Declarations that keep som ```ts host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }); - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); const db = self.resource("db", { version: 1, @@ -230,7 +230,7 @@ host.serve(Visits, (_, _p, self) => { // right host.serve(Visits, (_, _p, self) => { - const visits = self.state("visits", Type.Number(), 0); + const visits = self.state("visits", num(), 0); return { hit: async (ctx) => { visits.change(ctx, (n) => n + 1); @@ -240,7 +240,7 @@ host.serve(Visits, (_, _p, self) => { }); ``` -**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, Type.Number())` takes no arguments and returns a number. +**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, num())` takes no arguments and returns a number. ## Activation @@ -428,7 +428,7 @@ chat.release(); ## State ```ts -const todos = self.state("todos", Type.Array(TodoSchema), []); +const todos = self.state("todos", arr(TodoSchema), []); todos.change(ctx, (draft) => { draft.push({ id: "t1", text: "Write docs", done: false }); @@ -457,7 +457,7 @@ view.replace(ctx, rebuilt, { diff: true }); // always diff ```ts const db = self.resource("db", { version: 1, open: () => openSqlite(path), close: (db) => db.close() }); -const notes = self.state("notes", Type.Array(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); +const notes = self.state("notes", arr(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); ``` `init(ctx)` runs once the resources are open and the references live, before the object goes live, so no method, diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md new file mode 100644 index 000000000..801a30c66 --- /dev/null +++ b/packages/chord/docs/next/benchmarks.md @@ -0,0 +1,974 @@ +# Chord v2 benchmarks + +Light benchmarks of hot paths, run by hand (not in CI), recorded per slice for later slices to compare against +(owner, 2026-10-22). Each section names its script, the machine and what the numbers include. + +## Slice 3a: remote objects over one link + +Script: `test/next/bench/links.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc test/next/bench/links.ts` (about 15 s). Node 24.4.1, +Intel Core i9-13900, one process: server and client host on one memory link (`queueMicrotask`), every case once per +codec, side by side. + +Payloads, modelled on p2: entries `{ id, text, author, at }` in one document of a local store, exposed by a remote +view; "1,000 items" are entries with 80-character texts (the notes list), "500 KB" are 500 entries of 960 characters +(a conversation document). Commits are server-side method calls (one transaction each) on the store. + +What the numbers include: + +- *first hold to live*: the first `client.get`, which also activates the view on the server; one sample, so it moves + by a few hundred µs between runs (JIT and collector state). +- *hold to live (mean of 100)*: further `client.get`s of the live view, each released: the `hold`, the snapshot, the + `bound` message encoded, decoded and checked, the replicas built (two link hops). +- *wire bytes*: one whole message as its codec writes it (memory links add nothing). +- *encode / decode*: that message through the codec alone (`decode` without the schema check), averaged over 10,000 + runs (1,000 for snapshots and the 1 MiB call), the fastest of five such loops (one loop of a microsecond-sized + message is at the mercy of the collector and the JIT). +- *heap per encode + decode*: heap bytes allocated per message while encoding and decoding it that many times, + measured with `GCProfiler` (the growth plus what the collections freed); `ArrayBuffer` backing stores (the message + bytes themselves) are outside the heap. +- *array buffer kept per encoded message*: the `process.memoryUsage().arrayBuffers` growth while 50 encoded copies of + the message are kept, per message: the backing store a queued or held message keeps. Freed buffers are not reported + reliably inside a loop, so allocation of temporary buffers is not measured. V8 keeps very small typed arrays on the + heap, so the smallest binary messages show 0 B here. +- *commit to frame*: from the server's method call to the client's watch listener, averaged over 1,000 commits: the + call and its transaction, the encodes, the hop, decode, check, apply, and the watch's delivery. +- *heap / array buffers retained per hold*: the heap and `arrayBuffers` growth of holding the 1,000-item view, after full collections, divided by the + holds; both ends (the provider's member encoders and subscriptions, the consumer's member decoders and replicas, + each its own decoded copy of the snapshot), median of seven trials. + +| Case | binary | text | +| ------------------------------------------------------------------------------- | --- | --- | +| notes list of 1,000 items: first hold to live (activation, one sample) | 0.958 ms | 1.041 ms | +| notes list of 1,000 items: hold to live (two hops, mean of 100) | 0.666 ms | 0.686 ms | +| notes list of 1,000 items: snapshot: wire bytes | 133.7 KiB | 133.7 KiB | +| notes list of 1,000 items: snapshot: encode / decode | 291.1 µs / 266.4 µs | 312.3 µs / 302.8 µs | +| notes list of 1,000 items: snapshot: heap per encode + decode | 441.5 KiB | 441.8 KiB | +| notes list of 1,000 items: snapshot: array buffer kept per encoded message | 137.7 KiB | 133.7 KiB | +| 500 KB document: first hold to live (activation, one sample) | 1.461 ms | 1.555 ms | +| 500 KB document: hold to live (two hops, mean of 100) | 1.138 ms | 1.309 ms | +| 500 KB document: snapshot: wire bytes | 496.5 KiB | 496.5 KiB | +| 500 KB document: snapshot: encode / decode | 615.4 µs / 390.8 µs | 687.5 µs / 528.3 µs | +| 500 KB document: snapshot: heap per encode + decode | 1511.2 KiB | 1511.2 KiB | +| 500 KB document: snapshot: array buffer kept per encoded message | 500.5 KiB | 496.5 KiB | +| one appended item: commit to frame | 0.037 ms | 0.031 ms | +| one appended item: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one appended item: state message: wire bytes | 165 B | 175 B | +| one appended item: state message: encode / decode | 1.4 µs / 1.4 µs | 0.9 µs / 1.1 µs | +| one appended item: state message: heap per encode + decode | 2607 B | 3034 B | +| one appended item: state message: array buffer kept per encoded message | 165 B | 175 B | +| one changed field: commit to frame | 0.036 ms | 0.031 ms | +| one changed field: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one changed field: state message: wire bytes | 27 B | 38 B | +| one changed field: state message: encode / decode | 0.6 µs / 1.0 µs | 0.5 µs / 0.7 µs | +| one changed field: state message: heap per encode + decode | 2005 B | 2527 B | +| one changed field: state message: array buffer kept per encoded message | 0 B | 38 B | +| one removed item: commit to frame | 0.031 ms | 0.029 ms | +| one removed item: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one removed item: state message: wire bytes | 23 B | 34 B | +| one removed item: state message: encode / decode | 0.6 µs / 1.0 µs | 0.5 µs / 0.7 µs | +| one removed item: state message: heap per encode + decode | 1704 B | 2575 B | +| one removed item: state message: array buffer kept per encoded message | 0 B | 34 B | +| streamed reply, 2,000 appends: time per append | 0.031 ms | 0.027 ms | +| streamed reply: bytes per append, total | 23 B, 45.0 KiB | 34 B, 66.5 KiB | +| streamed token: wire bytes | 23 B | 34 B | +| streamed token: encode / decode | 0.5 µs / 1.0 µs | 0.5 µs / 0.6 µs | +| streamed token: heap per encode + decode | 1696 B | 2527 B | +| streamed token: array buffer kept per encoded message | 0 B | 34 B | +| 1 hold(s) of 1,000 items: heap / array buffers retained per hold (both ends) | 154.1 KiB / 47 B | 159.7 KiB / 21 B | +| 1 hold(s): messages per commit | 1 | 1 | +| 10 hold(s) of 1,000 items: heap / array buffers retained per hold (both ends) | 175.2 KiB / 41 B | 175.2 KiB / 44 B | +| 10 hold(s): messages per commit | 1 | 1 | +| call, small argument: round trip | 0.016 ms | 0.015 ms | +| call, small argument: call message: wire bytes | 16 B | 23 B | +| call, small argument: call message: encode / decode | 0.6 µs / 0.4 µs | 0.5 µs / 0.6 µs | +| call, small argument: call message: heap per encode + decode | 1580 B | 1603 B | +| call, small argument: call message: array buffer kept per encoded message | 0 B | 23 B | +| call, small argument: result message: wire bytes | 9 B | 14 B | +| call, small argument: result message: encode / decode | 0.4 µs / 0.3 µs | 0.5 µs / 0.5 µs | +| call, small argument: result message: heap per encode + decode | 1219 B | 1509 B | +| call, small argument: result message: array buffer kept per encoded message | 0 B | 14 B | +| call, 1 MiB bytes argument: round trip | 0.192 ms | 0.159 ms | +| call, 1 MiB bytes argument: call message: wire bytes | 1024.0 KiB | 1024.0 KiB | +| call, 1 MiB bytes argument: call message: encode / decode | 113.7 µs / 1.4 µs | 123.5 µs / 1.5 µs | +| call, 1 MiB bytes argument: call message: heap per encode + decode | 3042 B | 3383 B | +| call, 1 MiB bytes argument: call message: array buffer kept per encoded message | 1028.0 KiB | 1024.0 KiB | + +Observations: + +- One message per commit and one encode per hold in every case (W1). +- The binary codec makes small messages 30-45 % smaller (a streamed token 23 B against 34 B; a changed field 27 B + against 38 B) and allocates less heap per message; both are about 1 µs per small message each way. Binary decodes + small messages a little slower than text: it makes one call per field (varints, strings, one `JSON.parse` per value) + where text makes one `JSON.parse` of the whole tuple; the difference is under 1 µs per message and not worth + optimizing. Snapshots are JSON in both and about the same size; binary decodes them faster, text encodes them a + little faster (binary encodes the JSON text, then copies it into the message once). +- A binary message keeps a right-sized backing store (a 500 KB snapshot keeps about 500 KiB, not the 1.5 MB a buffer + reserved for the worst-case UTF-8 length kept before the review). +- Kept backing stores of small binary messages (owner, 2026-10-22): a message is written into a 256-byte starting + buffer, whose exact size is not known before the message is written, so in-place encoding does not right-size it. + `finish` copies to an exact-size buffer when the spare room is at least 64 bytes and more than an eighth of the + message. The appended-item state message (165 B) now keeps 165 B instead of 256 B, for about 0.2 µs more per encode + (1.1 against 1.3-1.4 µs); messages up to about 100 B were copied before already (they show 0 B: V8 keeps such small + arrays on the heap); large ones end within their 4 KiB of room and are not copied (the 1 MiB call keeps 1028 KiB). +- Large snapshots (owner's profiling request, 2026-10-22): the earlier "hold to live" gap (2.67 ms binary, 1.67 ms + text) was one sample each; averaged over 200 holds both codecs took about 1.2 ms. The remaining real gap was the + encode (886 against 718 µs): binary encoded the JSON text into its own buffer, then copied it into the message. + Long strings (from 4,096 UTF-16 units) are now encoded straight into the message buffer behind a fixed five-byte + length (a padded varint), and short ones (up to 42 units, a one-byte length) too: binary now encodes a 500 KB + snapshot in 650 µs against text's 690 µs and holds it in 1.14 ms against 1.30 ms. +- A 1 MiB `bytes()` argument travels as 1 MiB plus a few bytes in both codecs (not base64's 1.33 MiB); its decode is a + view (under 2 µs), its encode the one copy into the message. +- Encoding is not the bottleneck at these sizes; no CBOR. +- Each hold's replica is its own decoded copy (about 160-175 KiB for 1,000 entries); two references from one host are + two holds by design. + +## Slice 3b: surviving drops + +Script and machine as for slice 3a (`test/next/bench/links.ts` now runs the 3b cases after the 3a ones, about 37 s). + +New cases, both on the 1,000-item notes list view (133.7 KiB snapshot): + +- *reconnect, N reference(s)*: N references held on the client (N holds of one provider object); the server's end + of the link closes, the client sees the drop, then a new link is accepted and connected. *Time to all live*: from + `connect` to every reference's `ref.state` being `live` again (the re-holds, N snapshots encoded, decoded and + adopted as one frame `[["r", value]]` per member); the mean of 20 reconnects for one reference, 5 for 100. *Bytes*: + what the new link carried both ways until then (the handshake included). +- *1,000 calls parked while down*: a small call (`ping`) made 1,000 times right after `connect`, before the re-hold is + bound, so every call parks; time from `connect` to the last result (the re-hold, then 1,000 calls sent in order and + answered), mean of 10; the order is checked at the provider and in the results. + +| Case | binary | text | +| --- | --- | --- | +| reconnect, 1 reference of 1,000 items: time to all live (mean of 20) | 0.870 ms | 0.791 ms | +| reconnect, 1 reference of 1,000 items: bytes down / up | 133.7 KiB / 63 B | 133.8 KiB / 110 B | +| reconnect, 100 references of 1,000 items: time to all live (mean of 5) | 63.1 ms | 66.4 ms | +| reconnect, 100 references of 1,000 items: bytes down / up | 13370.5 KiB / 4718 B | 13371.1 KiB / 6736 B | +| 1,000 calls parked while down, flushed on reconnect: time to the last result | 14.9 ms | 13.8 ms | +| 1,000 parked calls: run in the order they were made | yes | yes | + +Regression rows (the live paths of 3a, same run): + +| Case | binary | text | +| --- | --- | --- | +| notes list of 1,000 items: hold to live (two hops, mean of 100) | 0.750 ms | 0.682 ms | +| 500 KB document: hold to live (two hops, mean of 100) | 1.134 ms | 1.308 ms | +| one appended item: commit to frame | 0.037 ms | 0.031 ms | +| one changed field: commit to frame | 0.036 ms | 0.029 ms | +| streamed reply, 2,000 appends: time per append | 0.031 ms | 0.027 ms | +| 1 hold of 1,000 items: heap / array buffers retained per hold (both ends) | 154.6 KiB / 47 B | 160.1 KiB / 21 B | +| 10 holds of 1,000 items: heap / array buffers retained per hold (both ends) | 175.1 KiB / 41 B | 175.4 KiB / 44 B | +| call, small argument: round trip | 0.016 ms | 0.014 ms | +| call, 1 MiB bytes argument: round trip | 0.192 ms | 0.191 ms | +| messages per commit, encodes per hold (every commit case) | 1, 1 | 1, 1 | + +Observations: + +- The live paths are unchanged within noise (3a: 0.67 / 0.69 ms per hold, 0.037 / 0.031 ms per appended item, 154.1 / + 159.7 KiB per hold); message sizes and encodes per commit are identical. A live reference's call checks its state + once and makes no queue; `ref.state` is not made until read. +- A reconnect costs what the holds cost the first time: about 0.63 ms and one 133.7 KiB snapshot per reference of + this size (100 references: 63 ms, 13 MiB). The consumer sends only the `hold`s (47 B each binary). +- 1,000 parked calls take about 14 ms to flush and answer, a little less than 1,000 sequential round trips (16 ms): + they leave in one turn once the re-hold is bound. Each parked call keeps its arguments and one timer until sent. + +## Slice 4a: links and relays + +Script: `test/next/bench/relay.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc --no-flush-bytecode test/next/bench/relay.ts` (about +5 s; the forked workers are the same file run with the argument `worker`). Machine as for slice 3a. Binary codec +throughout. The topology is the one of slice 4a's example: a tab (a client host), a server with an address and a +route for `Session`, and per session a worker host whose `Store` (a local document) backs a remote `View` placed on +the session; the tab holds the view through the server, which relays (no part parsed). + +What the numbers include: + +- *call to frame*: the tab calls `stream` (one commit appending `" tok"` on the worker) and waits for both the result + and its watch frame; mean of 1,000 after 100 warm-up calls. Through a relay that is four hops for the call and four + for the frame and result (tab → server → worker → server → tab); the baseline is a tab directly on the worker. + Slice 3a's *commit to frame* started at the provider, so these are not comparable with it. +- *bytes per commit*: bytes the server sent to the tab, and received from the worker, per commit (the `state` + message and the call's result; acks and the call go the other way). From the worker the messages carry the tab + link's origin (two bytes each). +- *first hold, `resolve` included*: a `tab.get` of a new session: the route resolves (an in-process worker host is + created, connects, loads its facet with 1,000 entries and activates the view), the relayed hold, the 133.7 KiB + snapshot through two links. The IPC row's first hold includes forking a Node process with strip-types (one sample). +- *the throttled tab*: the tab's end of its link delivers at 64 KiB/s (whole messages, in order; idle time banks no + budget) in **virtual time**, 10 ms steps, so 60 s stalls run in well under a second. Per step: one token committed + on the worker (100 tokens/s), the throttle's delivery, then every host's turn. Phases: the first snapshot of the + 500 KB view (8 s of virtual time at 64 KiB/s), 2 s of streaming, the pause (a stall: the tab receives nothing, so + it acknowledges nothing; or the control: the same streaming without a stall), then 10 s more of streaming and + until the tab's text equals the worker's. *Caught up after the stall*: virtual time from the end of the pause to + the first step where the tab's view equals the worker's document; *bytes until then*: what the throttle delivered + meanwhile. *Heap growth during the stall*: the + process's heap after full collections at the end of the pause minus at its start (all hosts share one process; + compare with the control row, which is the same streaming without a stall). *Commit at the worker*: the mean time + of `store.stream` (the transaction, its flush: an encode when the window has room, a push into the pending queues + when it is full), outside the pause and during it. *Longest message handling at the provider*: the longest single + received message handled after the pause at the worker, the provider of the tab's state: its end of the tab link + directly, of the route link relayed (the `ack` that sends the folded `state`). +- *calls*: tab B's sequential `ping`s through the relay, alone and while tab A has issued 10,000 `ping`s at once on + the same session (two holds, two origins at the worker, each with its N = 256 running calls; the relay forwards at + once). + +| Topology (1,000 streamed tokens into the 1,000-item notes list view) | call to frame | bytes per commit: to the tab / from the worker | +| --- | --- | --- | +| memory tab, in-process worker (memory links) | 0.053 ms | 27 B / 29 B | +| memory tab, forked worker (IPC) | 0.097 ms | 27 B / 29 B | +| first hold to live, the fork included (one sample) | 240 ms | - | +| loopback WebSocket tab (`ws` serving, Node's `WebSocket`), forked worker (IPC) | 0.148 ms | 27 B / 29 B | +| no relay: memory tab on the worker (baseline) | 0.034 ms | 27 B / - | + +| Hold to live, the 1,000-item notes list view (133.7 KiB snapshot) | time | +| --- | --- | +| first hold, `resolve` of an in-process worker included (mean of 20) | 1.375 ms | +| hold of a resolved session through the relay (four hops, mean of 100) | 0.763 ms | +| no relay: hold on the worker (two hops, mean of 100, baseline) | 0.703 ms | + +| Throttled tab (64 KiB/s), 100 tokens/s into the 500 KB view | caught up after the pause | bytes to the tab until then | heap growth during the pause | commit at the worker, outside / during the pause | longest message handling at the worker after the pause | +| --- | --- | --- | --- | --- | --- | +| direct, 1 s stall | 0.02 s | 485 B | 28.0 KiB | 0.018 ms / 0.030 ms | 0.169 ms | +| direct, 10 s stall | 0.08 s | 4147 B | 134.2 KiB | 0.029 ms / 0.023 ms | 0.053 ms | +| direct, 60 s stall | 0.38 s | 23.7 KiB | 392.2 KiB | 0.028 ms / 0.017 ms | 0.138 ms | +| direct, 60 s without a stall (control) | 0.01 s | 23 B | 482.7 KiB | 0.025 ms / 0.017 ms | 0.005 ms | +| relayed, 1 s stall | 0.02 s | 485 B | 7.5 KiB | 0.025 ms / 0.025 ms | 0.057 ms | +| relayed, 10 s stall | 0.08 s | 4147 B | -25.3 KiB | 0.030 ms / 0.015 ms | 0.059 ms | +| relayed, 60 s stall | 0.38 s | 23.7 KiB | 513.5 KiB | 0.026 ms / 0.015 ms | 0.146 ms | +| relayed, 60 s without a stall (control) | 0.01 s | 23 B | 257.1 KiB | 0.023 ms / 0.021 ms | 0.003 ms | + +| Calls through the relay (in-process worker, memory links) | p50 | p95 | max | calls | +| --- | --- | --- | --- | --- | +| tab B alone | 0.012 ms | 0.018 ms | 0.600 ms | 1,000 | +| tab B while tab A has 10,000 calls in flight or queued | 4.851 ms | 10.701 ms | 10.708 ms | 27 | +| tab A: 10,000 calls at once, until the last result | 657 ms | - | - | 10,000 | + +Observations: + +- A relay hop costs about 0.019 ms per commit-to-frame on memory links (0.053 against 0.034 ms for call, commit, frame + and result) and adds no bytes to the tab: the server forwards the worker's message with the origin taken off (ids + unchanged, parts copied once); the origin costs two bytes per message on the route link. IPC to a forked worker adds about 0.04 ms, a loopback WebSocket to the tab another 0.05 ms. +- Hold to live through the relay is within 9% of a direct hold (0.763 against 0.703 ms): the relay never parses the + 133.7 KiB snapshot. A first hold that resolves an in-process worker (create, connect, load, activate) takes 1.4 ms; + forking a strip-types Node process takes about 240 ms, which is the app's to hide (start workers ahead). +- Directly, the provider's pending queue folds a stall of any length into one append (24 KiB of text after 60 s, + sent in 0.38 s at 64 KiB/s); committing while the window is full costs no more than committing with room (about + 0.02 ms), and the folded message is encoded in under 0.2 ms. The stall adds no heap beyond the control's, which is + the streamed text itself on the worker and the tab. +- Through the relay a stall now costs what it costs directly (decision "Relay flow control per origin"): the worker + keeps the tab origin's window and folds its commits into its pending queues, the relay forwards and acknowledges + per message. Before this rework the relay ran the tab's window itself and recovered with `reset`: every stall, even + 1 s, cost three full snapshots (about 1.5 MiB) and about 24 s to catch up under sustained streaming. +- Tab B's calls are their own origin at the worker, with their own N, but B's call still waits behind the up to 256 + calls of A running there (one worker, one event loop): about 4.9 ms median, 11 ms p95, against 0.01 ms alone. A's + 10,000 calls take 0.07 ms each (its window of 256 refills one reply round trip through the relay at a time). +- A first run of this script found about 350 B retained per call or commit: `AbortSignal.any` on Node 24.4.1 keeps + cleanup entries per call while its sources live (nodejs/node#62367, fixed in 24.16.0). Calls and gets now link + their signals only until they settle (`withLinkedAbortSignal`); the numbers above are after that fix (heap during a + 60 s stall: 2.3 MiB before, 0.4 MiB after). + +## Slice 4b: authority and housekeeping + +Script: `test/next/bench/compression.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc test/next/bench/compression.ts` (about 5 s). Node +24.4.1, Intel Core i9-13900, one process. + +Payloads: the 500 KB document of slice 3a (500 entries `{ id, text, author, at }` of 960 characters), but with +prose-like text (words from a small vocabulary in a fixed pseudo-random order, some with numbers: it compresses about +3.6x at level 1, as conversation JSON does; the `"x".repeat` texts of the earlier benchmarks would compress about +100x and say nothing), and a 1 MiB string as a call's result (a file export). Compression is option (b) as built: +`nodeCompression` (zlib's raw deflate at level 1) at the origin; the owner inflates with `nodeCompression` +(synchronous, a Node client) or `DecompressionStream` (asynchronous, a browser). + +- *deflate at the origin / inflate*: one part through the adapter alone, mean of 20, the fastest of three loops. +- *end to end*: a server and a client host on a memory link (binary codec); *hold to live* is a further `get` of the + live view (snapshot encoded, deflated, sent, inflated, parsed, replicas built), *call to result* a call returning + the 1 MiB string; *bytes received* is the one message as the client's link delivered it. + +| Payload | JSON | deflated (level 1) | deflate at the origin | inflate, sync (`nodeCompression`) | inflate, async (`DecompressionStream`) | +| --- | --- | --- | --- | --- | --- | +| 500 KB snapshot | 494.9 KiB | 137.4 KiB | 3.308 ms | 0.915 ms | 1.400 ms | +| 1 MiB result | 1024.0 KiB | 286.1 KiB | 6.856 ms | 1.757 ms | 3.113 ms | + +| End to end over a memory link (binary codec) | snapshot: bytes received | hold to live (mean of 20) | 1 MiB result: bytes received | call to result (mean of 20) | +| --- | --- | --- | --- | --- | +| uncompressed | 494.9 KiB | 1.197 ms | 1024.0 KiB | 1.939 ms | +| deflated, inflated synchronously (Node client) | 137.5 KiB | 5.632 ms | 286.1 KiB | 10.868 ms | +| deflated, inflated asynchronously (browser client) | 137.5 KiB | 6.356 ms | 286.1 KiB | 12.198 ms | + +Rerun after the review fixes (inflate only after admission by the part's owner, a tagged adapter result, byte counts +for the threshold): the same within noise (the asynchronous inflate of the 500 KB part varies between 1.4 and 2.0 ms +from run to run). + +Observations: + +- Level 1 cuts prose-like JSON to 28% (494.9 to 137.4 KiB, 1 MiB to 286 KiB) at about 6.7 ms per MiB deflated at the + origin and 1.8 ms per MiB inflated synchronously; `DecompressionStream` takes about twice as long as zlib's + synchronous inflate (3.1 against 1.8 ms per MiB) and adds its own turns, which the in-order holding of the link's + later messages absorbs. +- On a memory link (no transport time) compression only costs: hold to live 1.2 → 5.6 ms, the 1 MiB call 2.0 → + 10.9 ms. It pays where bytes take time: at 10 MB/s a 500 KB snapshot takes about 50 ms uncompressed and 14 ms + compressed (plus 4.3 ms of deflate and inflate); at 1 MB/s (a phone) 500 ms against 140 ms. On IPC to a worker + (gigabytes per second) it does not pay for itself on that hop: it pays on the tab's WebSocket, where the relay + forwards the worker's deflated part as it is (a test counts no inflate at the relay). +- Parts at or below 16 KiB (UTF-8 bytes) are never compressed (a state change, most call arguments), nor is a part + that deflating does not make smaller (sent as the bytes encoded for the attempt, never encoded twice), nor hold + params, nor call arguments sent up a parent link: the cost per message of the slices 3a-4a benchmarks is unchanged. The binary codec's part prefix (length doubled, plus one when deflated) costs + a byte more for parts between 64 and 127 bytes. + +## Slice 6a: the store + +`test/next/bench/store.ts` (run by hand: `node --conditions=source --experimental-strip-types --expose-gc +test/next/bench/store.ts` from `packages/chord`): Node 24.4.1, `chord/sqlite/node` on a fresh SQLite file per burst +for both sides (the same pragmas and file system), 7 bursts of 1,000 commits after a warm-up burst; per commit, the +median and p95 of single commits and the burst's mean. + +The gate (decision "The commit-step benchmark"): the store without outbox no slower than Durable's `SqliteStorage`. + +| Case | Durable | Chord store | Chord store + outbox | +| --- | --- | --- | --- | +| (a) one document delta per commit (step level) | 23.7 / 31.4 / 32.9 µs | 13.4 / 18.7 / 20.9 µs | 16.6 / 29.2 / 26.7 µs | +| (b) one table `put` (Durable: one entry append) | 19.3 / 27.1 / 29.5 µs | 13.2 / 23.6 / 20.6 µs | 17.5 / 28.1 / 26.3 µs | +| (c) whole transactions back to back, no watch (Durable `Session`, Chord host) | 35.0 / 61.1 / 44.4 µs | 18.1 / 27.2 / 25.7 µs | 20.3 / 38.8 / 30.7 µs | +| (c) the same with a watched document | 34.1 / 55.4 / 42.4 µs | 20.6 / 26.3 / 27.1 µs | 20.2 / 39.0 / 29.9 µs | +| (d) one in-memory document per transaction (no store) | | 2.5 / 2.9 / 2.7 µs | | +| (d) the same, watched | | 2.6 / 4.4 / 2.7 µs | | + +Every gate case passes. The outbox adds 3-5 µs per commit (one `INSERT` of the commit's writes as JSON). The heap after +GC at the end of the whole script is about 3 MiB above its start (the modules and statement caches loaded on the way). + +In-memory transactions against HEAD (before slice 6a), after the second fix round (`Transaction.draft` returns its +draft's promise as it is: an in-memory `change` settles in 5 microtask turns, HEAD 6, a test guards it): one `change` +per transaction through a host, 15 bursts of 1,000 per process with a timer around each `change`, HEAD's source +extracted beside this checkout, the same script on both, alternating processes pinned to one CPU (`taskset -c 4`), +three runs; median / p95 / burst mean: + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched document | 2.52-2.55 / 4.96-5.02 / 5.41-5.44 µs | 2.56-2.60 / 5.02-5.06 / 5.69-5.71 µs | +| watched document | 2.96-3.07 / 6.44-6.54 / 6.81-6.90 µs | 2.96-3.01 / 6.35-6.42 / 6.88-6.95 µs | +| watched derive through a reference | 3.14-3.29 / 6.77-6.93 / 8.38-8.50 µs | 3.06-3.08 / 6.49-6.53 / 7.52-7.54 µs | + +What the differences are (the second review's measurements, Opus): + +- The change path itself is at HEAD's level or below. Without the per-operation timer, a tight loop of 200,000 changes + costs 3.67-3.69 µs against HEAD's 3.70-3.78 µs (and the loop of the timed harness 4.85-5.06 µs against + 5.15-5.24 µs). Allocation per change is lower (7,703 B against 8,035 B, measured with a semi-space of 128 MB so no + scavenge lands in a burst), and so are the collections (40 hosts of 2,000 changes: 34 scavenges against 38, 2 + mark-compacts on both). +- The median difference of the timed harness (+1-2% here, up to +3-7% elsewhere) comes from the per-operation timer: + it stays with zero collections in the bursts and disappears without the timer, so it is harness-sensitive (code + layout, inline caches), not work in the change path. +- The burst means differ by where a mark-compact lands: the worst burst per process coincides with a mark-compact over + a larger old space (67 MB against 45 MB before the collection, about 18 MB after on both; a pause of 5.2-7.1 ms + against 3.0-3.7 ms), which is heap-growth timing, not extra garbage. Host setup is about 3% above HEAD (21.9-23.2 µs + against 21.1-21.9 µs per host, create, load and get; close equal), which the burst means include too. + +Calls and references (`taskset -c 4`, five alternating pairs of processes; one host, 21 bursts of 3,000 operations +after 3 warm-up bursts, a timer around each; reads: 200,000 per burst, burst means). Every call admits its handler to +the drain and its cleanup (call count, `ctx.get` references) in one map entry with one reaction: 19 promises per call +against HEAD's 20, a test guards it. A reference's state members are its object's member sources (one per object and +member, re-bound when a new attempt goes live): a watch through a reference gets the member's frame passed on as it is +(one call more per frame), a read goes through one getter more. + +| Case | HEAD | now | +| --- | --- | --- | +| local call: median / p95 / burst median / burst p95 | 1.49-1.51 / 1.93-1.99 / 1.64-1.67 / 2.51-2.75 µs | 1.47-1.48 / 1.93-1.96 / 1.59-1.61 / 2.40-2.47 µs | +| `change` under a watch through a reference, derive member: median / p95; frames per second | 2.80-2.92 / 4.64-4.80 µs; 268,000-282,000 | 2.76-2.87 / 4.75-4.84 µs; 278,000-288,000 | +| the same, document member | 2.68-2.79 / 4.58-4.79 µs; 285,000-306,000 | 2.67-2.76 / 4.57-4.79 µs; 285,000-306,000 | +| `ref.member.value`, derive member | 9.24-10.82 ns | 8.53-8.57 ns | +| `ref.member.value`, document member | 10.02-10.10 ns | 10.30-10.32 ns | + +Calls are 1-3% faster than HEAD (bursts' p95 10% lower), watches through a reference at HEAD's level (p95 +1-2% for a +derive member, more frames per second), a derive member's read faster; a document member's read costs 0.25 ns more +(the member source's getter in front of the slot's). + +A local document member re-exposed by a remote object and held over a memory link (one `change` and its state message +per operation; Astra's third review harness, `taskset -c 4`, alternating pairs, 41 bursts of 10,000 after 3 warm-up +bursts): a member source passes its subscribers (holds, member sources re-exposing it) on to the member it follows, so +their frames take no extra hop. Burst median 7.80-8.38 µs against HEAD's 7.82-8.35 µs, burst p95 8.84-9.44 against +9.52-10.06 µs, median per operation 6.76-7.34 against 6.82-7.29 µs (before the pass-through, one run: 8.51 / 13.05 µs +against 7.99 / 10.36 µs). Host setup (+2-4%) and a document member's read through a reference (+0.25 ns) are accepted +as they are. + +After Delta's tracker stopped allocating a `WeakRef` per change (pi a92283056, merged as 95ec93a08), against HEAD +95ec93a08 (which has that change too) extracted at a path as long as this checkout's (44 characters). Sol's tight +harness without a per-operation timer (`taskset -c 4`, five alternating pairs of processes, 31 bursts of 10,000 after +3 warm-up bursts; medians of the burst medians, ranges in parentheses): + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched `change` | 1.96 µs | 1.96 µs (0.0%) | +| `change` under a watch of the document | 2.24 µs | 2.19 µs (-2.1%) | +| `change` under a watch through a reference | 2.23 µs | 2.21 µs (-0.8%) | +| `change` under a watch of a derive through a reference | 2.33 µs | 2.26 µs (-2.9%) | +| local call | 1.36 µs | 1.33 µs (-1.9%) | +| a document member forwarded into a remote object, held over a memory link | 6.60 µs (6.54-6.63) | 6.52 µs (6.48-6.59) (-1.2%) | +| the same, yielding a macrotask every 1,000 operations | 6.67 µs (6.57-7.02) | 6.54 µs (6.42-6.79) (-1.9%) | + +Collections over one forwarded-link run (`--trace-gc-nvp`, 340,000 operations, two processes each; scavenges and +promotion were the same in both): + +| Run | Promoted | Scavenges | Mark-compacts | GC pauses | +| --- | --- | --- | --- | --- | +| HEAD | 6 MB | 158 | 2 | 154 ms | +| now | 7 MB | 148 | 2 | 147-148 ms | +| HEAD, yielding every 1,000 | 6 MB | 186 | 2-3 | 149-162 ms | +| now, yielding every 1,000 | 7 MB | 178 | 2-3 | 145-149 ms | + +History: the fifth fix round's measurement, before the Delta change (failure calls once per episode, watches on +failed sources, keyed pins still loading). Sol's tight harness without a per-operation timer (`taskset -c 4`, five +alternating pairs of processes, 31 bursts of 10,000 after 3 warm-up bursts; burst medians, HEAD's source at +`/tmp/chord-head4`, this checkout at its own path): + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched `change` | 2.91 µs | 2.95 µs (+1.5%) | +| `change` under a watch of the document | 3.14 µs | 3.10 µs (-1.4%) | +| `change` under a watch through a reference | 3.10 µs | 3.08 µs (-0.4%) | +| `change` under a watch of a derive through a reference | 3.21 µs | 3.26 µs (+1.6%) | +| local call | 1.37 µs | 1.35 µs (-2.0%) | +| a document member forwarded into a remote object, held over a memory link | 7.80 µs | 8.32 µs (+6.6%) | +| the same, `--no-allocation-site-pretenuring` | 8.12 µs | 8.04 µs (-1.0%) | +| the same, yielding a macrotask every 1,000 operations | 7.17 µs | 8.61 µs (+20%) | +| the same, yielding, `--no-allocation-site-pretenuring` | 7.21 µs | 7.52 µs (+4.2%) | + +Allocation per operation, with a fixed semi-space of 256 MB so no scavenge lands in the measured 20,000 operations, is +below HEAD's in every case: 6,942 B against 7,317 B unwatched, 8,383-8,393 B against 8,764 B watched, 8,935-8,939 B +against 9,338 B for a derive, 8,405-8,413 B against 8,764 B through a reference, 2,885 B against 3,089 B over the +link. What stays after the link drains and a GC is equal (18.9 against 19.0 B per operation). + +The forwarded-link gap is not work or retention in 6a's code. It is a V8 decision that depends on heap layout: + +- With huge semi-spaces (no scavenges) the two take the same time (10.01 against 10.00 µs). Without pretenuring (the + flag above) they are equal too. +- An identical copy of this checkout's source runs at HEAD's level (7.1 µs, 7 MB promoted over the run) when its path + is at most 40 characters long (`/tmp/wexp/packages/chord/src`). From 44 characters on it flips: 46 MB promoted, 4 + mark-compacts instead of 2, about 9 µs. The repository's own path is 44 characters. HEAD's source does not flip at + any tested path length (28 to 62 characters). +- `--trace-deopt` shows the flip: V8 marks the allocation site of the delta tracker's per-change overlay context + (`createContext`, inlined into `beginChange` and `Transaction.#begin`) as tenured. From then on every change's context + is allocated in old space and dies there. +- The contexts survive scavenges because each change registers its context as a `WeakRef` target + (`TrackerImpl.#register`). A `WeakRef` keeps its target alive until the current job ends, and scavenges do not + clear it. So a site that allocates one context per change shows a high survival rate. +- Heap snapshots taken during and after the run are equal between the two trees, and no 6a object outlives its + change. +- pi a92283056 ("Delta's tracker holds its one open change without a WeakRef") removed the cause: the numbers above + it show the forwarded link at HEAD's level with 6-7 MB promoted and 2 mark-compacts on both. + +A retirement under a diamond of derives (`test/next/bench/derives.ts`: 40 derives in 13 layers of two derives joined +by a third, 2^13 paths from top to bottom, the bottom watched; one step retires and recreates the document in one +transaction): 9.8 µs median, 17.7 µs p95 per step with one visit per derive per batch; 349 µs without the visited set +(one visit per path). + +Binary `state` messages (`test/next/bench/links.ts`, against HEAD, alternating, pinned): an entry without retirements +costs no extra byte (165 / 27 / 23 B for one appended item, one changed field, one removed item, as at HEAD); encode +and decode are at HEAD's level (one changed field, 200,000 runs, best of 9: encode 1,167-1,203 ns against +1,172-1,275 ns, decode 692 against 685-705 ns); heap per encode and decode 1,592 / 1,631 / 1,363 B against 1,684 / +1,726 / 1,355 B; commit to frame within HEAD's range (0.112-0.166 ms against 0.111-0.164 ms). After the second fix +round (`taskset -c 4`, two runs each): commit to frame 0.086-0.087 / 0.097-0.099 / 0.065 ms against 0.084-0.089 / +0.098-0.099 / 0.064 ms, a streamed append 0.053 ms against 0.051-0.053 ms, the same bytes and heap. + +Other costs (one run each): + +| Cost | Result | +| --- | --- | +| 100 commits submitted at once: time waiting in the queue against the step itself | median wait 956 µs, step 11.3 µs | +| a query page of 100 rows in an index's order (10,000 rows, cursors) | 113 µs per page | +| one `updateWhere` over 10,000 rows | one step of 10.6 ms (every owner's commits wait behind it) | +| `put` with 10-character text cells / 10,000-character cells (2,000 rows, one commit) | 2.4 / 58.4 µs per row | +| a `rows` migration of 10,000 rows (100 chunks; `taskset -c 4`) | 69-71 ms in all, 7.0 µs per row; chunk commits: first 0.34 ms, last 0.21 ms, median 0.22 ms | + +Observations: + +- The commit step is cheaper than Durable's per commit: Chord keeps no generic entry log beside the documents engine + and its tables, and serializes writes before the queue (rule 8). +- Between chunks a migration yields one macrotask without the 1 ms clamp of `setTimeout(0)`: `setImmediate` where the + runtime has it, else a `MessageChannel` message (Workers, browsers). The same 10,000-row migration takes 191 ms with + `setTimeout(0)` (99 clamped yields) and 100 ms with messages in Node (0.27 ms per yield there, against 13 µs for + `setImmediate`). +- Late migration chunks cost what early ones do (paging forward by the last key keeps each chunk's read bounded); the + spec's estimate of about 15 µs per migrated row holds with room. +- A bulk write's step is the one place a single commit holds every owner's commits for milliseconds; the guide says so. + +## Slice 6b: Durable on the store, persisted jobs (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), SQLite files in `tmpdir()`. HEAD is `b2fcfee08`, extracted at a path of the same +length (`/home/badlogic/pi-6h`); runs alternate HEAD and the build. + +**Durable's storage benchmark** (`packages/durable/test/storage.bench.ts`, `TIMING_SCALE`), `StoreStorage` (outbox off) +against `SqliteStorage` in the same process, mean ms per operation over three runs (p75 where the mean is one outlier): + +| Case | SqliteStorage | StoreStorage | | +| --- | --- | --- | --- | +| exact entry lookup | 0.0030 | 0.0030 | 0% | +| entry page scan (100) | 0.0942 | 0.0940 | 0% | +| filtered task scan (50) | 0.0206 (p75 0.0200) | 0.0220 (p75 0.0207) | +7% mean, +3% p75 | +| exact document address among many | 0.0037 | 0.0037 | 0% | +| document replay tail (0) | 0.0089 (p75 0.0084) | 0.0092 (p75 0.0085) | +3% mean, +1% p75 | +| document replay tail (16 / 128 / 1024) | 0.0300 / 0.1726 / 1.224 | 0.0270 / 0.1522 / 1.135 | -10% / -12% / -7% | +| ancient / recent historical read | 0.1681 / 0.1611 | 0.1541 / 0.1513 | -8% / -6% | +| fork-depth history scan (100) | 0.1177 (p75 0.1156) | 0.1208 (p75 0.1159) | +3% mean, 0% p75 | +| fork-depth head lookup | 0.0361 | 0.0332 | -8% | +| commit one entry (20 samples) | 0.0439 (p75 0.0416) | 0.0601 (p75 0.0373) | +37% mean, -10% p75 | +| commit 100 entries | 0.743 | 0.620 | -17% | +| commit mixed entry/task/submission/document | 0.195 | 0.187 | -4% | +| reopen and first exact read | 0.294 | 0.443 | +51% | + +Warm micro-benchmarks behind the same cases (each read 20,000 times; commits 10,000 after 2,000 warm-up): page scan +91.2 vs 90.6 µs, task scan 19.9 vs 20.1 µs, fork-depth scan 114.1 vs 114.9 µs, head lookup 35.3 vs 32.9 µs, document +address 3.57 vs 3.68 µs, replay tail (0) 8.41 vs 8.44 µs; one entry per commit 29.6 vs 27.8 µs mean, 18.4 vs 17.1 µs +median. What it took, beyond the first build's compiled reads, point lookups without a bound `LIMIT`, version columns +selected only past version 1 and the counting read guard: + +- **Reads select only the record.** A scan selected an entry's `commit_seq` too (only `entry(id)` needs it): 7% of a + page scan. +- **No index for head markers on every entry.** `entries.by_head (conversation_id, has_head, id)` cost about 5.5 µs per + commit (a raw insert: 39.7 vs 45.2 µs); `SqliteStorage` has a partial index (`WHERE head IS NOT NULL`), which the + table vocabulary does not have. The head markers are an `entry_heads` table instead, written only with an entry that + has one: a commit without one writes no second index, and a head lookup reads the newest head's id, then the entry. +- **A document record is one JSON text.** `meta` keeps the whole Durable record, its creation seq included; a read + selects `meta` (and `retired_at` only at a point). +- **A store already at the newest schema opens with one read** (schema version, encoding and metadata row), without + the migrations' write transaction: 84 → 71 µs on a fresh connection. + +Commit one entry: the mean is one sample (Maglev compiling the commit path on the first fixtures: 130 and 386 µs at +fixtures 25 and 30 of the first store round of a 60-fixture replay, none in the second round, no GC or deopt traced); +the median and p75 are lower than `SqliteStorage`'s. Reopen: a fresh connection on a copied file, measured in parts +(`openNodeSqliteDatabase` is the same for both): `SqliteStorage.open` 42-60 µs; `openStore` 63-71 µs (the schema read, +the catalog with its JSON parse) plus `StoreStorage.open` 75 µs (declaring six tables: their SQL rendered, claims; the +catalog check of the table preparation; `max(record_ids.id)` for the allocator, as §2.1 asks; compiling the point +reads), first read +3 µs: about 0.1-0.15 ms once per open, cold code, not on any hot path. + +**Statements per commit** (cost guard in `packages/durable/test/store-storage.test.ts`): an entry append and a document +delta take 4 each (`SqliteStorage`: 5 and about 7). + +**Memory** (`packages/durable/test/storage-memory.ts`): heap after seeding 0.16 → 0.30 MiB (1k), 0.17 → 0.32 MiB (10k): +a fixed cost of the store (catalog, compiled statements, claims, the store's code), the same at both scales; heap after +reads 0.13 → 0.16-0.18 MiB. Disk 0.61 → 0.58 MiB (1k), 4.34 → 4.38 MiB (10k). + +**Chord's commit gate** (`test/next/bench/store.ts`), HEAD → build, two alternating runs each, medians: + +| Case | HEAD | build | +| --- | --- | --- | +| (a) one document delta per commit | 13.6-13.7 µs | 13.8-14.0 µs (+2%) | +| (b) one table put per commit | 12.8-13.3 µs | 13.0-13.2 µs | +| (c) whole transactions (no watch / watched) | 17.2-17.3 / 17.1-17.2 µs | 17.4-17.7 / 17.2-17.7 µs (+2%) | +| (d) one in-memory document (median / burst) | 2.1 / 2.9-3.0 µs | 2.1 / 4.9 µs | +| heap after GC | 2940 KiB | 2996 KiB | + +The revisions index is a partial index on bases (`(document_id, seq) WHERE kind = 'base'`): a delta writes no index +entry (the first build's `(document_id, kind, seq)` cost (a) 15%). (d)'s burst is not the in-memory path: run alone +(only section (d), three alternating runs) HEAD gives 4.9-5.1 µs and the build 4.9-5.0 µs per commit, and 40 bursts of +1,000 in a fresh process 2.17 vs 2.18-2.38 µs; in the full gate, HEAD's in-memory path runs warm from the persisted +sections before it, the build's does not share as much of their code (`StoredChange`'s variants). + +**Durable's conversation view benchmark** (`test/conversation-view-bench.ts`, `PI_DURABLE_BENCH_STORAGE`), two runs +each, `SqliteStorage` → `StoreStorage`: append one entry per commit 56.1-57.2 → 56.2-57.0 µs with one watch, 61.0-62.1 +→ 62.7-63.1 µs with four (+2%); bursts of 50 884-888 → 891 µs; `pi.live` only 62.7-63.0 → 65.1-66.6 µs (+4-6%); a +long-lived watch over 20,000 appends 0.83-0.85 → 0.84-0.85 s CPU; heaps 21.5 → 21.7 MiB. + +**Fix round 1** (2026-10-28, after the build reviews): `StoreStorage`'s reads take one promise besides the store's guard +(no `#call` wrapper; `close` waits for its commits, a read changes nothing), their request errors in a tagged result. +Warm micro-benchmark (20,000 runs, two runs each, `SqliteStorage` → `StoreStorage`): filtered task scan 19.8 → 19.8-20.1 +µs, `entry(conversation, id)` over the fork chain 23.1 → 21.9-22.3 µs. `storage.bench.ts` (two runs): task scan mean +0.0207 → 0.0217-0.0218 ms (p75 0.0200-0.0203 → 0.0204-0.0206), fork-depth scan 0.1177-0.1188 → 0.1210-0.1242 (p75 +0.1154-0.1167 → 0.1160-0.1193), the rest as above. +`conversation-view-bench` "resume 20 back", three runs: CPU 4.23-4.47 → 4.50-5.14 ms for a 3 ms scenario (one sample +each; the head lookup and scans under it are at parity or faster in the micro-benchmark). The persisted-job fixes +(one invocation per call of `run`, held state folded in a `PendingOps`): start 55.5 → 55.2 µs, immediate state change +21.3 → 21.2 µs, deferred 3.2 → 3.5 µs (the fold), in memory unchanged. + +**Persisted jobs** (`test/next/bench/jobs.ts`, 7 bursts of 500, medians): a start (resolved after its commit) 8.4 µs in +memory, 55.5 µs persisted (two commits: the record and state created, then the reservation); a state change inside +`run` 2.7 µs in memory, 21.3 µs persisted (one delta commit, as the gate's (c)), 3.2 µs deferred (stored with the +outcome). + +## Slice 8: registries and `host.root` (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `391e83fb1`, extracted at a path of the same length +(`/home/badlogic/pi-h8`); the build was measured in the worktree and, where the worktree itself skewed a result (below), +in an extract of the same sources at `/home/badlogic/pi-x8`. + +**Paths under every object** (`test/next/bench/calls.ts`, new: runs on HEAD as it is), median per operation over nine +bursts, heap allocated per operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.80-2.81 µs, 6,894 B | 2.81-2.83 µs, 6,894 B | +| commit through a watched derive (`batch.compute`) | 10.34-10.43 µs, 15,312 B | 10.46-10.51 µs, 15,310 B | +| activation churn: get, live, release (an object with params) | 57.5-59.3 µs, 43,696-43,717 B | 58.1-58.9 µs, 43,782-43,787 B | +| retained per live object (10,000 kept) | 6,586 B | 6,602 B | + +The attempt keeps its registry reads, contributions, wake signals and writables in one record made by the first such +declaration, and iterates nothing when there is none: the first build allocated 1,020 B more per activation (six +arrays per attempt, an array iterator per `for … of` over an empty list, a copy from `splice`) and retained 72 B more +per live object; now +70 B allocated and +16 B retained (one field on the attempt, one on the object). + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-9.8 → 9.5-9.9 µs median; its +pooled p95 16.5-17.1 → 18.4-20.8 µs comes from the run in which V8 tiers the commit path up (per run, after it: HEAD +10.7-11.3 µs, build 10.1-11.3 µs; the first measured runs 15.0-25.6 µs on either side, in another run each), the same +with HEAD's `batch.ts` and `derive.ts` in the build. The store gate (`store.ts`, three runs): every median within +0.4 µs (Chord store 13.0-13.9 → 13.0-14.2 µs, Chord host 17.3-18.2 → 17.3-17.8 µs, in-memory document 2.1 → 2.2 µs), +heap 3,498 → 3,144 KiB. Jobs (`jobs.ts`): start 8.4 → 8.5 µs in memory, 55.4 → 56.2 µs persisted; state change 2.7 → +2.7, 21.3 → 21.8, 3.4 → 3.5 µs. Links (`links.ts`, three runs): wire sizes, encodes, decodes and heap per message +unchanged; its round trips are one cold burst each and vary as much between HEAD runs (commit to frame, text codec, +0.055-0.076 ms). A warmed remote call (2,000 calls per burst, nine bursts, a memory link): binary 7.16-7.24 → 7.17-7.29 +µs, text 7.66-7.72 → 7.70-7.83 µs in the extracts; the worktree itself runs the same sources at 7.8-7.9 µs (binary), +an effect of the tree, not the code. + +**Registries and writables** (`test/next/bench/registries.ts`, new), median over nine bursts: `writable.set` delivered +to one watch 0.81-0.91 µs (2.9 KB allocated: the batch and the watch's call); a set of the same value 0.008 µs, 0 B +(one comparison: its batch work is a method of its own, so `set` allocates no closure context); one of 10 contributions +changed while the registry is watched 0.74-0.77 µs, of 100 1.52-1.59 µs (the table built again over every entry, one +compose per set: the counted `chord.registry.composes` is 1.00 per set). + +**Fix round 1** (2026-10-28), HEAD extract → build extract (`/home/badlogic/pi-x8`), CPU 6. `calls.ts`, two runs: local +method call 2.80 → 2.80-2.81 µs, 6,894 B both; commit through a watched derive 10.51-10.65 → 10.72 µs; activation churn +57.9-59.0 → 59.3-60.0 µs, 43,714 → 43,769-43,787 B. The derive diamond (five runs): median 9.4-9.8 → 9.5 µs, pooled p95 +16.4-16.7 → 19.0-19.2 µs. Per run, both have one slow run while V8 tiers the commit path up (HEAD its second, median/p95 +15.5/22.6-22.9 µs; the build its first, 16.4/26.7-27.2 µs) and equal runs after it (HEAD 8.8-9.6/9.6-11.2 µs, build +8.9-9.8/9.7-10.6 µs). `batch.compute` is not the cause: HEAD's `batch.ts` and `derive.ts` in the build give the same +p95, and swapping host files moves it without a pattern (code layout, not work on the path). `registries.ts`: +`writable.set` 0.87 µs, the same value 0.011-0.016 µs and 0 B (the seal check now comes first: a set of an ended +attempt's writable throws whatever its value); one of 10 contributions set 0.71-0.78 µs, of 100 1.43-1.61 µs (one run +in six 3.4 µs on either variant tried: JIT). + +## Slice 7a: the dependency model (2026-10-29) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `babe30a7e`, extracted at a path of the same length +(`/home/badlogic/pi-h7`); the build in the worktree (`/home/badlogic/pi-7a`). + +**Paths under every object** (`test/next/bench/calls.ts`), median per operation over nine bursts, heap allocated per +operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.81-2.82 µs, 6,894 B | 2.80-2.82 µs, 6,894 B | +| commit through a watched derive | 10.43-10.70 µs, 15,310 B | 10.38-10.53 µs, 15,310 B | +| activation churn: get, live, release (an object with params and one dependency) | 58.5-59.9 µs, 43,716-43,719 B | 53.7-55.4 µs, 41,540-41,559 B | +| retained per live object with one dependency (10,000 kept, an ad-hoc probe) | 7,398-7,406 B | 6,931-6,936 B | + +An activation whose dependencies are live at the check does not park: one check of its bindings, then its own steps a +microtask later (as HEAD's wait did). The first build parked every attempt with a dependency (a waiter list, a +`parked_long` record, a microtask per wake even outside parking) and cost 62.0-63.6 µs and 580 B more per activation; +now a wake outside parking is a flag, the first-outcome promise is made only for whoever waits (the start barrier), +and an object keeps no `Set` of waits and no settlement promise per attempt (R18's), hence the lower numbers. + +**Links** (`links.ts`, three runs each): wire sizes, encodes, decodes, heap per message and retained heap per hold +(176.0 → 175.4-176.0 KiB) unchanged. Hold to live over two hops (mean of 100), binary: 1,000-item list 0.781-0.817 → +0.807-0.811 ms, 500 KB document 1.250-1.278 → 1.289-1.310 ms (+1-3%: the hold counted at its arrival, the index +check of the publication); text 1.545-1.580 → 1.521-1.573 and 3.299-3.379 → 3.317-3.393 ms. Commits to frames, +streamed appends, call round trips and reconnects within the runs' spread. + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-10.1 → 9.5-9.6 µs median, +p95 19.3-19.9 → 19.3-19.4 µs. Registries (`registries.ts`, three runs): `writable.set` 0.82-0.97 → 0.77-0.87 µs; one of +10 contributions set 0.72-2.71 → 0.70-1.06 µs; of 100 1.50-3.46 → 1.44-3.40 µs (one slow run per side: JIT); +allocation within 20 B. A registry's last good composition keeps the list of contributions it was built from: the +list is replaced (not spliced) when a contributor comes or goes, so a composition costs no copy. + +**Fix round 1** (2026-10-29: revocation over paths, retry operations on the wire, the provider's own host-address +check for same-host remote requests, mapped values' provenance), `calls.ts`, three alternating runs, HEAD → build: +local call 2.81 → 2.79-2.82 µs (6,894 B both), watched derive commit 10.45-10.70 → 10.35-10.47 µs, activation churn +58.6-59.8 → 53.9-54.2 µs (43,716-43,720 → 41,559 B); retained per live object with one dependency 7,399-7,405 → +6,930-6,938 B. The address check runs only for a remote type whose consumer and provider `hostAddress` differ; a +retry's operation id is made per `ref.retry()`, and an object keeps its handled ids only once it was retried. + +## Slice 7b: reload in the kernel (2026-10-29) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `0c3675e64`, extracted at a path of the same length +(`/home/badlogic/pi-hb`); the build in the worktree (`/home/badlogic/pi-7b`). + +**Paths under every object** (`test/next/bench/calls.ts`), median per operation over nine bursts, heap allocated per +operation, alternating runs (the last two after the fixes below), HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.80-2.83 µs, 6,894 B | 2.67-2.70 µs, 6,757 B | +| commit through a watched derive | 10.38-10.69 µs, 15,309 B | 10.22-10.39 µs, 15,312 B | +| activation churn: get, live, release (an object with params) | 53.6-55.3 µs, 41,539-41,560 B | 54.2-55.6 µs, 41,606-41,630 B | +| retained per live object with params and one dependency (10,000 kept, an ad-hoc probe, median of 5) | 8,696-8,700 B | 8,761 B | + +The first build cost 60.4-62.7 µs and 2,480 B more per activation: every attempt made an `AbortController` for its +routines and two maps (routine calls, job invocations) whether it had any. Now they are made when routines start or a +run is admitted, and the first dependency check's answer is a shared constant unless someone asks for it; what a reload +did to an attempt is one record made by the first such step. What remains: a few fields on the object (its params key) +and the attempt (its routines, its reload record, its first check), +46-70 B allocated and +61-65 B retained (0.7%). +A local call allocates 137 B less: the method path is split (`#invoke`) so the current version's call builds no +mapping closure. + +**Links** (`links.ts`, two runs each): wire sizes, encodes, decodes and heap per message unchanged. Hold to live over +two hops (mean of 100), binary: 1,000-item list 0.780-0.794 → 0.797-0.821 ms (+1-4%), 500 KB document 1.246-1.275 → +1.270-1.275 ms; commits to frames and streamed appends within the runs' spread. Retained per hold of 10: 175.8 → 176.2-176.5 +KiB (the hold keeps its request, to be placed again after a reload); the single-hold row 176.0 → 180.2 KiB is one +fixed amount (~4 KiB, code of the new paths), divided by ten in the other row. + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, two runs) 9.5-9.6 → 9.6 µs median, p95 +16.4-19.3 → 14.1-14.3 µs. Registries (`registries.ts`, two runs): `writable.set` 0.94-0.96 → 0.77-0.86 µs; one of 10 +contributions set 0.77-1.05 → 0.69-0.72 µs; of 100 1.57-1.58 → 1.54-3.30 µs (one slow run: JIT, as in 7a). Jobs +(`jobs.ts`, two runs): start 8.6-8.7 → 8.5 µs in memory, 55.3-56.1 → 55.3-55.6 µs persisted; state changes within 0.1 +µs. The store (`store.ts`): every median within 0.3 µs. The relay (`relay.ts`, three runs): tab B alone 0.014 ms both; +10,000 calls at once 530-723 → 491-785 ms (the spread of either side). + +**Reload** (`test/next/bench/reload.ts`, new): 20 plugins, 1,000 live objects (with params, a document, a derive and a +method each), one plugin reloaded 20 times; medians after two warm-up reloads, two runs: + +| Case | reload resolves | all live again | heap after 20 reloads, above after the first | +| --- | --- | --- | --- | +| swap, 50 of the 1,000 objects in the reloaded plugin | 1.58-1.59 ms | 1.64-1.65 ms | 332-338 KiB | +| swap, all 1,000 in it | 35.5-36.3 ms | 36.6-37.2 ms | 191-192 KiB | +| restart (a native each), 50 of 1,000 | 1.20-1.54 ms | 1.90-2.05 ms | 105-106 KiB | +| restart, all 1,000 | 20.0-20.2 ms | 35.8-40.8 ms | 57-58 KiB | + +A swap costs about 36 µs per object (its staging record, the new attempt's activation, the hand-over of its old one), +a restart about 20 µs until the reload resolves (staging and the start of each fate; the new attempts activate after +the drains). The heap after reloads levels off (swap, 50 objects: 332 KiB after 20, 437 KiB after 80, 550 KiB after +300 reloads): compiled code and feedback of the new facets, not objects (the GC tests collect old attempts, removed +objects, providers and facets). + +**Fix round 1**, CPU 6, alternating against HEAD. `calls.ts`, three runs: local call 2.80-2.82 → 2.66-2.68 µs +(6,894 → 6,757 B); watched derive commit 10.42-10.66 → 10.31-10.47 µs; activation churn 53.7-55.8 → 54.3-55.9 µs, +41,540-41,560 → 41,566-41,584 B (an `?? []` per end, found here, is gone: the round's first measurement allocated +41,698 B); retained per live object with one dependency 8,699-8,700 → 8,768 B. Links (a review measured binary +commit-to-frame +19% and small calls +6% on another CPU): `links.ts`, two runs, binary commit to frame +0.085/0.097-0.099/0.063-0.065 → 0.087/0.097/0.064 ms (append/change/removal), small call 0.038-0.039 → 0.035 ms. A +warmed ad-hoc probe (binary memory link, 2,000 calls and 1,000 commits per burst) measures no regression: call round +trip 9.8-10.2 → 9.9-10.1 µs, commit to frame 60.6-65.1 → 61.0-61.6 µs. Reload benchmark after the round: swap 50 / +1,000 objects 1.50 / 35.9 ms, restart 1.67 / 21.4 ms until the reload resolves (2.12 / 38.1 ms until all are live); +heap after 20 reloads 339 / 193 / 104 / 68 KiB. + +**Fix round 2**, alternating against HEAD. Stock `links.ts` on CPU 22, six rounds each before and after the fix +(medians, binary / text): before, small call 0.062 → 0.068 / 0.058 → 0.064 ms (+10%), commit to frame within 1% +(append 0.156 → 0.156, changed field 0.165 → 0.162, removal 0.120 → 0.119 ms binary). Cut-down copies located it: +calls alone, or after the snapshot, commit and stream sections, or after the hold-memory trials alone, measured +equal; only the full sequence showed the gap, and a CPU profile of its calls section put the difference in the +remote call path (`Hold.call` with the attempt's call inlined, about 6.7 µs per call). The cause was `Attempt.call` +delegating the invocation to a helper method shared with the older-version path; one body for every version, and +no `async` (a local call allocated about 250 B more as an async function), fixed it. After: small call 0.063 → +0.065 / 0.058 → 0.058 ms, append 0.156 → 0.157, changed field 0.164 → 0.163, removal 0.119 → 0.120 ms, hold to live +(1,000 items) 1.037 → 1.023 ms. `calls.ts` on CPU 6: local call 2.89-2.92 → 2.77-2.85 µs (6,894 → 6,660 B), watched +derive commit 10.44-10.58 → 10.65-10.81 µs (15,309 → 14,969 B), activation churn 54.8-55.1 → 54.6-54.7 µs (41,560 → +41,592 B); retained per live object with one dependency 8,700 → 8,777 B. Reload benchmark: swap 50 / 1,000 objects +1.84 / 33.7 ms, restart 1.31 / 22.3 ms until the reload resolves (2.18 / 41.0 ms all live); heap after 20 reloads +323 / 200 / 104 / 70 KiB. + +**Fix round 3: the commit-to-frame gap is V8's young generation, not frame work.** Astra (CPU 22, five pairs) measured +binary append commit-to-frame 132 → 161 µs, changed field 148 → 168, removal 112 → 121 on the stock `links.ts`. +- *Path length.* The HEAD copy `/home/badlogic/pi-hb` measured append 0.155 ms; Astra's `/tmp/s7b-hd-astra`, with + identical sources, 0.129 ms. A copy of the same tree at a path of Astra's length (17 characters) measured 0.130 ms, + at a longer one 0.155 ms: on the same code the result depends on the checkout path alone. Compare copies at paths of + equal length. +- *Stock numbers, equal paths* (`/tmp/s7b-hd-astra` vs a fresh build copy `/tmp/s7b-xx-astra`, CPU 22, three + alternating runs, means): append 0.128-0.129 → 0.155-0.156 ms binary, 0.106-0.107 → 0.116-0.117 text; changed field + 0.144-0.145 → 0.161-0.162 binary, 0.097-0.098 → 0.107-0.110 text; small call 0.061-0.063 → 0.065-0.066 binary, + 0.058-0.059 → 0.058-0.059 text. +- *Same work.* Heap allocated per commit is equal (191.9 vs 191.4 KB including the codec rows); so are the telemetry + records per commit (5.012, the same names: two validations, two link checks, one encode). +- *The difference is scavenges.* In the timed append loop HEAD ran 5 scavenges costing 3.6 ms in total, the build 3 + costing 21 ms (changed field: 2 for 2.7 ms vs 1 for 10.9 ms). At the loop's start the young generation was 33.5 MB + in HEAD and 67 MB in the build (134 MB by the changed-field loop): V8 grows it from the bytes surviving its + collections, and the build crossed that threshold a few scavenges earlier. Over the whole run the GC work is equal + (scavenge time 1,620 vs 1,605 ms, survived bytes 471 vs 480 MB, 285 vs 284 scavenges); where the expensive + scavenges fall relative to the timed loops differs. Tracing the GC (`--trace-gc-nvp`) moves HEAD into the build's + pattern by itself. +- *With the young generation pinned* the two are equal: means 0.139/0.139 ms (append), 0.145/0.144 (changed field), + 0.116/0.115 (removal), 0.065/0.064 (small call), HEAD/build; per-commit medians and p90s too (three runs, warmed + script): append 0.085-0.086 / 0.085-0.086 ms median, 0.101 / 0.101-0.103 p90; changed field 0.073 / 0.072-0.073, + 0.083-0.084 / 0.083; removal 0.060-0.061 / 0.060-0.061, 0.073 / 0.072-0.073 (binary; text alike, within 1 µs). +- *The extra live heap is code.* Build minus HEAD after a full collection: +249 KB right after loading the modules + (old space +229 KB, before any object), +290 KB after a snapshot section, +296 KB after 1,000 appends, +281 KB after + 1,000 edits, +345 KB after 70 hold trials (old space +259 KB, the rest bytecode and code as more paths compile). It + does not grow with commits or holds beyond the retained-per-hold and per-object figures above. + +So `links.ts` changed: each commit section runs 1,000 untimed commits first (`WARM_UP_COMMITS`; the removal section +starts with 1,000 more notes for them), and reports commit-to-frame as mean, median and p90 of the 1,000 timed commits. +The warm-up stays because the timed loop then runs nearer the young generation's steady size: the warmed medians +match the pinned ones (append 0.082 vs 0.085 ms, changed field 0.072 vs 0.073, removal 0.060 vs 0.060), and HEAD and +build agree in five of six sections, against two of six without it (run-to-run spread is about 1 µs either way). The +mean still carries the scavenges that fall in the loop. After the change, equal paths, CPU 22, three alternating runs, +HEAD → build (mean; median; p90, ms): + +| Section | binary | text | +| --- | --- | --- | +| append | 0.106-0.108 → 0.142-0.144; 0.081-0.083 → 0.090-0.091; 0.093-0.096 → 0.125-0.127 | 0.095-0.119 → 0.121-0.125; 0.078-0.080 → 0.080-0.082; 0.090 → 0.094-0.095 | +| changed field | 0.084-0.087 → 0.098-0.099; 0.071-0.075 → 0.072-0.073; 0.081-0.084 → 0.081-0.082 | 0.086-0.090 → 0.099-0.101; 0.067-0.071 → 0.070-0.071; 0.079-0.082 → 0.082-0.084 | +| removal | 0.103-0.104 → 0.107-0.109; 0.060-0.061 → 0.060-0.061; 0.071-0.073 → 0.072-0.073 | 0.100-0.102 → 0.077-0.103; 0.058-0.060 → 0.059; 0.067-0.069 → 0.067-0.070 | +| small call | 0.052-0.058 → 0.053-0.054 | 0.054-0.065 → 0.059-0.069 | + +The binary append differs unpinned (median +11%, p90 +31%) and is equal pinned: the young generation's size in that +section, not per-append work. To compare commit-to-frame across two copies, run both at paths of equal length, pinned +to one CPU, alternating, with the young generation pinned: + + taskset -c 22 node --min-semi-space-size=16 --max-semi-space-size=16 --conditions=source --expose-gc test/next/bench/links.ts + +Fix round 3 hot paths, `/tmp/s7b-hd-astra` → a build copy at a path of equal length, CPU 22, three alternating runs +(`calls.ts`): local call 3.69-3.71 → 3.56-3.63 µs (6,894 → 6,660 B), watched derive commit 17.39-17.50 → 17.41-17.53 +µs (15,310 → 14,967 B), activation churn 104.2-104.8 → 107.6-108.3 µs (+3%; 41,526 → 41,426 B); retained per live +object with one dependency 8,700 → 8,585 B (one table per version replaced the current version's separate member +map). Reload benchmark, same CPU: swap 50 / 1,000 objects 2.88 / 51.9 ms, restart 2.49 / 35.7 ms until the reload +resolves (3.22 / 65.7 ms all live); heap after 20 reloads 345 / 188 / 119 / 85 KiB. + +Sol's independent runs (CPU 24, five pairs of the stock `links.ts`) show the same commit-to-frame gap (binary append +135 → 162 µs, changed field 145 → 168, text 109 → 119 and 100 → 108): the young-generation effect above. Sol's +isolated round trips (`sol-isolated-links.ts`, nine bursts each, no telemetry) measured a binary call at 44.83 → 47.20 +µs (+5.3%) on the round-2 code. On this round's code, `/tmp/s7b-hd-astra` → a build copy at a path of equal length, +CPU 24, five alternating pairs, medians: binary call 47.23 → 45.74 µs (-3.2%; HEAD's samples are bimodal, 44.3 or +46.9-47.9), binary commit 92.68 → 93.28 (+0.6%), text call 39.02 → 39.76 (+1.9%), text commit 82.86 → 85.28 (+2.9%); +with the young generation pinned: binary call 45.45 → 43.85 (-3.5%), binary commit 92.73 → 93.06, text call 39.97 → +39.65, text commit 83.65 → 85.46 (+2.2%). + +**Rework: references are the only indirection** (decision of the same name). `/tmp/s7b-hd-astra` (HEAD) → a build +copy at a path of equal length, CPU 22, three alternating runs. `calls.ts`: local call 3.72-4.03 → 3.62-3.77 µs +(6,894 → 6,660 B); watched derive commit 17.46-17.92 → 17.53-18.40 µs (15,310 → 14,900 B); activation churn +104.7-107.9 → 114.7-118.3 µs (+9.6%, accepted by the owner, 2026-10-29; 41,507-41,526 → 41,473-41,501 B). The churn's object derives over its dependency's +member: that derive now attaches through its own reference's member (one `ReferenceSource` made, bound and ended per +cycle) where HEAD shared the provider's one member source; the allocation is HEAD's, the time is the new node's (a +copy of round 6, before the rework, measured 55.2 µs against HEAD's 53.5-54.6 µs and the rework's 58.4 µs on CPU 6). +Retained per live object with one dependency (an ad-hoc probe, 2,000 objects, median of five): 6,924 → 6,738 B; a +first cut made a source for every member at the binding and allocated four sets per source (8,627 B): sources are made +on first use or when a reference that had values detaches, and a source's watch, subscriber and pass-through sets are +made by their first user. `links.ts` (warmed, mean / median / p90 in ms), binary: append 0.107-0.113 / 0.082-0.083 / +0.095-0.097 → 0.141-0.142 / 0.089-0.090 / 0.123-0.124 (the young-generation effect above), changed field 0.084-0.086 / +0.071-0.072 / 0.081-0.082 → 0.098-0.099 / 0.072-0.073 / 0.081-0.082, removal 0.102-0.108 / 0.060 / 0.071-0.073 → +0.107-0.109 / 0.060-0.061 / 0.072; with the young generation pinned, append 0.140-0.143 / 0.086 / 0.100-0.102 → +0.143-0.147 / 0.085-0.087 / 0.101-0.103, changed field 0.091-0.092 / 0.072-0.073 / 0.082-0.084 → 0.091-0.092 / +0.073-0.074 / 0.082-0.084, removal 0.108-0.111 / 0.061-0.062 / 0.072-0.075 → 0.108-0.110 / 0.060-0.062 / 0.072-0.076; +text alike within 3 µs. Small call 0.058-0.060 → 0.053-0.056 ms. Retained per hold (both ends): 175.4-176.0 → +176.4-177.0 KiB with one hold, 175.8 → 176.4 KiB with ten. + +The 1 MiB call row (final review): one burst of 50 calls read 0.389 → 1.088 ms (binary, young generation pinned) in +one review run. Not per-call work: a probe with the row's setup and a collected heap before each of 20 bursts of 50 +(CPU 22, three alternating pairs) measured, per call without the collector's pauses, 0.27-0.31 → 0.30-0.31 ms binary +and 0.31-0.36 → 0.33-0.36 ms text, heap 23,025 → 22,707 B binary and 23,964 → 23,650 B text. A burst's time is about +0.3 ms per call plus 0.2-0.27 ms per call when a full collection falls in it (the arguments fill memory outside the +heap); one burst of 50 sees one or none, in either tree. `links.ts` now collects the heap before each of 20 bursts and +reports the median and p90; three alternating pairs of the new row, stock, binary median / p90 0.386-0.455 / +0.409-0.473 → 0.279-0.357 / 0.306-0.394, text 0.344-0.395 / 0.510-0.566 → 0.363-0.500 / 0.522-0.621; pinned, +binary 0.689-0.910 / 0.738-1.050 → 0.695-0.882 / 0.829-0.992, text 0.415-0.896 / 0.958-1.049 → 0.534-0.864 / +1.042-1.175. The row still moves by run with the collector's phase; compare it across several runs, not one. +The small call, measured apart (ten bursts of 1,000 after 3,000 warm-up calls, three alternating pairs): stock, binary +29.5-30.6 → 27.6-27.9 µs and text 25.1-25.9 → 27.5-28.3 µs; with the young generation pinned, binary 29.7-30.7 → +30.3-30.7 µs and text 23.6-23.7 → 23.9-24.1 µs. The stock text gap is the young generation's sizing, as for commit to +frame above (pinned, the two match); the snapshot before the final review's fixes measures the same. + +## Simplification audit (2026-10-30) + +HEAD `4b535d57e` → the audit build (A1, A2, A5, A9-A12), copies at paths of equal length, CPU 6, alternating. +`calls.ts`, three runs: local call 2.70-2.73 → 2.69-2.73 µs (6,660 B both), watched derive commit 10.41-10.75 → +10.42-10.74 µs (14,899 B both), activation churn 59.0-60.9 → 59.1-61.4 µs (41,473-41,482 → 41,478 B). Retained per live +object with one dependency (10,000 kept) 6,745-6,754 → 6,745-6,752 B; a first cut of the declaration-ordered +dependency list (A11) kept it on live attempts, +225 B, and is now let go after the dependency check. `links.ts` with +the young generation pinned: commit to frame, hold to live and retained per hold equal within run spread (176.4 → +176.2-176.6 KiB per hold). + +**Text-codec calls in isolation** (Sol's probe: a remote object over one memory link, 5,000 warm-up calls, ten bursts +of 1,000 small calls, then 200 warm-up and 30 bursts of 10 calls with a 1 MiB argument; binary first, then text; the +heap collected before each burst and the collections in it recorded: none in any burst), seven alternating pairs, +median of the pairs' medians: + +| Case | stock | young generation pinned | +| --- | --- | --- | +| binary small | 26.53 → 26.37 µs | 25.72 → 25.90 µs | +| text small | 19.31 → 20.95 µs (+8.5%) | 19.11 → 19.93 µs (+4.3%) | +| binary 1 MiB | 985 → 993 µs | 539 → 412 µs (bimodal: 187-607 µs per run) | +| text 1 MiB | 958 → 963 µs | 273 → 284 µs (221-352 µs per run) | + +Heap per call is the same (text small 15,171 → 15,160-15,202 B). The text small gap is not per-call work: the codec +alone (encode, decode and the message check of a call and a result message, 20,000 per burst) measures the same in both +trees (text call 1.55 µs and 52 B, result 1.52 µs and 171 B), deoptimizations and inlining decisions are the same, and +the gap follows the shape of the text decoder's `fromFields`, not what runs: the build with HEAD's `text.ts` matches HEAD +(19.0-19.2 µs), the one change in it that matters is the never-executed `T` (retry) case, and writing that case with an +object spread again (`...(f.length >= 2 ? { hops } : …)`) restores HEAD's 19.1 µs while other edits of the same size do +not. With text measured first the gap is 22.4-22.5 → 23.0-23.3 µs (+2-3%). V8 code generation, recorded as noise. + +## Slice 7c: plugins (2026-10-30) + +**The plugins component** (`test/next/bench/plugins.ts`, new): 20 plugins, one worker facet each (a store object over a +document, a derive, a method, a helper module and a copied contract; 80.9 KiB of scripts in all), Node 24.4.1. Medians +of seven runs; three runs pinned to CPU 8, one unpinned: + +| Measurement | CPU 8 | unpinned (32 CPUs) | +| --- | --- | --- | +| cold build of the 20 worker facets (one esbuild build each, all at once) | 240.1-240.7 ms | 26.4 ms | +| rebuild of one facet after an edit | 13.1-13.3 ms | 11.0 ms | +| evaluation of one facet (`vm.Script`, its source map registered) | 0.16-0.27 ms | 0.09 ms | +| reload of one plugin on a host (evaluation and `host.reload`, a pure swap) | 0.44-0.45 ms | 0.44 ms | +| heap after 100 reloads of one plugin, against after a warm-up | +230-235 KiB | +229 KiB | + +The heap does not grow per reload: 2,000, 4,000 and 8,000 reloads leave +897, +1,186 and +1,432 KiB (sublinear); the +same evaluations through plain `vm.Script` with V8's compilation cache off leave +420 KiB at 8,000 against +465 KiB +through `evaluateFacet` (V8's own per-script bookkeeping), and the kernel's reloads alone wander between +200 and ++600 KiB at any count. Old facets and their closures are collected (the GC test: 200 reloads, every facet but the last +two collected, the heap within 2 MiB of the warm-up). + +Fix round 1 (inline source maps without `sourcesContent`: positions only), CPU 8: the 20 scripts take 53.4 KiB (80.9 +KiB before); cold build 241.1 ms, rebuild 13.0 ms, evaluation 0.14 ms per facet, a reload 0.39 ms, heap after 100 +reloads +238 KiB. + +**Kernel hot paths** (per plugin at load and reload changes only staging), HEAD `18a155e2f` → build at paths of equal +length, CPU 6, three alternating runs. `calls.ts`: local call 2.69-2.71 → 2.71-2.75 µs (6,660 B both), watched +derive commit 10.34-10.36 → 10.43-10.49 µs (14,899 → 14,901 B), activation churn 58.93-59.58 → 59.44-59.73 µs +(41,452-41,470 → 41,470 B). `links.ts` binary commit to frame (median / p90): append 0.035-0.036 / 0.058-0.059 → +0.034-0.036 / 0.058-0.060 ms, changed field 0.027-0.028 / 0.031-0.033 → 0.027-0.028 / 0.031-0.033 ms, removal 0.022 / +0.026-0.028 → 0.022 / 0.026-0.027 ms; with the young generation pinned alike within 2 µs; text alike; small call +0.026-0.030 → 0.028-0.033 ms (pinned 0.030-0.035 → 0.030-0.032 ms); retained per hold equal (one binary hold 176.2 → +170.1 KiB). `reload.ts`: swap of 50 / 1,000 objects 1.44-1.73 / 30.83-32.71 → 1.52-1.53 / 30.55-33.25 ms, restart +1.01-1.05 / 15.54-16.87 → 0.95-0.99 / 15.90-16.02 ms until the reload resolves. + +## Slice 10: HTTP and workers (2026-10-31) + +`test/next/bench/http.ts` and `test/next/bench/workers.ts`, Node 24.4.1, CPU 6 (the forked workers inherit the pin). + +| Measurement | Result | +| --- | --- | +| keep-alive round trip, a JSON body: plain Node handler / a route (same process as the client) | build: 42.0-42.5 / 46.4-47.0 µs: +4.2-4.9 µs (9.9-11.7%); fix round 1: 42.5-42.9 / 62.8-64.4 µs: +20.3-21.5 µs | +| the same after 30,000 warm-up requests (fix round 1) | 30.9-31.3 / 43.4-44.0 µs: +12.1-12.9 µs; with an authenticator reading `request.headers`: +15.9 µs | +| the same, server and client in two processes (CPU 6 and 8; build) | 36.4-37.0 / 41.7-42.4 µs: +5 µs | +| 100 MiB file: `pipeline` of a read stream / `fileResponse` | build (64 KiB reads): 602-618 / 487-565 MiB/s; fix round 1 (256 KiB, pulled): 477-751 / 560-611 MiB/s | +| a worker's cold start with one shipped script: fork to `started` / to `ready` (median of 7) | 531-532 / 538-539 ms | +| the server's heap per idle worker (20 workers, one reference each) | 49.5 KiB | + +A request's overhead was 16 µs at first: `host.root` cost about 7 µs of it per request (a linked abort signal, a +`Promise.race` and an `awaitWithContext` per scope). Root scopes now take the caller's signal with one listener and +settle one promise (`root.ts`; behaviour and tests unchanged): 4.2-4.9 µs, about the bar of 10% of a round trip. The +rest is the request's scope (its abort controller, `CallContext`), admission and the response's header merge. The cold +start is mostly Node's startup stripping the types of the source tree it imports; the start timer (60 s) is far from +it. + +**Fix round 1** (the `fetch` core, the security baseline, Chord's listener as a native object). A route's overhead in +the standard run grew from +4.5 µs to +20-21 µs (+41 µs before the Node fast path: the listener reads Node's parsed +headers and makes a Web `Request` only when something reads `request.headers` or the body or a relay verifies; whole +bodies are written as strings, in one write with the head). Warmed, +12-13 µs; a CPU profile of the warmed route path +(per request, `test/next/bench/http.ts`'s route): the listener's request function 13.6 µs inclusive against the plain +handler's 8.9 µs, of which writing the answer 8.3 µs (the plain handler's `writeHead` and `end`: 9.1 µs); `serve` 4.2 +µs (the root scope 2.1, the URL parse 0.9, default headers 0.6, admission and routing 1.1, the answer 0.7); the root +scope's seal in a later microtask 0.9 µs. The baseline's extra headers cost 0.4 µs (a plain handler sending them). +The rest, about 5 µs, sits in no function of the component: Node's tick and microtask machinery (33 µs inclusive +against 18 µs for the plain handler). Opus's two trims are in (`lookup` decodes only segments with `%`; one abort +controller per request: the Node side's own is gone). Kernel rows, HEAD `bc9a20f84` → fix round 1, CPU 6, alternating: +binary 1 MiB call median 0.182-0.220 → 0.196 ms; reconnect of 1 / 100 references 1.02-1.10 / 79.1-80.3 → 1.04-1.05 / +81.3-81.5 ms; watched `writable.set` 0.836-0.868 → 0.861-0.863 µs; watched derive commit 10.42-10.58 → 10.36-10.60 µs; +activation churn 58.9-60.5 → 59.1-60.8 µs. + +**Kernel hot paths** (routes without a timer, `RouteHandle`, `Path` events, root scopes), HEAD `bc9a20f84` → build at +paths of equal length, CPU 6, alternating runs. `calls.ts`: local call 2.70-2.83 → 2.70-2.83 µs (6,660 B both), +watched derive commit 10.36-10.60 → 10.34-10.79 µs, activation churn 59.43-61.30 → 59.48-60.72 µs. `links.ts`: commit +to frame and encode/decode alike; reconnect of 1 / 100 references 0.94-1.15 / 80.8-84.0 → 0.94-1.15 / 80.6-86.5 ms; +1,000 parked calls 24.4-25.2 → 24.3-24.6 ms. `reload.ts`: swap of 50 / 1,000 objects 1.44-1.55 / 30.78-31.42 → +1.32-1.56 / 31.20-31.83 ms, restart 0.96-0.98 / 15.75-16.34 → 0.95-1.10 / 15.91-15.94 ms until the reload resolves. +`registries.ts` alike (one of 100 contributions set 1.47-2.45 → 1.48-3.41 µs, run to run). + +## Slice 12: the plugin model (2026-11-01) + +**The build's checks** (`test/next/bench/plugins.ts`, the 7c set: 20 plugins, one worker facet each, a copied contract +by relative path), Node 24.4.1, CPU 8, medians of seven runs, three alternating runs. HEAD `ebce9debd` (its bench +importing the contract by plugin name, which HEAD resolves) → this build: cold build of the 20 facets 257.6-257.8 → +257.2-258.8 ms, rebuild of one facet 13.2-13.5 → 13.3-13.9 ms. With the app's API set and the kinds named (every +import checked by version and kind, every resolved file by its kind segment; the plugins importing `@bench/app/v1`): +257.4-258.8 ms cold, 12.9-14.3 ms rebuild, against 258.2-259.7 ms without. The checks cost nothing measurable; no hot +path changed (the kernel gains two accessors read only by `testPlugin`). + +Fix round 1: the bench itself now builds with the app's API and the kinds (its plugins import `@bench/app/v1`; Opus +L5), and every package file's real path is checked against the plugin's `node_modules`. CPU 8, three runs: cold +build 261.1-270.2 ms, rebuild 13.6-14.4 ms (the first two runs as before, the third with a slower cold build on a busy +machine). diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md new file mode 100644 index 000000000..d510d2c2d --- /dev/null +++ b/packages/chord/docs/next/decisions.md @@ -0,0 +1,3033 @@ +# Redesign decisions (input for target design v2) + +Decisions made with the owner after the plugin review round on `target-design.md` (6559589bd), in the order they were +made. Later entries supersede earlier ones; superseded entries are marked inline and point to their replacement. The target design v2 is `packages/chord/docs/next/design.md`, written from this log; new decisions land here +first and win until the design is updated. + +## Plan + +- **p3 next to p2, fully isolated.** The new design is built as a new app (`apps/p3`) with its own data directory. p2 + keeps running unchanged with its data. Plugins are ported to p3 one at a time. +- **No legacy data migration.** p3 starts empty: no import of p2's `catalog.sqlite`, `shares.sqlite` or subagents table. +- **Chord v2 lives in `packages/chord/src/next`** (export `@earendil-works/chord/next`), developed on `private` only + (owner override of the "pi first" rule for this directory) and upstreamed to pi once stable; then `next/` replaces + today's Chord and p2 is deleted. `next/` imports `../context`, `../delta`, `../json`, `../output` directly, so + Durable's `Context`, `Draft` and `JsonValue` are the same modules and types in v2 (no copies, no type conflicts). + (Replaces "Chord v2 lives inside p3".) +- **Build a playground before p3:** a small system with p3's shape (server, worker processes over IPC, clients over + WebSockets) and toy plugins, one per concept; it proves the concepts and reload and is the e2e test ground. + Location `apps/chord-playground` (private). Client: a web page first (a CLI client possibly later). Order: kernel + (hosts, links, objects, references, principals), then plugin loading and reload with the dev loop, then the web shell + with a private preview, then each feature built live as a plugin. Plugin changes reload; Chord changes restart the + hosts (fine for the owner). +- **Chord v2 is written fresh** in `packages/chord/src/next`, copying what fits from today's Chord (link, wire, cell, + delta and the like). It needs the same test coverage as today's Chord (tests under `packages/chord/test/next`). +- **Playground web stack:** React, built with esbuild (loader D) instead of Vite, if feasible. +- **Reload control in the playground's web shell:** when sources changed, a control pops up listing the changed + plugins (reload them) or base code (restart all hosts); the owner triggers it manually. The agent can trigger the + same actions (command or tool). Never automatic on save. +- **Preview:** `https://p3.mariozechner.at` on slayer (wildcard DNS already resolves), set up like + `~/p2.mariozechner.at` (Caddy container on `caddy-network`, unix-socket bridge, `run.sh` in tmux) with `basic_auth`; + the password is chosen by the owner and never stored in the repo. `p2.mariozechner.at` stays the live p2. + +## Natives + +- **Natives only in objects.** An object opens a native in its synchronous factory with + `self.open(async ({ signal }) => value, close)`; Chord opens it during activation (the object goes live after all + opens) and closes it when the object ends, in reverse order. The value never changes during the object's life: no + `self.resource`, `version`, `replace`, `deps`, reopen, stale handles or "read `.value` at use". +- **The factory principle:** synchronous, declares only (references, documents, natives, derives, routines, jobs, + contributions); Chord performs activation; the factory receives handles usable in everything it registers; handles + are valid for the object's whole life. +- **No natives across plugins:** other plugins reach a native only through the owning object's methods and state, or + through the owner's registry entries ("Registries and HTTP"). No callbacks as method arguments, also not on + `local` types ("Object kinds"). +- **The facet is a pure object.** (**superseded in part** by "Facet documents are removed" and "Facets only declare" (2026-10-26): no documents, no routines) It may hold documents, references, routines and contributions, and may not open + natives (`self.open` does not exist on the facet). Served objects of a facet may read and write the facet's documents; + a served object's transaction may span its own documents and its facet's documents (lock order: own line, then the + facet's). The facet may not reach into its served objects' documents. + +## API style + +- **Tagged unions, always,** instead of a status string plus fields that only mean something in some states ("field + soup"). Each state carries exactly its own fields, and consumers narrow on `kind`. Example: `ref.state` is + `{ kind: "failed"; cause; retryAt } | { kind: "live" } | ...`, not `status` plus `error` plus `retryAt`. + +- **Context last (owner, 2026-10-21):** parameters are the required arguments, then the context (`ctx` or a Durable + `Context`), then optional arguments, as Durable already does (`Storage.scanEntries(query, limit, cursor, context)`, + `transaction(change, context)`). Examples: `self.tx(fn, ctx)`, `handle.change(fn, ctx)` and keyed + `handle.change(key, fn, ctx)`, `handle.retire(ctx)` and keyed `handle.retire(key, ctx)`, `chord(fn, context)`, + `job.start(args, ctx, { key })`. Applied to design.md, the guide and the walkthrough; the frozen `step-*.md` and + `kernel.md` document the frozen code and keep its signatures. pi's `traceIn(context, options, fn)` (implemented in + `chord/context`) becomes `traceIn(options, fn, context)` in the same pi change as S1 (owner, 2026-10-21). + +## Hosts and placement + +- **Placement is a function** of the object's typed input, not data naming an input field (`hostFamily(prefix, + key)` looked a key up in the input untyped). A function is type-checked and composable. The caller runs it and + sends the result; the provider runs it too and refuses an object that is not its own. Placement is then no longer + part of the contract fingerprint; a changed rule surfaces as that refusal. +- **Incarnations are minted by Chord** (uuidv7, the code copied from pi-ai into Chord: no dependency), never passed + in; the link handshake carries it, so the other side sees a restart as a new incarnation of the same identity. + +- **The low-level host API** (no topology layer; Radius and DO show entry points vary, so the kernel has no + "listen"): + - host kinds are typed: `hostKind("server")`, `hostKind("session", Type.Object({ id: Type.String() }))`; an address + is `Session({ id: "S1" })`; placement functions return addresses; no host-name strings in app code; + - `createHost

({ address?, principal? })`: a host with an address is addressable and acts as `principal` on its + own host; no options is a client (not addressable, no own principal); the incarnation is minted inside; + - (**superseded in part** by "Host values are removed" (Review round 1): start data is a core object) `host.provide(Key, value)` before `load`; factories read host values with `self.value(Key)`; + - `host.load(facets)` takes imported facets (loaders are adapters); + - `host.connect(link, { principal })`: the link to the parent (at most one), `principal` = who the parent is; + today's duplicate `upstream` option is gone; + - `host.accept(link, { principal })`: a link from a client; every entry point (loopback/tunnel WebSocket server, + Radius channels, a DO edge, memory) is app or adapter code that authenticates and calls `accept`; + - `host.route(Kind, { authorize(principal, address), resolve(address, { signal }) => { link, principal } })`: + how to reach hosts of a kind; `authorize` runs before anything starts; Chord calls `resolve` once per address + while its link lives and again after a drop; the returned principal is stamped on that link (defined once, in + the parent; the app sends it to the child in its own start message, with any init data); + - starting processes, init data and keep-alive are app code (helpers later if apps repeat them). + Close to today's shape (`createHost`, `connect`, `accept`, `route`, `load`); gone: string host names and prefix + parsing, app-managed generations, `upstream`, `opaque` runtime objects, the loader inside the kernel. + +## Quick decisions on the design's open points (design.md section 26) + +- **Call-scoped natives (#4):** no `ctx.open`; a native that lives for one call is opened and closed in the method with + `try/finally` (documented pattern). Revisit only if it shows up often. +- **Documents across plugins (#6):** no plugin reads or writes another plugin's documents (Chord or Durable). Data + crosses plugins only through state and methods. ~~Within a plugin, the facet's documents are readable and writable + by the objects that facet serves (decided earlier).~~ (**superseded** by "Facet documents are removed") Data core enforces lives in a format core owns and is handed to + core through its API when the data is created (e.g. `createConversation` with `init`), never written into core's + documents directly. +- **`overlap` for routines (#8):** no `overlap: "latest"` option; a handler compares with the current value itself. +- **Async activation guard (#3):** none; authority checks live in the per-principal view's methods (and in + `route.authorize` before a host starts). +- **`close` with a reason (#7):** `self.open(open, (value, reason) => …)` with `reason: { kind: "reload" } | { kind: + "stop" } | { kind: "end" }`, decided by Chord (its plugin is in the reload set and the object is native / the host is + shutting down / the last reference ended or its type was removed). Owners decide what to do with it. Chord aborts + only the object's own calls and jobs; work elsewhere that uses the native (e.g. Durable tool executions using a code + mode sandbox) is the owner's to drain or end in `close` ("old code keeps running" applies to natives too). +- **Code mode in p3 drains its sandbox in `close`** (**superseded in part** by R16 (`close` detaches the sandbox and disposes of it in the background)) (not moved into Durable): the plugin keeps owning the + `pi-codemode` sandbox; on `reload` (and `stop`) `close` seals it (no new scripts), running scripts finish or hit + their deadline, then its VMs and worker threads end (`closeWhenIdle` in `pi-codemode`, or a counter in the plugin). + New scripts start on the new sandbox through the registry entry. `RunningCalls`, the removal message and + `abortTask` on reload go away; after a worker stop Durable replays as today. +- **Invariants across owners (#5):** the owner of the dependent data enforces the rule by reconciling from the other + owner's state with an idempotent routine; no cross-owner transactions, no "notify the other owner" calls that can + be lost. Example: ports' forward store watches the catalog and closes forwards of archived sessions (also after a + crash, from the current value); sessions does not know ports exists. + +## Wire (#15) + +- ~~**Compression:** large JSON envelopes are deflated on WebSocket links (relayed and mobile links benefit); byte + segments stay raw and uncompressed; IPC never compresses.~~ **Superseded** by R1 and "Compression at the origin". +- **Frame size limit per link adapter** (**superseded in part** by Q2 (limits per part from the header; `maxPayload` only as the hard cap)) (e.g. 16 MiB on client WebSockets, none on IPC): a frame over the limit fails + that call or member with `too_large`, the link stays up. Large uploads go through owner chunking anyway. +- **One batch per commit per link:** a commit that touches several holds on one link travels as one `state` frame; + the consumer applies every member update of it before listeners run. Remote consumers never see a torn commit, + matching local subscribers, who are notified once after every document of a commit is adopted. +- **Adapted bindings get diffs:** a consumer bound through `accepts` (an older contract version) is served like a + derive of the provider's state through the adapter: Chord keeps the last adapted value per binding, applies the + adapter to the new value and sends the diff (`diffRevisions`), never the whole adapted value per commit. + +## Gaps closed after the walkthrough rework + +- **Clients** serve only `local` types (not addressable); local calls have no principal; what a client calls on the + server is stamped with the client's authenticated principal. +- **Placement refusal** is `invalid/placement`; during a rollout it is handled like a mismatch (provider on another + revision: park; same revision: fail). +- **HTTP route signature:** `(req, ctx) => Promise`; `ctx.principal` is the request's principal (from + the request resolvers), so `ctx.get(Notes, …)` gives that principal's view. +- **Start barrier** (**superseded in part** by R17 (the harness starts after the barrier, which waits until every plugin loaded or failed)) counts facets and `lifetime: "host"` objects; an object that must contribute at start (the model + runtime) is `lifetime: "host"`. +- **`persist` only in `local` objects and `lifetime: "host"` objects** (**superseded** by P3: only in input-less `local` + objects) (identities without a principal); `storeKey` + is dropped (two principals' objects writing one document would be two lines on one document). Enforced at the type + level (`self.state` in a remote factory takes only non-persisted definitions) and as a declaration error at runtime + ("persist needs a stable identity: put the documents into a local core"). +- **Call deadlines:** a call ends through the caller's signal; there is no per-call deadline of its own. Parked calls + (link drop, loading, rollout) share one default deadline (60 s, like `readyTimeout`), then fail `unavailable`. +- ~~Lingering through `ref.release({ linger: true })`~~ **superseded** by D8 (no consumer-side linger). Was: a closing view releases with `ref.release({ linger: true })`; the hold stays, marked lingering, until + its timeout. A link drop is still a departure (presence details stay open). +- **Contributor order** (**superseded in part** by "Contributor order is sorted by plugin name (never load order)" in the design, section 17) is load order (alphabetical by plugin name), deterministic on every host and across reloads, + and carries no meaning: wraps in one phase must not depend on each other; meaningful order comes only from phases. +- **Revocation without field patterns:** the app defines a typed revocation type and a function + `createHost({ …, revokedBy: (p: P) => R[] })` (the same app code on every host); `host.revoke(r)` ends every + object and closes every link whose principal `p` has `r` in `revokedBy(p)`, and propagates `r` (a small JSON value) + to the children, which apply their own `revokedBy`. No `PrincipalPattern`, no local-only predicates, no re-stamping: + a changed grant (mode `write` → `read`) is a revocation of the old one; the client reconnects and authenticates + anew, and its references re-hold. +- **Per-session native owners:** (**superseded in part** by "Host values are removed": the session is read from the core object `SessionInfo`) the facet cannot `self.get` a type it serves, so a worker-wide native owner is a + `lifetime: "host"` object without input that reads its session from a host value (`self.value(SessionInit)`). + +## Documents step + +- **Plugin identity:** every plugin has a `package.json` whose `name` is its globally unique, stable plugin name (like + an npm package name); that name identifies its facets (~~persisted facet documents,~~ reload, diagnostics; **superseded in part** by "Facet documents are removed"). Loader D + reads it; static composition passes the same name. Never a directory name, never load order. +- **Document roots are `JsonObject`,** as in Durable (`defineDoc`); lists live in a field. Documents + hold JSON only. +- **In-memory documents end with their object** (last reference released, linger expired); they survive reloads and + native restarts of a live object. What must outlive the object is persisted or lives in a `lifetime: "host"` object. +- **Chord ships its own document storages, no adapter over Durable's storage:** (**superseded** by "The store" + (2026-10-21): Chord's persisted documents live in the per-session store, next to Durable's adapter and plugin tables) in-memory and SQLite (JSONL was cut later) + implementations of `DocumentStorage`, inspired by Durable's storage implementations (atomic commits, base plus + deltas, incarnations) but with Chord's own schema: ordered keys per owner and definition, a tag index. Durable's + storage is shaped for Durable (scans by id, no tags); Chord documents and Durable transcripts never commit together, + so they need not share a storage. Node-specific storages live outside the portable core; a Durable Object SQLite + storage can follow. (Replaces the earlier "adapter over Durable's node SQLite storage" for the playground.) +- **Durable's SQLite driver layer moves into Chord** (`@earendil-works/chord/sqlite`, `/sqlite/node`, + `/sqlite/cloudflare`): the `SqliteDatabase`/`SqliteExecutor` interfaces, the `node:sqlite` and Durable Object + drivers, the migration runner and schema-prefix helpers. Durable imports them from Chord (it already depends on + Chord, so no cycle) and re-exports them under its old paths so no Durable user breaks. Chord's SQLite document + storage uses the same drivers (`next/` imports `../sqlite` like `../output`). Done in pi first, then merged. + +## Derives and natives step + +- **`self.open` returns a handle** whose `.value` is the native (throws before activation); no forwarding proxy + (private fields and identity break through proxies). The value never changes during the object's life. +- **A partly failed activation** closes the natives already open (reason `end`) and opens all of them again on the + next attempt. +- **A broken part fails the whole object** (owner): a derive that throws, or a document whose hydration or `migrate` + fails, makes the object `failed`; an incomplete object could break invariants that rely on atomic commits across its + documents. (Replaces "member-local" errors for documents.) +- **`previous` of derives** is tested through failed/retry until the reload step exists. +- **`self.retryOn` may appear anywhere in the factory:** the factory specifies, the host orders and executes (watches + are installed before natives open). +- **Failure causes are tagged:** `{ kind: "transient" }` (an `open` throws, a reference is unreachable) gets the + automatic backoff; `{ kind: "broken" }` (factory, derive, document hydration or `migrate` throws; or an `open` that + throws `broken(…)` because retrying is pointless, e.g. invalid config) gets no automatic retries, only reload, + `ref.retry()` or `retryOn`. Chord decides by where it failed; only an `open` can choose `broken` itself. +- **Activation waits for its dependencies:** an object whose references are not live yet stays `loading` and goes live + as soon as they are (bounded by `readyTimeout`, then it fails `transient` with the dependency as cause); ~~a `broken` + dependency fails it at once. No retained references while failed.~~ **Superseded** by R18 (dependency failures are + `transient`, references are kept as an automatic retry trigger). +- **Timeouts** are host defaults with per-thing overrides: `closeTimeout` 30 s (per native), call drain on teardown + = `closeTimeout` (per type), `readyTimeout` 60 s (per `get`), park deadline 60 s (per type), routine handover 10 s + (per routine), job `finish` retry backoff 1 s to 60 s (per job), rollout install 60 s, linger per type; route + `resolve` is bounded by the app's signal. +- **Inbox path:** the notes plugin computes its own inbox path from generic start data (the data directory; host + values were removed later, start data is the core object `SessionInfo`). + Natives open sequentially in declaration order and close in reverse; a throwing `close` does not stop the others + (`host.close` rejects with all errors, background cleanup reports to `onError`). Type names `Native`, + `NativeCloseReason`. +- **`retryOn` sources** must live independently of the attempt (facet documents, registry values; host values were + removed later; + readables the host provides); a reference's member is not allowed (type error). Dependency waiting covers references. +- ~~**A facet that fails transiently in activation** retries automatically~~ (**superseded** by "Facets only declare": a facet cannot fail after `load`) with backoff, like an object; it does not wait + for a reload. +- **Park deadline per consumer:** `get(..., { parkTimeout })` overrides the host default for that reference; the + provider's per-type override applies to its own local bindings. +- **Playground:** the inbox watcher only opens and closes for now (the import comes with routines); a note-count derive + and the reader's redaction as a derive are visible to clients. + +## Telemetry + +- **Context:** (**superseded in part** by "Telemetry types and review cleanup": `Context.telemetry` is always a + `TelemetrySpan` and `withTelemetry` takes spans) Chord's `Context` gets `telemetry: TelemetryContext` as a fixed member, like `abortSignal` (never + undefined; `NOOP_TELEMETRY_CONTEXT` by default), with `withTelemetry(telemetry, context)` and + `traceIn(context, options, fn)` (child span, `fn` gets the derived context). `value()` and keys stay, forever, without + deprecation. Durable gets telemetry everywhere through the `Context` it already passes; principal and other app + values stay out of Durable's context (Durable work runs in the scheduler's contexts and resumes without a caller; + origin travels as data). In Chord v2, `Ctx` extends `Context`: `ctx.telemetry` is the current span, `ctx.trace(name, + attributes, fn)` gives `fn` a child `ctx` (principal and `get` kept), so calls inside are parented correctly. +- **One adapter per host:** (**superseded in part** by the adapter is a `TelemetryContext` with readonly `metrics`/`propagation` members, held only by hosts ("Telemetry types and review cleanup")) `createHost({ telemetry })`, the adapter implements `TelemetryContext` and optionally + `TelemetryPropagation` and `TelemetryMetrics` (no split). Default NOOP. +- **pi-telemetry additions** (pi first): `TelemetryCarrier = Record` (opaque, W3C-compatible); + `TelemetryPropagation { inject(context) → carrier | undefined; extract(carrier) → TelemetryContext }` (absent, + invalid or unknown carriers give the adapter's root, never throw; carriers are best effort: after an adapter change + only links are lost); span links (`SpanOptions.links`, required `TelemetrySpan.addLink(link)`, a link targets a + context); `NOOP_TELEMETRY_PROPAGATION`; the in-memory adapter implements propagation, links and metrics (carrier + snapshots, no forced W3C fields); conformance fixture with optional `propagation`. No `isRecording` for now (it would + break adapters, as would closure attributes). +- **Metrics** (in the same pi change): `TelemetryMetrics { counter, upDownCounter, histogram }` with `add`/`record` + (synchronous, cheap, never throw; histogram `record` takes an optional context for exemplars), `InstrumentOptions` + (description, unit), histogram boundaries, `NOOP_TELEMETRY_METRICS`, typed through `metrics` in telemetry schemas, + low-cardinality attributes only. +- **Across links:** (**superseded in part** by R5 (the carrier is a bounded field of the header schema)) `call` and `hold` frames carry the caller's carrier; the provider extracts it and runs the method + under it; the server adds a `chord.relay` span. Cancellation already travels as `cancel` frames. +- **Shared work** (an activation awaited by several getters, a worker start, the per-object line) is its own root span + with a link to its first trigger; every waiting span links to it. Work without a running trigger (routine runs, job + runs, eviction, backoff retries, background closes) is a root span, linked to its trigger where one exists. Chord + sets all of these; authors may add links through `ctx.telemetry.addLink`. +- **Tiers:** spans for calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine + runs, job runs and finishes; events on the enclosing span for commits inside a method (`chord.commit` with docs, ops, + duration), retries and parking; metrics for background high-frequency work (state frames, bytes on WebSocket links, + commit durations, in-flight calls, active holds, open links, park durations, backpressure waits). Verbose per-commit + or per-frame spans are an adapter decision (filter by name); Chord, Durable and pi-ai only report. Sampling, + buffering and limits belong to the adapter. +- **No measuring for telemetry:** only sizes the code already has are reported (encoded WebSocket frames, `byteLength` + of bytes, lengths a storage computed while writing); nothing is serialized just to count it. +- **A Chord telemetry schema now** (typed span, event and metric names; used by tests and debugging). +- **Debugging:** (**superseded in part** by the SQLite telemetry store and `TelemetryStore` ("Telemetry store")) a JSONL trace adapter (bounded rotating file, batched asynchronous writes, drops when full, never + blocks) and a text renderer of one trace now; a web trace viewer in the playground (span tree, durations, hosts, + links, attributes, events; also usable standalone on a JSONL file) with the web shell. +- **Durable** (pi): (**superseded in part** by "Durable and AI spans" and "Telemetry implementation decisions" (phase spans under the stored carrier; every input submission starts its own trace)) `HarnessOptions.telemetry`; an optional `trace` carrier field on submission and task records (the + submitting call's or the parent task's carrier, rewritten with the record as other immutable fields are; JSON records, + no migration, no format bump, old records simply have no link); task runs are child spans while their trigger runs in + the same process, otherwise root spans linked to the stored carrier; Durable spans get their own schema. Persisted + Chord jobs store their start carrier the same way. +- **Own schema, no OTel GenAI (owner, 2026-10-18):** pi's AI and Durable telemetry uses its own typed schema + (`pi.ai.*`, `pi.durable.*`); the OTel GenAI conventions (development status, renames, streaming still `TODO`) are not + followed. An OTel adapter may translate pi's names to `gen_ai.*` at export. Model stream details are span events, never + per delta. The pi part of telemetry (propagation, links, metrics, typed metric schemas, `chord/context` telemetry + member, `HarnessOptions.telemetry` type) is approved as diffed in `/home/badlogic/pi-telemetry`. +- **Durable and AI spans (owner, 2026-10-18), partly open:** (**superseded in part** by pi-ai's option is `telemetry` (a span), not `telemetryContext` ("Telemetry types and review cleanup")) every span is short and in one process. Spans: + `pi.durable.submit` (admission), `pi.durable.phase` (one invocation of any task's phase; fixed name, `task.name`, + `phase`, `kind`, `attempt`, `resumed`, outcome as attributes), `pi.ai.request` (one model request). Events: + `pi.ai.first_chunk`, `pi.ai.block` (per `*_end`: type, index, chars, tool; never per delta), `pi.durable.entry`, + `pi.durable.submission`, `pi.durable.replay` (a codemode call answered from its journal). No content ever; entry ids + point to the transcript. Submissions and tasks store their creator's carrier; phase spans are children of it. Every + input submission starts its own trace, linked to its trigger (bounded traces; subagent report chains do not nest). + No hook spans for now. Metrics: `pi.ai.request.duration`, `pi.ai.first_chunk.duration`, `pi.ai.tokens`, `pi.ai.cost`, + `pi.durable.phase.duration`, `pi.durable.tasks.active` (ids never on metrics). `pi.ai.request` is emitted by pi-ai + in one wrapper around `lazyStream` in `models.ts` (both `stream` and `streamSimple`; reads the event stream, changes + nothing; deferred results get `pi.ai.poll` the same way); Durable passes `telemetryContext`. Background tasks + (e.g. compaction) are children of the creating phase like every task. A task picked up by a new process sets + `resumed=true` and stays under its stored carrier (same trace). +- **Telemetry implementation decisions (owner, 2026-10-18):** `TelemetryContext` carries optional `metrics` and + `propagation` members (every span from an adapter carries its capabilities), so one type is used everywhere + (`HarnessOptions.telemetry`, `createHost({ telemetry })`, `Context.telemetry`, pi-ai's option, renamed `telemetry`). + Entry and submission events only after a successful commit. Several queued inputs started as one run: the run takes + the first input's carrier and stores the others' as immutable creation-time links on its first phase span; a steer's + submit span links to the running run's carrier. A phase closed or cancelled without a committed state ends + `aborted`. Submit spans only on the public submit path; internal admissions inside another commit show as events. + ToolTask sets `tool.name`/`tool.call_id` on its own span. Write submissions are children of the caller; only input + submissions start traces. Polls count tokens and cost only for final results. Import budgets raised (pi-ai models + 18, Durable root 71). +- **Errors in telemetry (owner, 2026-10-18):** a failed span carries `status: "error"` with a fixed category name and the + error message truncated to 500 characters (`provider_error` for a model request, `task_faulted` for a faulted phase). + Cancelling is not an error (`stop=aborted`/`outcome=aborted` keep status ok). Durable's `onReport` stays unchanged; + every report is additionally a `pi.durable.report` event on the current span (its own short root span outside any + span) plus a `pi.durable.reports { type }` counter. (Superseded in part below: no root span for reports.) +- **Telemetry types and review cleanup (owner, 2026-10-18):** `Context.telemetry` is always a `TelemetrySpan` (default: + the NOOP span); the adapter (`TelemetryContext` with readonly `metrics`/`propagation`) is held only by hosts + (`createHost`, `HarnessOptions`) to start roots, never put into a Context; pi-ai's option is `telemetry?: + TelemetrySpan`. Code that only has a Context calls the generic `addEvent` (no typed-events helper, no casts, no + duck typing). A report is an event on the current span plus the counter and `onReport`; outside any span only the + latter two. `TaskOptions.trace?: TelemetryCarrier | "root"` (absent: inherit). No protection against broken adapters. + Telemetry never walks or reads data just to measure (checkpoint event compares the phase only). Instrument caches are + `WeakMap`s keyed by the adapter's metrics object. +- **Telemetry store (owner, 2026-10-18):** a SQLite adapter in Chord (`@earendil-works/chord/telemetry-sqlite`, reusing + `chord/sqlite`, Node and Cloudflare): settled spans one row each (events and links inline), metrics aggregated per + series per 10 s, a 1 s batched flush that also deletes rows older than the retention (7 days), a bounded queue of + settled spans (dropped and counted), no retries. A shared read interface `TelemetryStore` in pi-telemetry, modeled on + Jaeger's span reader: `findTraces({ name, attributes, errors, start window, limit, cursor })` returns complete traces + (newest trace start first, paged by an opaque cursor), `getTrace(id)`, `findMetrics({ name, window, attributes })`. + Spans carry global ids and a trace id (a root's own id), links carry trace and span id. Both the SQLite and the + in-memory adapter implement it; one renderer `renderTrace(spans)` (subpath `pi-telemetry/render`) prints links and + never follows them. Live spans, incoming-link search and duration search are left out. The store's metric read method is + `findMetrics` (`metrics` is the recording capability). Adapters trust typed inputs (no validation of malformed + caller values, no try/catch around recording); only carriers from another process are checked (keys present, + strings), else the root. + +## Review round 1 (Astra, Fable): kernel fixes and decisions + +Source: `p3-reviews/v2-review-summary.md` (K = kernel fixes, D = decisions). + +- **Variants (D11):** a member name has the same kind (state or method) in every variant that has it, and a method has + the same signature (arguments and result) in every variant; state schemas may differ per variant (redaction). + Enforced at the type level in `remote(...)` and at runtime for contracts arriving as data. K7 becomes that check. +- **Registry composition is a runtime state, not a staging check (D1; reverses "compose runs in staging").** + Contributions can be dynamic (only known once a native is open, or changing at runtime), so no conflict check can be + complete in staging. `compose(table)` is pure and runs whenever the table changes (after a commit, when a contributor + goes live or ends, when a dynamic contribution changes; contributions may be readables). The composed value is a + plain value for readers: `compose(table, { report })` always delivers a value, builds the fallback in (e.g. the + first by plugin name wins) and reports conflicts through `report` (they become owner notices and telemetry events); + readers never handle conflicts. Only if `compose` throws is the registry in an error state, and readers get an error + when they use it (like a failed reference). Staging rollback stays for code errors only (load errors, throwing factories). A contribution + is in the table exactly while its contributor is live (a facet while its plugin is loaded; an object from `live` + until it ends, fails or restarts). The HTTP router follows the same rule: a duplicate route is a conflict state, not + a rollback. +- **Code mode reload vs. a pinned Durable phase (D2):** (**superseded in part** by R16 (the old sandbox is detached and disposed of in the background, not drained in `close`)) the code mode tool entry looks up the current sandbox when + `execute()` starts (not the one it was created with), so a call pinned by a phase before a reload (e.g. waiting in + an approval hook) starts on the new sandbox; the old sandbox is still sealed and drained in `close` (running scripts + finish or hit their deadline, then it ends). +- **Natives that follow a relocation (D3):** documented pattern, no new mechanism: the native is a stable manager + (`self.open` once); a routine on the session location switches its internal resource (e.g. the watched directory) + and bumps a generation, so late events from the old location are dropped. Used for watchers, prompt resources and + OS probes in p3. +- **Jobs keep their owner alive (D4):** (**superseded in part** by R11 (revocation ends the objects whatever their jobs)) a job that is running, finishing or blocked in `finish` counts as a reference + to its object; the object ends only once all its jobs are settled. Starting a job and releasing the reference (an + HTTP route returning the job id, a tab closing) no longer aborts it. Keep-alive already counts running jobs. +- **Reloading native objects (D5):** stays as decided: native objects of a reloaded plugin restart. In practice this + costs little: the Durable harness is core (base code, restarts only with the host), terminal PTYs live in the + separate terminal host process (the server's native is only the connection), model runtimes, sandboxes and the + search index rebuild in milliseconds; only working on envs (running remote commands are killed) or radius (relayed + connections drop) hurts, and only on reloads of exactly that plugin. Keeping natives by name was rejected: natives + have no line, no transactions and no schema, so old and new code would interleave unchecked. +- **Old handles stay valid until their calls settle:** when a pure object is swapped in place, a call that started on + the old code (and its transaction) keeps working with the old handles until it finishes; only new calls get the new + handles. Matches "running calls finish on old code". +- **Core vs. plugin is decided by where the code lives:** `src/` (base code, imported statically by the start code + under a fixed name such as `@p3/core`, not switchable, a change restarts the host) or a plugin directory (found by + listing, loaded through loader D, named by its `package.json`, switchable, a change reloads it). Core is what the app + always needs and everything builds on (harness, catalog, access, session location); optional features are plugins + (code mode stays a plugin). +- **Host values are removed** (no `hostValue`, `provide`, `self.value`, no host-value versions). Start data (session + id, init, data directory) is a `local` core object with `lifetime: "host"` and constant state, served by the core + facet from the start message; platform bindings the host owns (a Durable Object's `state`/`env`) are core objects + whose `self.open` returns the binding and whose `close` does nothing. Plugins read start data through a reference, + in openers, derives, routines and methods (after activation); nothing reads it synchronously in a factory. A + long-lived native from the host is a `lifetime: "host"` object in a core facet. +- **`accepts` adapters map values (D6):** (**superseded in part** by R6 (provider-owned old schemas plus mappings)) per member, `state: { member: (newValue) => oldValue }` (pure; Chord wraps it + as a derive and diffs per binding) and `methods: { name: (oldArgs) => newArgs }` (plus an optional result mapping + back); members an old consumer does not know do not appear. Authors never build readables for adapters. +- **Revocation and work in cores (D7):** no revocation hook. A core that runs work on behalf of a principal stores the + grant (or device) as data with the work and reconciles from the grant owner's state (the sharing core's live grants): + a routine aborts jobs, withdraws questions and drops queued input whose grant no longer exists. The same pattern as + invariants across owners; it also holds after a crash. +- **Lingering and keep-alive (D8, decides open #1):** (**superseded in part** by R26 (the counts are one readable, `host.activity`)) the consumer-side linger is removed (`release({ linger })`, the + lingering mark on holds, the consumer linger cache): a closed tab closes its link, so it could never survive the + case it was made for. What remains is provider-side `serve(…, { linger })` (an object stays alive for a while after + its last reference, whoever held it; this is what keeps a just-closed conversation warm). Presence is the live + references (`host.references`). Worker keep-alive works as in p2 (the worker counts, the app's supervisor decides): + core code on the worker reports viewers (from `host.references`), active calls and running jobs (Chord's counters) + and Durable tasks (the core `Harness`), plus provider-side lingering objects; the server's supervisor stops the + worker when all are zero for the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle + logic. +- **Validation cost (D9):** (**superseded in part** by R9 (no dev walks: one behavior everywhere) and R7 (arguments validated for every caller, inputs normalized)) document values are not walked against their schema in production. Instead each + document record stores the fingerprint of the schema it was written with (computed once per definition from the + structural schema, ignoring descriptions and titles; written at creation and with the new base after a `migrate`). + On access Chord compares the token's fingerprint with the record's: same version but a different fingerprint fails + the object `broken` ("schema changed without a version bump"). Always on, O(1). Every structural schema change needs + a version bump (an additive change gets a trivial `migrate`). Derives and readables are not validated in + production; in dev, values may be walked as a bug finder (documents at commit, state at publication). Inputs from + peers (method arguments, hold inputs, with the provider's schemas) are always validated. +- **Storage failure (D10):** Chord never ends the process. A storage error ends the store and closes the host, and + `onError` receives a tagged error (`{ kind: "storage_failed", … }`); the app's start code must handle it (log and + exit with its own code for a supervisor, or reopen). The design states that every app has to; the playground shows + how (`process.exit` with a distinct code). Fixes the server zombie (a closed host with a live listener). +- **Reading without starting a worker (D14):** (**superseded in part** by R20 (Chord stamps the caller's incarnation as `ctx.incarnation`)) plain pub/sub in an ordinary plugin, no Chord mechanism: the plugin's + worker facet records the summary locally and publishes it to the plugin's server object (a method call, from a + routine on the source); clients read the server object, which never starts a worker. The server object keeps the + latest value per session (a document if it must survive a server restart) and ignores reports from an older + worker incarnation. Session list, status, children, unread, artifact lists follow this. +- **The K list is confirmed** with these changes: + - K9: (**superseded in part** by R14 (`checkpointWhen` is required on every definition)) like Durable: storages do not validate deltas against content (the store already holds and validated the + authoritative value); a commit writes the delta; replay only when loading; checkpoints stay optional per + document, no default. + - K16: (**superseded in part** by "Flow control: an ack-clocked window" (call window N, not credits; no byte window)) no count limits that refuse; call credits as backpressure (throttle only, never an error); the byte window + applies to every frame including first snapshots; holds unlimited (protected by authorization and input + validation). + - K17: dropped (an IPC peer producing deep JSON shows up at once in development). + - K21: (**superseded in part** by "Only `tx.draft`" (`tx.get` is gone) and R12 (drafts are revoked; reads after the commit)) values are immutable by convention (Chord Delta's rule, as in Durable), documented for `tx.get`, + `handle.value`, `handle.initial()`; no copy, no freeze; the `keys()` index is fixed; the write on a read-only + transaction after a version bump is documented. + +## The state path: minimal compute and memory (after the cost review) + +Source: `p3-reviews/v2-cost-model.md` (before/after per hop, measured by Fable and Astra). Replaces the per-change +validation and copies the v2 kernel added around Delta. + +- **The model, end to end, as Delta and `replicatedState` (which Durable already uses):** (**superseded in part** by R1 and Q4 (parts encoded once at the origin per hold; relays rewrite headers; one binary format)) values are immutable + revisions with structural sharing; commits produce exact ops; subscribers on the same host get the same value and + ops by reference; derives publish through `diffRevisions` (identity short-circuit; bytes are atomic leaves; + equality never by `canonicalJson`); each frame is encoded once per link and sized from its encoded bytes; relays + read only the frame header and forward the body opaquely; receivers check a frame once by its header (`JSON.parse` + already yields strict JSON; bytes recognized in O(1)); consumers apply with `applyImmutable`. No validation of + state, no clone of snapshots or ops, no repeated walks. +- **A v2 readable is `replicatedState`:** today's `chord/src/state.ts` semantics (immutable value, ordered ops, + gapless attach with snapshot and cursor) is the one readable implementation, imported by `next/` like `../delta`, + so Durable's views and document watches are state members without conversion. +- **The store is a port of Durable's store design** (`durable/src/session/transaction.ts`, `session.ts`): one Delta + tracker per document, prepare and adopt by pointer swap, observers get the commit's ops, storages write the delta + (or a base when `checkpointWhen` says so) and check only metadata; replay only on load. +- **Only `tx.draft`:** it returns a draft of the current document, or a draft over `initial()` if the document is + absent; an unchanged draft (empty ops) commits nothing, so an absent document is never stored by being read. + `tx.get` is removed. +- **`derive` stays as it is,** documented with its footgun (a derive that builds new objects for unchanged data costs + O(value) per commit and per view) and a cookbook of efficient derives for p2's categories: select a field, look up + by key, small aggregates (just compute); filter/map a list (an identity cache per source element); redaction + (preferably separate documents per visibility, otherwise the identity cache); joins (identity cache plus the + looked-up source's revision); a window over a large list (reuse unchanged parts, as p2's chat window). No `project`, + no view algebra, no `memo` API. +- **Diffing and derived values come from Delta and today's Chord, not new code:** Delta's `diffRevisions` (tested, + identity short-circuit) gets bytes as atomic leaves (compared by identity, replaced when different), and v2 calls + it directly; `next/diff.ts` is removed. Derives reuse today's derived cells (`chord/src/cell.ts`) on top of the + readable. Delta gets a small, pure, tested helper `mapShared(f)` (an identity cache per source element: a new object + per changed element, the previous result for unchanged ones), used by derives that build a new object per source + element; the cookbook documents it alongside the patterns. +- **Same-host calls on `remote` types pass values by reference** (**superseded in part** by R9 (no freezing anywhere, no development mode)) (immutable by convention, as `local`); the dev freeze + catches mutation. (Replaces "validate and copy arguments and results also on the same host".) +- **Arguments and hold inputs are validated at the provider by the caller's trust:** (**superseded in part** by R7 (arguments validated for every caller; inputs always normalized)) whenever the originating + principal is not a host principal (guests, owners, any client, also when relayed over trusted IPC), once, with the + provider's schema; calls from server and worker code are not validated in production. Results are not validated in + production. Provider schemas are compiled lazily at their first use and cached in a `WeakMap` keyed by the schema + object; old tokens and their compiled schemas are collected by the GC after a reload. +- **Contract descriptions travel once per (link, fingerprint):** (**superseded in part** by R6 (no descriptions on the wire; strict fingerprints)) a hold sends the fingerprint; the provider asks for + the description once and caches the parsed token per link; the cache dies with the link. +- **Cost guards:** (**superseded in part** by R9 (the counters are ordinary telemetry metrics, always on; no test mode) and T7 (one encode per hold part at its origin, none at relays)) counting tests in the normal test run (in test mode Chord counts full-value walks, deep copies, + encodes, schema validations and materializations; per hop a test asserts the counts, e.g. one appended note to a + 500 KB document with 10 viewers: no walks, no copies, one encode per link, no validations) and a `vitest bench` + suite outside CI (500 KB and 1 MB values, 0/1/10 viewers, memory/SQLite, 1 MiB bytes) with results recorded + in `packages/chord/docs/next/benchmarks.md`. +- **Naming:** the pattern of a `local` object holding shared data behind per-principal views is the **shared store** + (not "core"); base code and plugins use exactly the same primitives. + +## Cuts and closed deferred points (review round 1) + +- **Cut:** the cross-relay reset coordination (`#collectReset`, `#resyncs`, `#resetRequests`): on an overflow a relay + releases the hold and re-holds, and the provider sends a fresh snapshot. The JSONL document storage (Chord ships + memory and SQLite). `MemoryDocumentStorage.prepareCommit`/`snapshot` (unused after K9). +- **Kept:** (**superseded in part** by R9 (no dev freeze)) the dev freeze (the safety net for passing values by reference). The Questions service stays a documented + pattern, not built. **Deferred:** the HTTP and channel components, middleware phases and the Durable Object profile, + until the kernel fixes and a first real proof (harness, one tool with an approval pause, a code mode reload) are + done; the playground's web shell serves its pages with plain app code until then. +- **Presence (#10):** a link drop is a departure (accurate and simple); smoothing (a phone switching networks) is the + UI's job, e.g. a debounced viewer count. Counts per person or origin are derived by the app from + `host.references` (principal, origin). Workers do not stop on a flicker because of provider-side linger (D8). +- **Optional dependencies (#2):** a rule, no mechanism: an optional dependency is a registry owned by the dependent + (ports contributes its forwards to `radius.forwards`; without ports the table is empty). No `optional` on `get`. +- **Rollout fixes (#12), built with the rollout step:** (a) a host checks its own install status before waiting on a + provider (a rejected own install fails at once with the reason); (b) installs are serialized per host; (c) a + revision also carries the code-set digest (the hash over the facet bundles' content hashes that loader D computes + anyway; no module analysis), and hosts compare the digest on reconnect, so a server restart with unchanged code + triggers no reinstall. +- **UI reload vs. leaving (#13):** "leaving" is never derived from an unmount, only from explicit navigation or + presence (a dropped reference); a component unmounted by a reload has not left. + +## Process (after review round 2) + +- **Layer by layer, as pi-durable was built:** the existing `next/` code (about 10k lines, kernel steps 1–6) is frozen as + a reference and not built on; Chord v2 is rebuilt bottom-up in small layers (data and contracts; readable and derive; + documents and store; objects; references and `ctx`; protocol and links; relay, routes, revocation; routines and + jobs; registries; reload and rollout; telemetry). Per layer: a one-page spec with data flow, cost per hop, a line + budget, footguns and what is reused, approved by the owner; a minimal build with counting and GC tests; a cleanup + loop with fresh reviewers and the owner until he is happy; only then the next layer. Target: 6–7k lines without + tests. The handoff for continuing after a compaction is `apps/p2/docs/p3-handoff.md`. + +## Review round 2: decided items + +- **R1, the message protocol and relays:** (**superseded in part** by Q4 (one binary message format on every link), R6 (no `describe` kind) and compression at the origin) a message is a header plus opaque parts, one part per hold. A part's payload + is encoded once by its origin with that hold's own Delta encoder (path interning per hold, end to end between + provider and consumer), packed with its byte segments, compressed at the origin above a threshold (`codec`). WebSocket: + `u32 header length | header JSON | part bytes`; IPC: structured clone of `{ header, parts: Uint8Array[] }`; memory: + the object. Kinds: `hold`, `release`, `call` (1 part: arguments), `cancel` (consumer to provider); `bound` (1 part: + snapshot), `state` (n parts: one per hold the commit touched on this link), `result` (1 part), `gone`, `describe` + (provider to consumer). Relays parse and rewrite only headers: map hold and call ids per link, stamp the principal on + `hold` and `call` (a principal field from a client link is always overwritten; only a trusted parent link may carry + one), group parts per destination link (each link gets only its own holds' parts); payloads are never parsed. One + `state` message per commit per link. Encoding happens when sending: while a link has credits a commit is encoded and + sent at once; under backpressure its ops are collected per hold and encoded into one part when credits return (still + atomic for the consumer). Flow control counts the parts' sizes from the header. A re-hold sends a fresh snapshot and + restarts that hold's encoder and decoder. Two holds of the same principal (two tabs) encode the same ops separately + (O(change) each). Replaces "relays forward bodies opaquely" from "The state path". + +- **R2, frame size limits (owner, 2026-10-19):** (**superseded in part** by Q2 (limits per part; `maxPayload` only as the hard cap)) configurable per link kind: the compressed (wire) size through the + WebSocket server's `maxPayload`, the decoded size through zlib's `maxOutputLength` in the one decode function, + wherever decoding happens (relays do not decode). Defaults for client WebSockets 16 MiB compressed, 64 MiB decoded; + none on IPC. An oversized message fails only its own call (`too_large`); the link stays up. +- **R4 and R5, protocol header schema (owner, 2026-10-19):** (**superseded in part** by Q1 (unknown ids are ignored; only a header failing the schema closes the link)) the message header is a TypeBox schema, a tagged union + over `kind` (`hold`, `release`, `call`, `cancel`, `bound`, `state`, `result`, `gone`); protocol types derive from + it, and headers from untrusted links are checked with the schema compiled once (trusted links such as IPC are not + checked, as for arguments). No kind allow-list: every message resolves against the receiving link's own tables; a + header that fails the schema, or an unknown kind, hold or call id, is a protocol error that closes that link. The + trace carrier is an optional bounded field (key count, string lengths) of `call` and `hold` in that schema; relays + read it from the header. Payloads stay opaque. +- **R7, inputs and arguments (owner, 2026-10-19):** (**superseded in part** by "Params instead of inputs": flat required + params, no normalization) object inputs are always normalized (defaults, unknown fields + removed, a canonical identity key), whoever the caller is, so equal inputs give one identity; inputs stay arbitrary + (nested objects, arrays, records allowed). (**Superseded in part** by Q12: closed schemas, unknown fields rejected.) + Method arguments are always stripped to the provider's schema and + validated with it (compiled once), whoever the caller is: trusted principals are not type-checked code (plugins are + bundled by esbuild without type checks, agents write JavaScript plugins). Bytes are leaves: `instanceof Uint8Array` + plus an optional length bound, never walked. Results and state are not validated (they come from the provider). + Replaces "arguments validated only for callers that are not host principals". +- **R8, migration output (owner, 2026-10-19; `initial()` part **refined** by "Simplification pass after 4a" D3: once per definition):** the result of `migrate` and of `initial()` is validated against the new + schema once, when produced (per document per version change, on load, off the commit path); a failure refuses the + access, keeps the old data and fails the object. Commits are still never validated. +- **R9, no development mode (owner, 2026-10-19):** one behavior everywhere; the host's `dev` option is removed. Values + are immutable by convention and never frozen (deep freezing was too costly in Delta), and state is never validated + (schema changes are caught by fingerprints, migration results by R8). The cost-guard counters (walks, copies, + encodes, validations, materializations) are ordinary telemetry metrics, always on; tests assert them through the + in-memory adapter. Replaces "values immutable by convention, with a dev freeze". +- **R10, revocation window (owner, 2026-10-19):** accepted. Work started on behalf of a revoked principal stops when + the reconciling routine runs; a step started in between still runs. No extra checks before tool calls or approvals; + there is always some window. +- **R11, jobs and revocation (owner, 2026-10-19):** "jobs keep their owner alive" stays for the normal end (the last + reference released: a closed tab does not abort a running job). Revocation takes precedence: revoking a principal ends + its objects at once, whatever references or jobs remain; their jobs get the abort signal and the normal + `closeTimeout`. A job runs as its object's principal. +- **R12, returning data from transactions (owner, 2026-10-19):** option B, as in Durable: drafts are revoked when the + transaction settles; transactions write and return at most plain values; data is read after the commit from the + document (`handle.value`, `handle.get`). No `tx.read`. +- **R13, existence inside transactions (owner, 2026-10-19):** as in Durable, no existence check: `tx.retire` of an + absent document is a no-op, `tx.draft(…, { tags })` creates a missing document with its tags, existence is read + outside the transaction. The R12 and R13 patterns are documented in `design.md` (section 9). +- **R14, checkpoints (owner, 2026-10-19; **refined** by "Slice plan and slice-1 review decisions": required only for persisted documents):** no default, but `checkpointWhen` is required on every document + definition, in Chord and in Durable (a pi breaking change), so forgetting it is a type error. Durable's task-scoped + documents (nested call, codemode run and journal) return false (they retire with their task); the codemode store + checkpoints after 64 deltas; p2's small settings documents checkpoint on every write, its growing ones after 64. +- **R15, old calls and migration (owner, 2026-10-19):** as in Durable: running calls keep their old handles until they + settle, except for documents whose version rose; old code touching a document migrated past its version throws, the + document stays intact, the call fails once. +- **R16, native restarts (owner, 2026-10-19):** no Chord change: a restart closes the old native, then opens the new. + Code mode's `close` detaches the old sandbox, returns at once and disposes of it when its scripts finish or after + the default `closeTimeout` (30 s, no override); its tool entry finds the current sandbox at `execute()`. Documented + as two patterns for natives: drain in the background, and swap through a registry contribution. +- **R17, harness start (owner, 2026-10-19):** (**superseded in part** by "The harness is an ordinary object") the harness is not part of the start barrier; it starts after it (the + barrier waits until every plugin loaded or failed, so the registries hold everything that will be there). No list of + required contributions and no paused state: a missing tool gives its call an error result; a missing model makes the + generation report "model unavailable" and wait until a model appears; failed plugins are reported with their error. +- **Note for p3, moving a session between instances (owner, 2026-10-19):** copying a session's database to an instance + with other plugins already works as Durable does: missing tools, sections and hooks are skipped; tasks without their + definition stay blocked (`missing_task`) until installed or aborted; documents of missing plugins stay untouched; + stored selections are never rewritten. The UI must show blocked tasks with their reason and offer to abort them (a + turn waiting on such a task hangs until aborted). +- **R18, dependency recovery (owner, 2026-10-19):** a dependency that fails (or does not become live in time) fails + the dependent `transient` with the dependency's cause inside, never `broken`. The dependent keeps its references + while failed; their state is an automatic retry trigger, so the dependent is retried the moment the dependency goes + live (backoff cancelled; it stays as a safety net). Invisible to factory authors. After `live`, a dependency leaving + `live` leaves the dependent live; its calls using the dependency get its error or park while it loads. +- **R19, flow control and large snapshots (owner, 2026-10-19):** (**superseded** by "Flow control: an ack-clocked window") snapshots are always admitted, even when larger + than the window (they consume it); overflow is decided on changes queued after them; after an overflow a re-hold + waits until the credits cover the last snapshot's size, with a minimum interval. +- **R20, incarnation fence (owner, 2026-10-19):** Chord stamps the calling host's incarnation (from the link handshake) + on calls arriving over a link: `ctx.incarnation` next to `ctx.principal`, not part of the principal or its key; a + server object ignores reports from an older worker incarnation by comparing it. Nothing is sent as data. +- **Testing the core (owner, 2026-10-19):** every mechanism decided in review round 2 gets exhaustive, individual tests + with its edge cases; every layer spec lists them (design section 28). +- **R21, who contributes to registries (owner, 2026-10-19):** only facets and `lifetime: "host"` objects, enforced by + the type of the factory's `self` (only theirs has `contribute`). Reading registries stays open to every object. +- **R23 (owner, 2026-10-19):** `constant(value)`, a readable that never changes, used for start data. +- **R22 (owner, 2026-10-19):** a host-lifetime object without input gets session-scoped objects with `ctx.get` in a + routine or method (documented pattern); a `get` with an input computed after activation only if needed. +- **R24 (owner, 2026-10-19):** the container-width cost of structural sharing is documented (one array slot copy and + one element visit per element per change); a benchmark of long arrays sets the limit; large collections are keyed + documents or bounded windows. No new mechanism. +- **R25 (owner, 2026-10-19):** rejected are registries created at runtime and entries resolved through mutable + indirection; a contributor may change its contribution at any time (a readable contribution recomposes on change), + and readers may look up the current value at use. +- **Changing contributions, `self.writable` (owner, 2026-10-19):** a contribution is fixed for its contributor's life + or a readable; Chord subscribes to readable contributions and reruns `compose` on every change; use sites follow the + composed readable or read its current value. `self.writable(initial)` is a host-local readable of any in-process + value (identity-compared, no JSON, no wire) with `set`, for contributions such as MCP tool lists. Tools, sections, + hooks, wraps and tasks reach the harness as one Durable `Extension` per plugin; skills and other lists likewise. +- **Removing a contribution (owner, 2026-10-19):** no remove call; an entry leaves when its contributor ends or a reload + stops contributing; a live contributor with nothing to offer sets its readable to the registry's empty value (no + `undefined` entries). +- **R26, drift (2026-10-19):** fixed in the text (the description frame is gone with R6; §9 no longer says values read + in a transaction are shared). Keep-alive counters: one readable `host.activity` (`{ references, calls, jobs, lingering }`; owner). +- **Q1, unknown ids (owner, 2026-10-19; review round 3):** unknown hold and call ids are ignored on both sides (races + such as `result` crossing `cancel` or `call` crossing `gone`); a consumer receiving `gone(h)` rejects its own pending + calls on `h`. Only a header that fails the schema closes the link. Replaces "an unknown hold or call id closes the + link" (R4). +- **Q2, size limits per part (owner, 2026-10-19):** limits per part from the header sizes, before decoding; an oversized + part fails only its hold or call, other parts apply; an oversized snapshot fails its hold without automatic re-hold; + the admitting link's decoded limit is stamped on relayed work and enforced where it is decoded; `maxPayload` is only + the transport's hard cap (closes the connection). Replaces R2's per-message wording. +- **Q3, re-hold after overflow (owner, 2026-10-19):** (**superseded** by "Flow control: an ack-clocked window") re-hold once the previous snapshot's debt has drained (credits at + the full window), with the minimum interval; replaces "once the credits cover the last snapshot's size" (R19), + which stalls for snapshots larger than the window. +- **Q4, one message format and window (owner, 2026-10-19):** (**superseded in part** by "Wire protocol" (codecs, not the JSON header framing) and "Flow control: an ack-clocked window" (no byte window)) every link carries the same binary message + (`u32 header length | header JSON | part bytes`, one `Uint8Array`): WebSocket as a binary frame, IPC through + `serialization: "advanced"`, memory delivered asynchronously. Encoding, framing, checks, ids, flow control and limits + are one shared protocol layer; adapters only move bytes; one byte window everywhere from `byteLength`. Replaces the + IPC frame-count window and R1's per-adapter formats ("IPC: structured clone of { header, parts }; memory: the + object"). Tests on memory links run the production path. +- **Q5, dependency cycles (owner, 2026-10-19):** rejected before activation waits: on one host by following the + waiting edges of activating objects; across hosts through the activation chain of uuidv7 instance ids carried in the + header of holds made during an activation (bounded; ignored from untrusted links). A cycle fails the activation + `broken` naming it; no references are kept; checked again after reloads. +- **Q6, Chord from Durable tools (owner, 2026-10-19):** (**superseded in part** by "Context last": `chord(fn, context)`) p3's core puts the host into the harness's base context under + a typed key; `chord(context, fn)` opens a call scope (host principal, the call's abort and span, references released + when `fn` settles) in tools, hooks and sections; no wrapping, no p3 `defineTool`. Needs good docs: the pattern, + lifetime and principal, and the Questions service as the example (tools reach users only through owner state; + clients are not addressable). Replaces "hooks open a call scope with `host.root`" (plugins never hold the host). +- **Q7, code mode on reload (owner, 2026-10-19):** R16 stands: the detached old sandbox is disposed when its scripts + finish or after 30 s at most; a reload during a longer script aborts it and that turn fails. After a reload at most + 30 s of old code keeps running. +- **Q8, registry ownership and versions (owner, 2026-10-19):** one authoritative declaration per registry, from the + defining facet (`compose`, version, `accepts`); other tokens only identify it; a reload replaces it atomically. + `accepts` stays (third-party plugins are an explicit goal of p3), owned by the defining facet, pure, following the + owner-adapter pattern of documents, object types and jobs; a contribution without a matching version or adapter is + left out and reported, not a load error; contributions are not schema-checked. Every use site of a contribution + catches throws and rejections, reports naming the contributor, and continues. Replaces "newer ones are refused at + load". +- **Q9, job dedupe and retention (owner, 2026-10-19):** dedupe covers unsettled runs only; after `finish` commits, + Chord drops the run's record and key (a persisted record in the same commit) and releases waiters; dedupe after + completion is the owner's, through the result in its own document. Chord keeps nothing beyond a run's lifetime. +- **Q10, contract evolution (owner, 2026-10-19/20):** forward evolution in code, no build step: a base written in + full with an explicit version (`remote()`/`local()`; `local` keeps its own kind with flat `members`, C1 rejected), + `evolve(prev, e => …)` always continues the history with structured edits (`state`, `addState`, `removeState`, + `args`, `result`, `method(name, args, result)`, `addMethod`, `removeMethod`); every version is materialized and + fingerprinted at declaration. Input or variant-set changes start a new history (a new declaration with an explicit + version; no older accepts). Mappings only with the provider, in the pure `checkAccepts(contract, accepts)` (no + `serve` in this layer): required at type level and at runtime for every member of the old version that changed or + was removed; removed state from the whole current variant state or `"refuse"`. Fingerprint: `hash64` over a + schema-aware data-only normal form (`NORMAL_FORM_VERSION`; union and intersection order kept; `required`/`enum` + sorted; TypeBox keyword whitelist; bytes as a `{ type: "bytes" }` data marker); history private, deep-copied and + frozen (**superseded in part** by "L1 rewrite review": nothing is frozen). Schema helpers `str num int bool nul lit oneOf union obj arr map opt bytes unknown` + `type Static`. + Implemented and reviewed (Fable, Astra: thumbs up) in `packages/chord/src/next/contracts/` (`7d2bb8ea8`). +- **Q11, absent singletons (owner, 2026-10-21):** a singleton always has a value: `handle.value` is the written value + or `initial()`, never `undefined` (an exposed singleton is typed `T`); `undefined` only for absent keyed documents + (`handle.get(key)`); singleton existence has no API. A deliberate difference from Durable's `snapshot()`. +- **Facets typed by the app's principal (owner, 2026-10-21):** base code exports `defineFacet` bound to its `P` + (`bindFacet

()`, working name) next to the principal union; every facet imports it from there; `host.load` + accepts only facets typed for the host's `P`. No runtime check. +- **`placement`, not `host` (owner, 2026-10-21):** (**superseded in part** by "Params instead of inputs") the contract field naming where an object lives is `placement` + (a typed function of the input); `input` stays (the object's key among objects of its type, part of its identity). +- **Q-B2, in-memory documents across input changes (owner, 2026-10-21):** not carried over. An input change of a + type ends its live objects on reload; their in-memory documents, flows and jobs are lost; documented ("in-memory + documents survive a reload only while the object identity is unchanged; persist what must survive"). +- **The store (owner, 2026-10-21; backends **superseded in part** by W14: no IndexedDB; direction accepted, details under review in `p3-reviews/store-proposal.md`):** + P1 recorded: a session is self-contained (its store plus its directory); export from the session alone, import + into another instance, sync to Radius and continue elsewhere; the server only mirrors (catalog) and keeps instance + data. One transactional table store per session, layered backend (SQLite, memory, IndexedDB) / store (tables, a + small query DSL, controlled writes, commit queue, seq, outbox, export/replay) / adapters: Durable's `Storage` as a + new `StoreStorage` adapter (Durable's existing storages stay unchanged), Chord's persisted documents, and plugin + tables (`self.table`), each table owned by one object like a document. Writes are only `put`/`delete` + (`deleteWhere` resolved to keys), recorded as row operations `{ table, key, v, row }` in the same backend + transaction as the rows, so a replica or an import needs no plugin code; indexes are local (**superseded** by "Index + definitions are recorded"); migrations run only + where the owning code runs and ship as ordinary puts. Lines stay per Chord object and per Durable Session; the + store has one short commit queue per store that assigns the seq; Durable's seq is the store's seq (Durable sees + gaps; it only compares seqs). Rejected on the way: derived-only plugin data, opaque plugin directories, export + hooks, plugin record logs, per-store outboxes with a global position counter, a SQLite-changeset or SQL-statement + store, raw SQL reads, an IndexedDB-only API, Chord documents inside Durable's `Storage`. Open: physical layout + (generic rows or real tables), DSL scope, migration timing, `next_id`, backend details, the ordering review + ("seq 40 after seq 50"). +- **S1, adoption inside the commit step (owner, 2026-10-21; store review):** every owner's in-memory adoption runs + synchronously inside the store's commit step, after the backend commit and before the next commit starts, so memory + always shows a complete prefix of the log. Durable's `Storage.commit(writes, adopt, context)` gains a required `adopt` + callback (pi breaking change): every storage calls it synchronously once the commit is durable and before its next + commit; the Session catches its own adoption error and fails; a rejection after adoption fails the Session without + `tx.discard()`; `writeAtClose` passes a no-op; `StoreStorage` forwards `adopt` into the store's step; Chord objects + adopt (pointer swap) the same way. Listeners stay where they are: after adoption, in each owner's continuation + (unchanged), so their order across owners may differ from the seq order while every listener reads a complete + prefix. Checked against Durable's code (memory, JSONL, SQLite, GuardedStorage, writeAtClose); the pi change and its + test run come with the store work. +- **S2, references to later commits (owner, 2026-10-21; store review):** no mechanism and no new rule. Ids are minted + inside the transaction that creates them (Durable `tx.appendEntry` → `mintId`) and reach other code only after that + commit, so a reference to a later commit only arises when code leaks data out of an uncommitted transaction, which + is already wrong today (the transaction may still fail). The guide states it once: ids and data leave a transaction + only after it commits; reference what is committed. +- **S3, commit completion (owner, 2026-10-21; store review; IndexedDB parts **superseded** by W14):** the commit step finishes only on the backend's terminal + success (SQLite: `COMMIT` returned; IndexedDB: the transaction's `complete` event); only then adoption (S1) and the + next commit run. The memory backend stages a commit and swaps it in at the end. An ambiguous outcome (an error during + `COMMIT`) stops the store (`storage_failed`, D10), as Durable's `GuardedStorage` does. After a restart a local head + below the uploaded head is normal: the driver pulls its own uploaded batches back before writing. SQLite keeps + `synchronous=NORMAL` (no extra cost: a crash loses nothing, a power loss the last commits, recovered from Radius); + IndexedDB uses `durability: "strict"` where supported. Condition (owner): SQLite must not get slower. +- **S4, consistent reads (owner, 2026-10-21; store review):** an export pauses commits (holds the commit queue) while + it reads, so the archive is the state at one head; streaming from a snapshot comes only if exports get large. Reads + that need several queries (a document from its base and revisions) run in one backend read transaction, as + Durable's SQLite storage does. Ordinary reads in object code (separate awaits) are not snapshot reads; the guide + says so; no global snapshot guarantee. +- **S5, replica apply (owner, 2026-10-21; store review; a sync-layer rule):** a replica applies a whole batch in one + backend transaction together with its applied head. Uploads may run in parallel (owner); the driver resubmits a + batch until Radius acknowledges it and trims a batch from its outbox only after its acknowledgement. Every batch + carries `from` and `to`; a replica applies a batch only when its `from` equals the replica's applied head, so it + never applies past a gap: a later batch waits until the missing one has arrived. +- **S6, compaction (owner, 2026-10-21; store review):** (**superseded** by W9: no compaction at first) covered by S5: a compacted batch is applied as a whole, so + its intermediate states are never visible; takeover happens at batch boundaries. Nothing to add. +- **S7, losing the driver role (owner, 2026-10-21; store review):** no rollback (tools ran, files changed, natives and + derived data followed the tail). When an upload is refused for an old epoch, the local copy becomes a separate + session with a new id and keeps running; commits in flight land in it; the old id's sync stops there; the new session + may sync on its own from a full snapshot; the two never sync again. The catalog shows both (the split-off copy marked + with its origin and seq). No `not_driver` error. To work out with sync: changing the id (store metadata, session + directory, catalog), links from other sessions keep the original id. +- **S9, S12, S14, S17 (owner, 2026-10-21; store review):** S9: the store persists its head with every commit; the + prototype's `Sequencer` and safe cut go away (prototype bug, nothing to decide). S12: the query DSL starts without + `join`, `groupBy` and `select`; plugins aggregate in JS over indexed scans; added when a real case needs them. S14: + tables and documents of an object are written in one transaction API, `self.tx(fn, ctx)` (no separate + `table.transaction`). S17: bulk operations are designed now, together with migrations, after the remaining S items + (full-row puts make a 10k-row rewrite about 40 MB in the outbox; revision pruning is one record per row). +- **S8, work that finishes out of order (owner, 2026-10-21; store review):** an app concern, no store or job mechanism. + §14 states that the handler calls of one routine never overlap; the guide shows two patterns: chain through the + routine (await the run's outcome, skip values that are no longer current) or drop stale results in `finish` by + comparing the source position inside the destination's transaction. +- **S10, ids (owner, 2026-10-21; store review):** no persisted reservations. Durable's `mintId` contract is narrowed to + "greater than every id minted since this storage was opened, and every id stored, also across reopen" (an id from a + transaction that never committed was never seen outside it, S2; Durable's transactions are serialized on the + Session line). In the store each owner's id counter starts at open above that owner's highest stored id (one index + lookup per table; a replica taking over gets it the same way). Durable's existing storages already behave so; the + comment changes in the same pi change as S1. +- **S11, copying documents into a fork (owner, 2026-10-21; store review):** `StoreStorage` reconstructs the source + documents of a `document.copy` before entering the commit step (reads of committed history outside the step) and + enters it with plain creates carrying the base; Durable is unchanged. Safe because the source is a past state and + only Durable's Session line writes Durable's tables. The outbox only ever carries the resulting rows. Forks of forks + and selecting an entry in the parent work as today (rewindable documents keep their history). +- **S13, one documents engine (owner, 2026-10-21; store review):** the store contains one documents engine used by + Durable's `StoreStorage` and directly by Chord (Chord keeps no Durable dependency). Address `(namespace, owner, kind, + key?)` with an opaque `owner` string chosen by the adapter (Durable: `session`, `conversation:7`, `task:31`; Chord: + host/object type); records per incarnation with `history: "latest" | "full"`, `createdAt`, `retiredAt`, opaque + `meta`; revisions (base with version and optional fingerprint, deltas). Reads outside the step: `find(address, at, + context)`, `load(id, at, context)` (reconstructs base plus deltas once; no cache), `list(query, limit, cursor, + context)` ordered by id or key. Writes inside the step: `create(record, base)`, `change(id, content)`, `retire(id)`, + with the engine's checks (one live per address, create before change, delta on a base of the same version, nothing + after retire); `latest` prunes revisions on a new base and on retire. Callers decide base or delta + (`checkpointWhen`) and keep their in-memory trackers as today (Durable's Session caches, pins and evicts unchanged). + Fork copies stay in `StoreStorage` (S11). Chord's frozen storage layer and its conformance suite are not ported. +- **S15, Durable's existing storages (owner, 2026-10-21; store review):** kept, as decided; no plan to remove them. + Revisit whenever it becomes worth it. +- **S16, layout and text order (owner, 2026-10-21; store review, Astra's design; **superseded in part** by W8 (every text + column), W14 (no IndexedDB) and "Text is plain UTF-8, in code point order" (the whole text order and encoding)):** real tables per logical table + (definitions and changes recorded in the outbox, the store renders DDL); keys and indexed columns of type text or + integer, the rest of a row in a JSON column (**superseded in part** by "Flat columns"). Text order is JavaScript's (UTF-16 code units) on every backend: memory + and IndexedDB use plain strings natively; SQLite stores key and indexed text columns as `BLOB`s of UTF-16 code + units, high byte first (`Buffer.from(s, "utf16le").swap16()`), so its bytewise comparison equals JS order; text that + is not indexed stays `TEXT`. Lossless (lone surrogates, NUL round-trip), no comparator, no normalization ("é" and + "e\u0301" are two keys). The SQLite backend encodes writes and query bounds and decodes results; outbox records, + exports and cursors keep logical strings. Prefix ranges come from one store helper (increment the last code unit + below `FFFF`). Every index ends with the primary key; cursors are opaque `{ dir, last }` with logical values and are + refused by a scan in the other direction; a row whose indexed field is missing or null is not in that index. Plugin + rule: text compares exactly like JavaScript strings; page with the store's cursors. Verified on Node 24.4: lossless + round trip, SQLite BLOB order equals JS sort, ~0.4 µs per encode plus decode. Cost: indexed text shows as hex in + `sqlite3` and takes two bytes per code unit. Rejected: generic JSON rows, store-maintained index entries, code point + order with `toWellFormed()` (Fable), WTF-8, text equality-only. +- **S17, bulk operations and migrations (owner, 2026-10-21; `p3-reviews/store-bulk-migrations.md`; JSON-part wording + **superseded in part** by "Flat columns"):** writes are `put` + (a whole row), `update(table, key, set)` (some columns of one row), `delete`, `updateWhere` and `deleteWhere` (many + rows, one record). Bulk operations are recorded as operations with the driver's affected-row `count`; every replica + runs the same operation (one SQL statement, or a loop in the memory backend) without plugin code; a different count + stops applying that store and restores it from a snapshot. `set` expressions touch real columns only: `value`, + `col`, `add`, `sub`, `max`, `min`, `concat`, `replacePrefix`, `coalesce`, `when`; no JSON-field operands (added + when a real case needs them); no subqueries, joins, aggregates or insert-from-query (read first, then `inArray` or + puts); no case folding, `substr` on unindexed text or floats; one conformance suite for both backends. Migrations: + shape changes of the JSON part migrate on access (each row stores its version `v`; `migrate(row, from)` on read; the + next write stores the new shape; old code refuses newer rows); changes to key or indexed columns run eagerly when + the owning object starts on the driver, before it goes live: recorded table changes (`alter` add/rename/drop, a + rebuild for key or type changes), then either an `update` step (one `updateWhere`) or a `rows` step (scan in key + order, `tx.update` per row, chunks of N rows per commit; table version `migrating` → `ready`; a crash resumes the + scan). Indexes are local and created after `ready` (**superseded in part** by "Index definitions are recorded": the + index changes after `ready` are recorded). +- **Blobs and derived data (owner, 2026-10-21):** blobs (artifact versions, uploads, sent files) are app layer: files in + the session directory named by the hash of their content, never changed, written (tmp, fsync, rename) before the row + or document that names them; any process may write one. Export copies `blobs/` with the store snapshot; import + copies them back and checks each hash. Sync: before uploading a batch, the sync layer uploads every blob file present + at that moment that has not been uploaded yet (no reference tracking; a replica never sees a hash without its file). + No blob is deleted before its session is deleted (an unreferenced blob only costs disk). Derived data (full-text + indexes, caches, a plugin's search database) is never exported or synced and is rebuilt where needed, typically by a + catch-up routine comparing it with the store. +- **Index definitions are recorded (owner, 2026-10-21):** index contents are never recorded (every backend maintains + them from the rows), but their definitions are: when the owning object starts on the driver, the store compares the + declared indexes with the existing ones and creates or drops the difference, recorded as `{ op: "index", table, name, + columns }` and `{ op: "dropIndex", table, name }` (after a migration's `ready`); every replica applies them. Every + copy of a store, an export and the file seen by outside tools is then identical and self-describing. Indexes never + affect results (queries order by index columns plus the primary key; rows without the field are excluded by rule). + Cost: replicas without the plugin maintain indexes they never use; adding an index builds it during apply. +- **P3, where persisted data lives (owner, 2026-10-21):** persisted documents and tables only in input-less `local` + objects (one per host and type); their address is (host, object type, name, key), with no input, so an input change + can never orphan persisted data. Per-entity data (per conversation, per session) is keyed documents or table rows in + such a store with an app-chosen key (data, not identity); objects with an input keep only in-memory state and read + persisted data from the input-less store by key. Not adopted (proposals of 2026-10-21): `previous`/`rehome`, the + split question, identity hashes as storage addresses. A persisted store's type and definition names stay stable; + renaming one, or changing how keys are formed, is an explicit one-off migration in the store. p2 fits: its four + stores with an input persist no Chord state. Enforced at the type level (`self.state(…, persist)` and `self.table` + only in input-less local factories) and as a declaration error at runtime. +- **Flat columns (owner, 2026-10-21):** `defineTable` takes `columns`, a flat record: each column is a scalar (`str`, + `int`, `bool`, `num`) or `json(...)`; no nested `obj`/`arr` outside `json(...)` (refused by the types and at + declaration). Every column is a real SQLite column; scalars can be filtered and ordered; keys and indexed columns are + `str` or `int` (S16); a `json(...)` column is one JSON `TEXT` column, opaque to queries, typed when read. Supersedes in + part S16 ("the rest of a row in a JSON column") and S17 ("shape changes of the JSON part"): adding, renaming or + dropping a column is a recorded `alter`; shape changes inside `json(...)` columns migrate on access. +- **P2, in-memory documents are instance state (owner, 2026-10-21):** an in-memory document is attached to the live + object: no storage, no address, no seq, never in the outbox. Same machinery as persisted documents (drafts, `self.tx`, + atomic commits, ops for subscribers and the wire); its commit is prepare and adopt. A transaction mixing both kinds + adopts the in-memory part inside the store's commit step (S1). It lives from activation to the end of the object: + survives plugin reloads (pure objects swapped in place) and native restarts of the live object, ends with the object + (last reference, linger, an input change on reload, Q-B2). Deletes the memory `DocumentStorage` for in-memory + documents and identity hashes as their keys. +- **Query runner (owner, 2026-10-21):** a query is immutable data built with `query(Table).where(…).orderBy(…)`; + `limit` is an argument, not part of the query; the same `where` serves `updateWhere`/`deleteWhere`. The table handle + from `self.table(Def)` runs it: `page(q, limit, ctx, { after })` returns `{ rows, next? }` (an opaque cursor with + direction and last key as logical values, bound to the query's order, refused with another query), `rows(q, ctx)` + iterates all pages, `get(key, ctx)`, `count(q, ctx)`. Outside a transaction it reads committed state, one query at a + time (S4). Reads inside `self.tx` (owner, 2026-10-21): `tx.page(table, q, limit, { after })` and `tx.rows(table, q)` + return committed rows, stable for the whole transaction (only the owning object writes its tables, P3, and its + transactions run one at a time on its line), without the transaction's own writes; `tx.get(table, key)` overlays + the transaction's own pending writes (put, update, delete) on the committed row (**superseded in part** by W10: after + a bulk operation on the table, `tx.get` throws `read_after_bulk`). + Documented rule: queries inside a transaction see the state before it; own writes become visible after the commit. +- **P4, plugins' own databases and files (owner, 2026-10-21):** state written by code lives in a store: the session's + store for session data, the server's store for instance data (p2's `catalog.sqlite` becomes a rebuildable mirror table, + `shares.sqlite` server tables, `subagents.sqlite` part of the session's metadata, the code-written JSON files such as + `models-store.json`, `forwards.json`, `owner-revoked.json`, `radius.json`, `device.json`, `prefs.json`, `envs.json`, + `voice.json` server documents or tables); a plugin's own SQLite file holds only derived data (`search.sqlite`); blobs + are files named by hash; files a person edits by hand (`settings.json`, `models.json`, `auth.json`) stay files + (open point #21). Logs are not data. +- **Params instead of inputs (owner, 2026-10-21):** an object type's `input` is renamed `params`: the fields that select + an instance among objects of its type (like route params), part of its identity. Params are a flat record of + required fields, each `str()`, `int()`, `bool()` or a union of string literals (`oneOf("chat", "agent")`); no + defaults, no optional, nested or unknown fields (a `get` with a missing or extra field fails `invalid/params`); the + identity key is the field values in declared order (no canonical JSON walk, no normalization). p2's 78 object types + fit (52 empty, the rest one or two required scalars). Supersedes in part R7 ("inputs stay arbitrary", "inputs are + always normalized") and the placement entry ("`input` stays"); Q12 (stripping) now concerns method arguments only. + "Input-less" in P3 and Q-A reads "with empty params". +- **Q-A, lifetime from kind and params (owner, 2026-10-21; both reviewers up):** no `lifetime` option and no lazy flag. + A `local` type with empty params lives as long as the host (started eagerly at host start, never ended by reference + count; it may hold persisted data, P3, and contribute, R21); every other `local` type and every `remote` type lives + while referenced (`linger` only for those). No remote object has host lifetime (its identity includes the + principal; long-lived parts go into a local shared store). A lazy flag was rejected for now: a lazy store would + contribute and run routines only after first access and escape the start barrier; p2's 11 such stores open only + cheap natives. Every "`lifetime: \"host\"` object" in earlier entries reads "`local` object with empty params". +- **The harness is an ordinary object (owner, 2026-10-21; Astra's C17):** no harness exception in the start barrier. + The harness is a host-lifetime object (`local`, empty params) that opens Durable during the barrier with scheduling + paused (Durable opens paused until `resume()`); base code calls `harness.resume()` after `await host.load(...)` and + before the worker connects to its parent (**superseded in part** by W6: connect first), so no request arrives earlier and every registry is filled. Chord knows + nothing about Durable's start. +- **Q12, closed argument schemas (owner, 2026-10-21; review round 3):** every object schema in a contract is closed + (`obj()` sets `additionalProperties: false`); an unknown argument field fails the call with `invalid/args` instead of + being stripped; the provider validates the caller's object in place, never copies or mutates it; arguments have no + defaults (optional fields are `opt()`, handlers write `args.limit ?? 50`). Supersedes in part R7 ("arguments are + always stripped") and the earlier "undeclared argument fields are stripped" (step 2); `checkArgs` is one compiled + validation. +- **Q15, `local` arguments are validated (owner, 2026-10-21; review round 3):** one rule for every method call: the + provider validates arguments against its closed schema (Q12) for `local` types too (callers are often other plugins, + bundled without type checks; `local` stores hold persisted data, P3). Never copied. Supersedes the `local` column + "neither validated nor copied" (design §5) and the `local` part of "Object kinds". +- **Q13, origin incarnation on relayed calls (owner, 2026-10-21; review round 3):** relays stamp the origin host's + incarnation into the `call`/`hold` header, taken from the link the call came in on and overwriting any value a client + sent (like the principal), so `ctx.incarnation` always names the host where the call started. One header field. + Refines R20. +- **Q14, exposable state (owner, 2026-10-21; review round 3):** a state member of a contract may be backed only by a + JSON readable: a document handle (persisted or in memory), a derive, `constant(...)` or a state readable + (`replicatedState`, e.g. Durable's views). `self.writable` (host-local, any in-process value, for registry + contributions) is refused as a state member: by type, and when the object is served (a kind tag check, no value + walk; the object fails `broken` naming the member). Values are still not validated (R9). +- **`reset` instead of release and re-hold (owner, 2026-10-21):** (**superseded in part** by "Flow control: an ack-clocked window": consumers never fall behind a window; only a relay sends `reset { hold }` upstream, for a downstream that fell behind; no debt or minimum interval) when a hold falls behind (the link's window + overflows), the consumer or relay sends `reset(hold)` on the existing hold, never a release: the reference count is + unchanged, so a slow connection can never end an object. The provider answers with `bound` (a fresh snapshot) and + restarts that hold's encoder; the consumer drops everything for the hold that arrived before the new `bound` + (per-direction order). Q3's rules stay (one snapshot in flight per hold, after the previous debt drained, minimum + interval). A link drop remains a departure. Supersedes "releases the hold and re-holds" (R19, design §3). +- **Routines get the changes since their last run (owner, 2026-10-21):** (**superseded** by "Watchable and one watch API": per-commit frames) `self.routines.on(readable, (value, changes, + ctx) => …)` runs one call at a time with the latest value; `changes` is `{ kind: "ops", ops }` (every op since the + previous run, in order, the commits' op arrays kept by reference, never re-diffed) or `{ kind: "reset" }` (the first + run, or more than N ops piled up). Same rule as Durable's watch overflow and the link's `reset`, applied per run; + routines that only want the value ignore `changes`. +- **Watchable and one watch API (owner, 2026-10-21; **superseded in part** by W1 (flush after the step), W2b (routines + start with the current value), W4 (two kinds) and "One pending queue for slow consumers of state" (the 100-frame + overflow)):** v2 has no `replicatedState`. Everything observable is a + `Watchable` = `{ value; watch(ctx): Promise> }` with `T` limited to JSON plus bytes at the type + level: document handles (singletons; keyed per key), derives, reference members, `constant(v)`, and Durable's + conversation view, task graph and documents. Durable's `WatchHandle` and `WatchEnd` move into Chord as the one + observation API and Durable imports them: `watch()` atomically takes the current value and buffers later commits; + `start(listener)` installs the one listener; each commit is delivered as `(value, ops, ctx)`, in order, async + listeners awaited; after 100 pending frames the backlog becomes one root replacement `[["r", newest]]`; a retired + document ends with `[["r", null]]`; `stop()` is idempotent, drops pending frames, never aborts or awaits a running + call, and returns the end; `closed` resolves to the same. `WatchEnd` reasons are generic: `stopped`, `cancelled`, + `retired`, `ended` (source closed: object ended, reference released, Session closed), `listener_error { error }`, + `failed { error }` (source failed); Durable adopts them (was `session_closed`, `session_failed`). `routines.on` is a + watch owned by the object (per-commit frames; Chord awaits a running call on reload), replacing "Routines get the + changes since their last run". Durable's state getters (`documentState`, `viewState`, `taskGraph()`) are removed + (unused by p2). Derives and the wire encoder use an internal synchronous hook inside the commit step so a derive's + change travels in the same wire batch as its source (no torn state); this needs exhaustive tests. Memory: each watch + holds at most 100 frames by reference; every watch ends (object end, reload, ctx, source end); GC tests count + retained revisions. +- **W4, two watchable kinds (owner, 2026-10-21; both reviewers agree; review round 4; JSON-plus-bytes **superseded in part** by "L2 spec decisions": watchables are JSON only):** Delta's `Op` is typed over + JSON, so natives never ride in ops. `Watchable` (JSON plus bytes): `value` + `watch(ctx)` → + `WatchHandle`, listener `(value, ops, ctx)`, overflow → one `[["r", newest]]` frame; documents, derives, reference + members, `constant`, Durable's views; the only kind that can be a contract state member or go on the wire. + `NativeWatchable` (any in-process value): `value` + `watch(ctx)` → `NativeWatchHandle`, listener `(value, + ctx)`, a change is a new value by identity (`!==`; mutating an existing native value notifies nobody), overflow + collapses to the newest value, never ops; `self.writable(initial)` (with `set`) and composed registry values. Both + share one lifecycle implementation (`watch`, `start`, `stop`, `closed`, `WatchEnd`) and carry explicit runtime kind + tags; a `NativeWatchable` is rejected as contract state even when its current value happens to be JSON. Derives may + read either kind (mixed sources) and always return JSON plus bytes: a native change marks the derive dirty without + ops; the derive recomputes and its result is diffed as usual. Native changes use the same synchronous dirty-marking + hook (W1) with the same reentrancy flag; `writable.set` never goes through the commit queue and has no seq; + registries recompose in the owner's flush, never inside another owner's commit step. Supersedes the JSON limit in + "Watchable and one watch API" and the earlier `Writable` interface. +- **W1, adopt and mark in the step, one synchronous flush after it (owner, 2026-10-21; review round 4):** the commit + step writes, adopts (pointer swap of every changed value: the owner's documents; Durable's views and documents + touched by the commit, dispatched by conversation/document id) and marks (dependent derives dirty; ops appended by + reference to each touched hold's batch and each watch's queue), O(1) per dependent. Right after the step, before any + other code runs and without any await, the owner's flush recomputes the dirty derives that someone holds or watches, + in dependency order (derives are synchronous pure functions), encodes one part per touched hold (a hold covers all + members of the held object's variant) into one state message per link, and queues watch callbacks. A derive that + throws fails only its member (`broken`); the commit and its sources' parts still go out. A dirty derive nobody + flushes recomputes lazily on its next read. During adopt and flush, `self.tx`, `watch` and `release` throw + `reentrant`; a derive cycle throws once. The same adopt/mark/flush boundary runs without a store seq for in-memory + commits (P2), `writable.set`, registry changes and inbound wire batches. Durable calls the adopt synchronously + inside its commit step (no microtask). Async work belongs in routines writing in-memory documents. Supersedes + "derives and the wire encoder run inside the commit step" ("Watchable and one watch API"). Exhaustive tests: a + throwing derive, a chain A→B→C where B throws, two owners alternating with a derive over both, reentrancy, a watch + started after 150 frames, stop during a running call, a retire with a derive over it, a reload during a routine call, + GC counts of retained revisions. +- **W2, W3, watch fixes (owner, 2026-10-21; review round 4):** W2a: retirement is an explicit source event (last frame + `[["r", null]]`, then the end `retired`), never inferred from `value === null`; in Chord and in Durable alike (a pi + change; Durable's `CommittedWatch` stops treating `null` as retirement). W2b: only `routines.on` delivers the + acquisition value: it calls its handler once with `watch.value` (ops `[["r", value]]`), then starts the watch and + drains the buffered commits; a direct `watch()`/`start()` user keeps Durable's pattern (use `watch.value` first, then + `start`), unchanged. W3: termination clears every reference a watch holds (listener, detach/report/replace closures, + cancellation hooks); a running call keeps only what it needs; GC tests cover stopped but still referenced handles. +- **W6, worker start order (owner, 2026-10-21; review round 4):** connect to the parent first, then `load`, then + `harness.resume()`. A link carries nothing until a hold or call; a host-lifetime store that references a server type + in its factory no longer waits 60 s for a missing link; requests arriving during `load` park like any call to a + loading object; no turn starts before every plugin loaded. +- **W7, store addresses carry no host instance (owner, 2026-10-21; review round 4):** automatic: Chord builds the owner + string of persisted documents and tables from the host kind and object type (`session/notes.store`), never from a host + address (no session id), so a split (S7) or an import under a new id keeps every document reachable; each session has + its own store, so the session is the store, not part of each address. Durable's owner strings (`session`, + `conversation:7`, `task:31`) already qualify. Plugin code never writes addresses; the guide says keys need no session + id. +- ~~**W8, every text column as UTF-16 BLOB (owner, 2026-10-21; review round 4):**~~ (**superseded** by "Text is plain + UTF-8, in code point order") every `str()` column, indexed or not, + is stored as a UTF-16 `BLOB` (high byte first), so text has one order everywhere (JavaScript's), adding an index never + changes results, and lone surrogates and NUL survive; `json(...)` columns stay JSON `TEXT`. The SQL generator casts + `concat`/`replacePrefix` results back to `BLOB` and uses only even byte offsets (whole code units); conformance tests + cover emoji prefixes, NUL and lone surrogates. Cost: twice the bytes for ASCII text, hex in `sqlite3`, ~0.4 µs per + encode. Supersedes S16's "only key and indexed text". Revisit note (owner): Armin is not happy with the UTF-16 + encoding; no better option is known yet. +- **W9, no compaction at first (owner, 2026-10-21; review round 4):** batches carry the ordered operations exactly as + committed; a replica applies them in order, as a whole, in one transaction (S5). "Keep the last write per row" is + dropped (it loses rows with `update` and bulk operations). A later compaction must merge `update`s into the preceding + `put` and treat bulk operations as barriers. Supersedes S6's compaction note and the book's compaction text. +- **W10, no reads after a bulk operation (owner, 2026-10-21; review round 4):** once a transaction has an + `updateWhere` or `deleteWhere` on a table, `tx.get` on that table throws `read_after_bulk` (no JS evaluator of `where` + and `set` on any backend); other tables and reads before the bulk operation are unaffected. Like Durable's + `ReadAfterWrite` (spec §4: no table read after the first table write), but narrower. The overlay for `put`, + `update` and `delete` stays (owner): helpers doing read-modify-write compose within one transaction. Durable keeps its + stricter rule (point reads could get the same overlay, scans could not; not needed today). +- **W11, two version levels for migrations (owner, 2026-10-21; review round 4):** each `json(...)` column has its own + version and a value-only migration (`json(Schema, { version, migrate: (old, from) => new })`), stored with its value; + it migrates on access, and a write of that column stores the new shape and its version (writes of other columns never + touch it). The row's `v` means only "eager steps applied up to v": only an eager step and a full-row `put` set it; + `update`, `updateWhere` and `deleteWhere` never do. An eager `rows` step applies its row function to rows with + `v < target` in key order with a cursor, writes the changed columns plus `v`, one commit per chunk; after a crash it + scans from the first key again, skipping rows with `v >= target` (one pass, no temporary index, no progress record). + No ordinary writes happen between chunks (the owner is not live until `ready`); old code refuses a newer row `v` or a + newer JSON column version independently; replicas apply the shipped writes and alters without code; a new index is + created after `ready`. Key changes are a recorded `rebuild`. Supersedes in part S17 ("each row stores its version `v`; + `migrate(row, from)` on read") and "Flat columns" (shape changes inside `json(...)`). +- **W12, W13 (owner, 2026-10-21; review round 4):** an in-memory commit (P2) takes the owner's next publication + number, never a store seq, never enters the commit queue, and runs adopt/mark/flush (W1) when it happens. Durable's id + counter at open is the maximum over all five of its tables (S10). One message per link per commit comes from the W1 + flush, also when one Durable commit touches several holds on a link. `count(q)` stays in the query runner although + S12 starts without aggregates. +- **W14, cuts (owner, 2026-10-21; review round 4):** (a) no IndexedDB backend: the browser runs SQLite in WebAssembly + (official build 3.54.1: `sqlite3.wasm` 879 KB, ~406 KB gzipped), persisting into IndexedDB (a storage adapter such as + wa-sqlite's) or the browser's private file system; serverless targets use SQLite where it exists (Durable Objects, + Turso, D1) and a Postgres backend (a second SQL generator for the query language) when a real target needs it; a + backend needs atomic transactions, ordered indexes, bytewise binary keys and one writer per session. FoundationDB was + considered (Armin): the right model (ordered key-value with transactions) but the wrong shape (an operated cluster, + no embedding, 5 s / 10 MB transaction limits, no self-contained session files); at most a later backend for a hosted + multi-tenant service. (b) The `replicatedState` family (`replicatedState()`, `ReplicatedStateSource`, + `AttachedReplicatedState`, the microtask scheduling in Durable's source attachment) is removed as public API in Chord + and Durable. (c) `stopped` and `cancelled` stay two end reasons. (d) Derive `previous` is removed (`mapShared` and + lazy recompute cover it). (e) The per-row `rows` migration step is the explicit exception (done by W11). +- **No document tags (owner, 2026-10-21; Fable's consistency check, Astra earlier):** `defineTag`, the `tags` option of + `tx.draft` and `retireByTag` are removed. They existed only to delete a session's data; with one store per session, + deleting the session deletes its store and directory, and the documents engine (S13) has no tag index. Instance data + on the server that refers to a session is a table with a `session` column, cleaned up by its owner (`deleteWhere`, + reconciling from the catalog, "invariants across owners"). +- **Principals stay unversioned (owner, 2026-10-22; L1 part 4):** the app's `P` is a plain base-code type with no + schema or version: Chord never persists principals, base code changes every host at once, links stamp principals. + Objects persist a projection with its own schema, never the principal itself (p2: chat's `Author { kind, id, name }` + from `authorOf(principal, ownerName)`, stored in every input entry); the guide states the rule. +- **Direct mappings stay; params checked by TypeBox (owner, 2026-10-22; L1 build):** `accepts` maps each accepted old + version directly to the current one (no chains in Chord; one argument mapping per call and one state projection per + commit per old version); when a new version arrives, the author updates every still-accepted version's mappings, and + the guide shows composing step functions (`(old) => v2to3(v1to2(old))`). Params are checked with a compiled TypeBox + schema (a closed object), so constraints such as `str({ pattern })` or `int({ minimum: 0 })` are allowed and + enforced; the type level and the declaration check allow only scalar helpers (`str`, `int`, `bool`, `oneOf`) without + nesting. Refines L1 part 2 ("field by field, no TypeBox"). +- **`bindProvider` (owner, 2026-10-22; L1 build):** `serve` (L4) calls `bindProvider(Type, accepts)` once per load and + reload: it runs `checkAccepts` once and keeps the validated mappings (frozen); per call `checkArgs(provider, method, + raw, boundVersion)` checks with the bound version's schema and maps directly to the current version. Condition + (owner): no leak. The binding is held only by the served type's registration and by calls in flight; no global + strong cache keyed by contracts or bindings (compiled schemas only in `WeakMap`s keyed by schema objects); a GC test + shows a binding, its mappings and compiled schemas are collected once dropped (reload) and the last call settled. +- **L1 review round 1 decisions (owner, 2026-10-22):** same-host arguments are not checked for plain data (no + prototype, getter or `unknown()`-subtree checks): the documented convention is that callers pass plain data (JSON + plus bytes), immutable by convention. Params are never copied or frozen: the object keeps the caller's params + object, immutable by convention (no freezing anywhere). A method without arguments takes `null`: dispatch passes + `null`, the README says so. Cuts (owner): the error reason type is a closed union of the decided reasons (later layers + add theirs; plugin errors never use Chord reasons, shape decided in L4); `failure.ts` and its activation helpers stay; + the `copyJson` wrapper goes; `Provider` keeps only its runtime brand; `revocationKeys` stays (**superseded** by "Simplification pass after 4a": cut until 4b). +- **L1 review round 2 decisions (owner, 2026-10-22):** error reasons are part of the link protocol: adding a reason + bumps Chord's protocol version (hosts with different versions refuse each other, an old tab reloads), so a receiver + keeps rejecting unknown reasons; a test pins the reason list with the protocol version. Failed checks take TypeBox's + public error list (the internal first-error wrapper is removed). The provider's dispatch table is private (a + provider-keyed `WeakMap`). Accepted versions' argument validators compile on first use. The closed-object argument + check runs once per declared or edited method. Owner rule: no runtime checks of values TypeScript already guarantees + (AGENTS.md); declaration checks of decided rules the types cannot express stay. +- **Cost counters (owner, 2026-10-22):** the function that does the work counts it through its context (validation, + compilation, canonical JSON, a future counted `copyJson` with an optional context added in pi); call sites never + count work done elsewhere. Counters stay plain totals without site labels: mainly a test guard (a test isolates one + operation), plus a rough production number; attribution, if ever needed, comes from an event on the context's span. + The error rebuild from the wire does not copy (fresh decoder output) and checks only the envelope. +- **Code style and L1 cuts (owner, 2026-10-22):** all Chord v2 code follows the section "Chord v2 Code Style" of the + root AGENTS.md (one concept per file, top-down order, doc comments, no flag parameters, no typed-value checks, weak + caches with GC tests), so every agent has it in context. Cuts: the params descriptor pre-check is removed (params and + arguments follow one rule: the compiled closed schema; wire params are parsed JSON, same-host callers pass plain data + by convention); `bindProvider` keeps the app's accepts table as is (immutable by convention), no copy or freeze. +- **L1 rewrite questions (owner, 2026-10-22):** `checkArgs` returns a tagged value, `{ kind: "method"; args; result? }` + (args in the current shape, `result` maps back for an old-version caller) or `{ kind: "call"; args; call }` (the + checked old args and the provider's replacement, invoked by L4); no second lookup function. Failed-check paths are + TypeBox's instance path unchanged; keys containing `/` or `~` make it ambiguous (a diagnostic; not worth recovering). + The JSON Schema `oneOf` keyword is rejected (the `oneOf()` helper emits `anyOf`); local accepts tables and + `Implementation` are flat. +- **L1 rewrite review (owner, 2026-10-22):** nothing in L1 is frozen: contract handles, schema snapshots, params + schemas and providers are Chord-owned data, immutable by convention like everything else; the snapshot copy stays + (it keeps validation in sync with the fingerprint when an app changes its schema objects). Supersedes "frozen" in Q10 + ("history private, deep-copied and frozen") and the frozen metadata in the round-1 and round-2 entries. Size accepted + (about 1,900 lines, 1,450 without blanks and comments; about 420 type-only). Failed checks say "unknown field" and + "missing" for those two cases, TypeBox's message otherwise. A malformed error envelope from a peer throws a plain + `Error`. `Principal` is a type alias (apps declare principals as type aliases, not interfaces). +- **One watch-handle implementation, upstream first (owner, 2026-10-22; L2 spec):** Durable's `CommittedWatch` moves + into Chord's shared `src/watch.ts` (pi change, upstream first), generalized to serve native watchables too + (`NativeWatchHandle`, listener `(value, context)`, overflow to the newest value); Durable uses it and its + `observation.ts` goes away; Chord v2's L2 builds on it. Publications, derives, sinks, `constant` and `writable` stay in + `next/` (v2 only). Durable's synchronous view-change hook (so a derive over a Durable view is marked in the commit + step) is a later pi change, with L3d. +- **L2 spec decisions (owner, 2026-10-22):** watchable state is JSON only (no bytes in documents, derives, reference + members or any watchable; bytes only in method arguments and results and chunked transfers; Delta stays JSON-only; + supersedes W4's "JSON plus bytes" for watchables). `mapShared` lives in `next/watch/` until v2 goes upstream. C10 is + rejected (`constant` and `writable` stay two kinds, Q14). One publication object per host, passed to its sources; + no module-level singleton. No CBOR: the wire stays JSON plus raw byte segments (R1, Q4), revisited only if L6 + benchmarks show encoding as a bottleneck. +- **Contexts are required (owner, 2026-10-22):** as in Durable (356 required context parameters, none defaulted; + optional only on error-reporting paths), every Chord v2 function that takes a context requires it: no + `= BACKGROUND_CONTEXT` defaults, no `context?`. A getter that cannot take one (a lazy derive read) uses its owner's + context (the publication's). Applied to L1 and L2; written into the AGENTS.md style section. L2 review: the + publication resets its phase and queues in a `finally`, so a throwing sink cannot leave it stuck; L2's size (about + 680 lines) is accepted. +- **Carry-over ledger (owner, 2026-10-22):** obligations deferred to later layers are recorded in + `packages/chord/docs/next/carry-over.md`, under the layer that must do them, by whoever defers them, in the same + change; a layer's spec includes its section; done entries are deleted. The AGENTS.md style section states the rule. +- **Vertical slices instead of layers (owner, 2026-10-22):** building strictly bottom-up made L2 grow generic machinery + for callers that did not exist yet (nodes, sinks, deliveries, "publications"), hard to follow; its code was deleted, + not kept. From now on Chord v2 is built in vertical slices: each slice is the thinnest end-to-end path through + every layer it needs (slice 1: one host, one local object with `self.state`, `self.tx`, `self.derive`, a checked + method call and a watch), so every API has a real caller. Same process per slice: spec by this session, owner + approval, an Opus 5.5 build, review, owner review, commit. L1 stays (stateless checks, committed with required + contexts and an `index.ts`/`internal.ts` split, `ff4f4456d`). Kept from L2: Chord's shared watch handle + (`src/watch.ts`) and the decisions (JSON-only state, one batch per host, no CBOR, C10 rejected). Plain names from + the design's vocabulary or common usage (a signals library's `batch`), never invented internal jargon in APIs. + Plan: `packages/chord/docs/next/slices.md`. +- **`Implementation` maps state members to watchables (owner, 2026-10-22; slice 1):** L1's `Implementation` types a + state member as `Watchable>` (the document handle, derive or reference member the factory returns), + not a getter `() => value`: a getter cannot tell Chord when a value changed (design §11). +- **Slice plan and slice-1 review decisions (owner, 2026-10-22):** slice order 1 local objects, 2 failures and + natives, 3 two hosts, 4 topology, 5 routines and jobs, 6 persistence, 7 reload and rollout, 8 registries and the + harness, 9 telemetry (routines and jobs before the store and reload). A feature map in `slices.md` assigns every + design feature to a slice; implementers and reviewers check the slice spec, the map and the design (AGENTS.md). + Slice process includes reviews by Astra and Fable. An object that ends rejects new calls, aborts its running calls' + contexts and makes its `self.tx` throw `released` (a running method can change nothing after the end); no drain + without natives (the drain, design §3, is decided with natives and C13 in slice 2). Keyed in-memory documents, + `mapShared` and `defineFacet

` stay in slice 1 (base infrastructure), each with a real use in its example. + `checkpointWhen` is required only for persisted documents (refines R14: in-memory documents are never written, so + checkpointing cannot apply). `retryOn` waits for the first independently living source (registries, slice 8); + slice 2 has `ref.retry()` and dependency recovery. +- **Slice 1 approved (owner, 2026-10-22):** cycle rejection on one host (Q5) is in slice 1; watches belong to the + reference they came through and end when it is released; at `get` the consumer's contract version and fingerprint + must equal the provider's (`contract_mismatch` otherwise); `then`, `release`, `state` and `retry` are reserved member + names, refused at declaration; cut C14 rejected (the `handle.change`/`handle.retire` shorthands are built); cut C16 (**superseded**: accepted on 2026-10-26, "Facet documents are removed") + rejected (facet documents are built: served objects may draft them in their own transaction). +- **Callers when an object ends (owner, 2026-10-22; slice-1 review):** as design §3 says, callers are rejected at + once: when an object ends, every caller of a running call gets `released` immediately; the handler keeps running in + the background under an aborted context and cannot commit (`self.tx` throws `released`). A call whose caller context + is already aborted is rejected before its handler runs. Each call runs under `withAbortSignal(objectEndingSignal, + callerContext)` (one controller per object; a caller's cancellation aborts only its own call). The frozen v2 kernel, + its tests, the playground built on it and the kernel step docs were removed (`4a54fc38b`); `Watchable` went upstream + into `src/watch.ts` (`a2fd25f56`). Facets are typed by exactly the host's principal type (invariant). +- **No derive names, no `null` for no-argument methods (owner, 2026-10-22; slice-1 review):** `self.derive(sources, + compute)` takes no name: the name existed for `previous` (removed, W14); on reload exposed derives are matched by their + member name, internal ones are rebuilt (slice 7 adds matching only if a real need appears). A method without + arguments omits them in the API: references call `ref.bump(ctx)` and implementations are `bump: async (ctx) => …`; + the shape is fixed per method by the contract; `null` remains the internal and wire form. Refines the L1 round-1 + decision "no-argument methods take `null`". +- **Slice 2 approved (owner, 2026-10-22):** natives with per-native readiness, ordered opens and reverse closes, close + reasons and a close context aborting at `closeTimeout`; failure causes by where they failed; every attempt bounded + by `readyTimeout`; backoff per identity (a map holding no object); only R18 failures keep references; gap-free + dependency handover on retries; the start barrier waits for each host-lifetime object's first outcome; `host.get`: + broken rejects at once, transient waits up to the getter's own deadline; the drain waits for handlers' own promises + before natives close. Cuts: C5 rejected (no natives in facets); C6: `broken()` kept, `retryOn` in slice 8; C7: + `linger` kept, built in slice 3; C13: two host timeouts (`closeTimeout`, `readyTimeout`), the drain uses + `closeTimeout`. `ref.state`, `ref.retry()` and `host.hold` move to slice 3; the plugin error shape is decided in + slice 3. Openers take a `Context` (not `({ signal })`). +- **Slice 3a approved (owner, 2026-10-22):** slice 3 is split into 3a (remote objects over one link) and 3b (surviving + drops: re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger`, `host.root`, `state: false`). Guiding + rule for every choice: simple, fast, correct. A remote factory returns `{ variant, members }`; `ref.variant` is the + discriminant (C15 accepted: no `inVariant`/`expectVariant`; `variant` reserved). Calls are checked against the + binding's variant. A hold's state is one object with one Delta encoder; failed members travel in the header. A + release rejects the calls made through that reference, locally and over links (refines slices 1-2). `$bytes` is the + one reserved key in values crossing a link (byte leaves travel as raw segments referenced by `{"$bytes": i}`). + Plugin errors: `PluginError(code, message, details?)`, `internal` for anything else, never a stack. Headers are + checked on every link until IPC (slice 4). Hot-path slices add light benchmarks (timing, memory, wire size) recorded + in `benchmarks.md`. +- **Wire protocol (owner, 2026-10-22; slice 3a review):** the protocol works on a typed `Message` (one TypeBox + schema, `MESSAGE`, also checked after every decode); codecs are pluggable per link: `binaryCodec` (kind byte, varints, + raw ids, JSON values; about 23 B for a streamed token instead of 98) and `textCodec` (Delta-style JSON tuples, no field + names), compared by benchmarks for wire size, compute and memory. Member, method and variant names travel as indexes + into the hold's bound version; a reload that drops that version ends its holds first (slice 7). Checks by layer: the + codec's structural decode, the `MESSAGE` envelope, the bound version (ids, indexes, order), then the owners + (`checkParams`, `checkArgs`, Delta's decoder). The memory link delivers with `queueMicrotask` (no Node APIs). A + 2,000-token reply drops from about 196 KiB to about 46 KiB. Supersedes R1/Q4's JSON header framing. +- **Slice 3b approved (owner, 2026-10-22):** references survive link drops (`loading`, values readable, calls queued + with their own deadline and sent in order, sent calls fail), one owner of remote references across links re-holding + on every `connect`, a re-hold may bind another variant; `ref.state` (`loading`/`live`/`failed`/`released`) is a + `NativeWatchable` (interface added to `src/watch.ts` upstream first); `host.hold`; remote retries through two new + messages `retry { hold }` and `loading { hold }`; parked calls wait at most `readyTimeout` (C13 keeps two host + timeouts); `linger` for live objects; C8 (`state: false`) cut; `host.root` moves to slice 8. +- **Flow control: an ack-clocked window (owner, 2026-10-23):** supersedes Q4's byte window, R19/Q3 (snapshot admission + and debt), C12's credit accounting and K16's call credits. Measured by Astra and Fable (Node 24, `ws` 8.22, separate + processes, proxy at 64 KiB/s): a `bufferedAmount` cap plus latest-state coalescing still delivered state 9.4 s old + (kernel and proxy buffers hide megabytes); application acks with 1-4 messages in flight plus coalescing delivered + 0.0 s old state, heap +0.4 MB, 98% fewer messages; no control buffered 7 MB (+18.5 MB heap; a client that stops + reading: 23.4 MiB, RSS +121 MB in 15 s). IPC `send()` returns false from about 64 KB yet 623k sends cost +1,035 MB; + its callback acknowledges the write, not the handler. Rules: + - At most K=2 unacked `bound`/`state` messages per link; the consumer sends a small `ack` after applying each one. + - Ops queued meanwhile are merged per hold; a snapshot replaces them only when the merged ops exceed the member's + value size (2,000 appends: 46,024 B one by one, 26,021 B merged, 8,035 B as a snapshot; but a member with 500 KB + of history would cost 508,035 B as a snapshot). + - Calls: a call window N (default 256) announced in `hello`; the caller parks extra calls in 3b's parked-call queue, + FIFO, refusing nothing (K16 kept). Exceeding N is a protocol error, enforced by the provider. 100k calls of 10 ms: + a caller-side cap of 64 took 17.1 s, +4 MB, another client's p95 0.66 ms; unlimited ran 43,824 at once (+56.5 MB, + p95 129 ms); a provider semaphore queued 96,736 on the provider. + - Holds are unlimited by count; size limits stay a separate check. Dropped: the byte window and credit messages, + snapshot admission and debt, C12's credit accounting, an adapter `backlog()` API, `reset` from the consumer. + - **Relays ack per hop (option b):** the server acks upstream on receipt. When a downstream tab falls behind, the + server drops that tab's queued parts and sends `reset { hold }` upstream; the provider answers with a fresh + snapshot of that hold (a second `bound`). One slow tab never coarsens the other tabs of the same worker link + (option a, end-to-end acks, rejected for that). About 40 more lines than (a). + - Open for the slice-4 spec: Astra's circuit breaker (a hard transport-buffer cap plus a stall or heartbeat + deadline that closes the link). + Explainer: `packages/chord/docs/next/flow-control/` (Radius artifact 01m5n82vsbe5rtn4nm6wzaq0mp). +- **One pending queue for slow consumers of state (owner, 2026-10-23):** refines the snapshot fallback of "Flow control: + an ack-clocked window" and supersedes "Watchable and one watch API"'s "after 100 pending frames the backlog becomes + one root replacement" for state watches. Measured by Astra and Fable (`/tmp/fc2-*.md`, `/tmp/fc3-*.md`; real p2 entries plus synthetic stress + cases): size-based rules fail ("snapshot bytes + sent op bytes" is no bound: an interned path lets a value grow + faster than its ops; it drifts 1,254x under churn; exact size needs per-node bookkeeping), a time trigger punishes a + stalled stream into a large value, and a count of unfolded commits resends whole histories. Folding alone came + within about 10% of an exact oracle. Rules: + - Pending ops stay raw (never encoded per commit while waiting); `r` drops everything before it. + - **Leaf index:** a map from path key to the last pending leaf op (`a`, `s` of a primitive); a new leaf op on a + path in the map merges into that op in place (`a`+`a` joined, `s` replaces `s` or `a`, `a` after a string `s` + appends into it). Any other op (`p`, `m`, `d`, `t`, `s` of an object or array) clears the map. Correct because + leaf ops on different paths commute, except an ancestor primitive `s`, after which the descendant cannot be + written until a structural write clears the map (fuzzed: about 1.8M tracker and `diffRevisions` ops, arrays with + splices and reorders, adversarial cases; a broken index without the clear on `p` fails). Cost about 0.2-0.4 us and + 60-80 B per pending leaf path. Ops come only from Chord's tracker and `diffRevisions`: no defences against + hand-written ops (no path-key normalization). No folding of list appends (1% gain, quadratic copies) or trims. + - **Cap: 10,000 pending ops after folding;** past it only "replace with the current value" stays pending (nothing + more accumulates). Worst case about 2.2-2.7 MB per member per slow consumer. No size or text caps, no time trigger. + - Used by link hold members while the K=2 window is full (encoded once when an ack frees a slot) and by state + watches (`Watch` over `(value, ops)`): a slow listener gets one call per batch with the folded ops and the newest + value. Measured, listener taking 1 s: 438 calls and 3.39 MB of ops become 171 and 371 KB. The watch change is in + Chord's shared `src/watch.ts` (pi first); tests and `examples/04-watches` change with the contract. + - Unchanged: Durable's event stream (`watchEvents` batches are events: they keep their own queue until removed), + `NativeWatch` (newest value), relays (drop + `reset { hold }`: parts are opaque). + Animation: `packages/chord/docs/next/flow-control/fold/` (Radius 01m5nc9s24e9kscppen2fxgwye). +- **Slice 4 split and first decisions (owner, 2026-10-23):** slice 4 is split into 4a (links and relays: IPC and + WebSocket links, trusted links, parts as opaque bytes, flow control, host kinds and placement, routes, relayed holds + and calls) and 4b (revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression); + draft with structural changes and races: `packages/chord/docs/next/slice-4.md`. Decided: trusted links are an + explicit option (`trusted: true` on `accept` and in `resolve`'s result; default untrusted; stamping only from the + parent link). ~~Resets and queue overflow travel as `[["r", value]]` ops in an ordinary `state` message~~ (**superseded** by + "Slice 4a review decisions": resets are a list-`bound`; overflow stays an `r` op): a relay's + `reset { holds }` (a list) is answered with one message carrying every reset hold's members as `r` ops (snapshots + taken in one step, applied in one batch), entries flagged `reset: true` in the header for relays; a second `bound` + only for slice 7's restarted objects. ~~Relayed calls are sent round-robin per origin while the worker's call window + is full (each origin keeps its order).~~ (**superseded** by "Relay flow control per origin": a relay forwards calls at + once; each origin has its own N at the provider). The Durable conversation view's size on re-hold is a known cost until pi + #11327 lands. +- **Relay flow control per origin (owner, 2026-10-24):** supersedes per-hop acks, `reset { holds }`, relay recovery + and id renumbering ("Flow control: an ack-clocked window" relay part, "Slice 4a review decisions" reset part). The + provider keeps the window (K=2), its pending queues and its call queue per **origin**; the relay never drops, folds or + resets. The relay assigns each downstream link an origin number (never reused on an upstream link; 0 = its own + work); upstream messages keep their ids and carry `origin` (read only on the parent link); the provider keys holds, + calls, windows and acks by (origin, id); replies carry the origin. The relay keeps per downstream hold its upstream + link and its original `hold` message (routing calls and releases; re-holds after a worker restart), and per downstream + link a FIFO of the upstream link of each message in flight (each in-order `ack` goes to the right worker). A new + message "origin closed" releases an origin's holds and calls when its link closes. A tab served by W workers may have + 2W messages in flight. Measured motivation: a slow tab through the relay needed ~24 s and 3 resets to catch up after a + 1 s stall (provider folding: 0.02-0.38 s). After the rework and the review fixes: a simplification pass with a budget + of 1,200 lines of net growth for 4a (owner). +- **Docs (owner, 2026-10-24):** `design.md` becomes the one current statement of all rules, updated in the same commit + as each decision; `mechanisms.md` folds into it, `carry-over.md` into `slices.md`; the decision log stays complete as + the history and the reasons; slice specs stay as history after their slice. Nothing is dropped in the move. +- **Slice 4b: `host.activity` is a watchable (owner, 2026-10-24):** a JSON `Watchable` of `{ references, calls, jobs, + lingering }`, published at most once per microtask turn with changes (design §3's "one readable"). +- **Docs consolidation (owner, 2026-10-24):** applied as decided in "Docs": `design.md` now states every current rule + (`mechanisms.md` folded in section by section, its 17 stale points fixed, then deleted); `carry-over.md` is the + section "Carry-over" of `slices.md` (AGENTS.md updated). Settled by what slices 1-4a built, as approved: resets do + not exist (per-origin flow control); trusted links skip only the `MESSAGE` check; a parked call waits its + reference's `readyTimeout` (no per-`get` `parkTimeout`); the object backoff is per identity; `host.get(Type, params, + context, options)` takes a context and `host.hold` returns at once and never rejects; a native's `close` gets + `(value, reason, context)`; a local object's handlers act as the host's principal. Still open, listed in design + §26 (#24-#29): the timeouts of later mechanisms, `host.root`'s signature, how a migration commit reaches consumers, + compression on browser links, the circuit breaker, cuts C3/C4/C11. +- **Simplification pass after 4a (owner, 2026-10-24):** two cutters (Opus 5.5, GPT-6.1 Sol) read all of `src/next` + without a line target (a target is not meaningful); the merged small cuts, two bug fixes (relays strip a client's + principal fields on every path; a closed link delivers no queued message) and D1-D4 are applied by a fresh builder. + D1: the unreachable "binding back to pending" branch of a served hold is cut from 4a; slice 7 builds it with reload + (carry-over). D2: L1's unused `revocationKeys` is cut; slice 4b writes revocation keys with their first user. D3: + `initial()` is checked once per definition at its first value (refines R8's "per document": `initial()` takes no + input, so a wrong shape shows at the first call); a failing keyed `initial()` fails only its transaction, a + singleton's fails the activation. D4: object instance ids (Q5) come with slice 4b, their first user; one-host cycles + follow the waiting objects without ids. +- **Slice 5 spec drafted (owner, 2026-10-24):** straight from the discussion to `slice-5.md` (the walkthrough + `routines-and-jobs.md` deleted, owner); design §14 and §15 state routines and jobs as decided; jobs in memory in + slice 5, persistence in 6, reload rules in 7 (carry-over); jobs as objects (#9) rejected. `tx.start` returns the + run's id (R12), `job.outcome(id, ctx)` waits by id; `unavailable/busy` is a new error reason. + On the draft: routines start right after the object went live (same turn); `abort(id)` drops a blocked run (no + `finish`, reported); `running` carries every run's args (views filter). Process: spec reviews, then the build. + After the spec reviews (Astra, Fable, Opus, Sol): `tx.start` returns a `Run` handle (its outcome rejects if the + transaction is discarded) instead of an id; `ctx.state.change(fn)` takes no context (changes while stopping after an + abort still commit); running and finishing runs are a count, not references, and an object without holders ends + (after `linger`) when its last run settles, dropping blocked runs; `finish` may chain the same job (its own run is + excluded from the concurrency check); a `run` throwing after an abort ends `aborted`; `job_failed` reported; + ~~routines also in facets~~ (**superseded** by "Facets only declare"); `persist: { resume: "rerun" | "abandon" }` is the final shape. + After the build reviews: the builder's ten points accepted (every object end reports dropped blocked runs; their + waiters get the end's error; `accepts` needs both mappings per older version; job args without bytes; routine labels + per kind; `start` before live is `not_ready`; a run cancels a linger; design fixes; rejected handles keep their stack + until read, documented); native `on` sources also get only the newest value after a busy call; the finishing run is + excluded from dedupe too (`finish` may start the same job with the same key); the retirement frame in `on` is a + slice-6 carry-over. +- **Routines and jobs open points (owner, 2026-10-24; walkthrough `routines-and-jobs.md` and its reviews):** at the + `concurrency` limit `start` rejects; the routine handover is bounded by `closeTimeout`; a persisted job's state + document gets the deferred persistence option (as in Durable), and authors throttle frequent state changes themselves + (Chord does not coalesce them: every change is a commit and a message). Jobs keep `accepts` (owner: everything + versioned has adapters; the walkthrough's "no job adapters" was never decided): a run started on an older version + finishes on its old code, its args and result are mapped to the current version for `finish` and for `running`, its + state document migrates through its own `migrate`; without an adapter the run is `blocked` and reported until a + reload brings one. So a version change aborts nothing and discards nothing. + Crash handling for persisted jobs (owner, with all four reviewers): a restart limit as Durable's (five crash + interruptions, counted and persisted, clean closes not counted; then parked as `blocked`, cleared by `abort` or a + reload); a `finish` that keeps throwing is retried with backoff and then `blocked` after five consecutive failures, + its outcome, state and key kept, retried on a reload; how a persisted run resumes after a crash (rerun, replay-safe, + or abandoned with an aborted outcome, wording aligned with Durable) is required, with no default. +- **Routines for slice 5 (owner, 2026-10-24; C4 decided):** two kinds, declared in the factory, no names and no + `routines` namespace: `self.on(source, fn)` (first call with the current value, W2b; then one call per change, calls + never overlap, changes meanwhile folded; a running call is not aborted by a change; a throwing call is reported and + called again with the current value after a backoff of 1 s doubling to 60 s, a change during the backoff calls at + once; the source ending ends the routine) and `self.onLive(fn)` (called once when the object goes live; returning + ends it until the next reload or activation; throwing is reported and restarts it after the same backoff, reset + after a run of a minute; an abort is no error). Both start when the object is live and stop when it ends (`ctx` + aborted, then the drain); on reload all old routine calls of the object finish (bounded) before the new routines + start: a handover per object, which needs no names. Routines are idempotent reconcilers; Chord keeps no state for + them; what must survive is a document. Helpers `sleep(ms, ctx)` and `untilAborted(ctx)`; routines and handlers wait + only through calls that take `ctx`. Errors: `HostError` `routine_failed { object, routine, error }` (`routine` = + kind and position). Routines do not count in `host.activity`. Dropped: `every` (an `onLive` with `sleep`; a due + time that must survive is a document), `custom` and `run(fn)` (an `onLive` with `try`/`finally`), names, the due + time across reloads. Kept separate from jobs (different lifecycle, reload behaviour and output). Open: the + handover's bound (`closeTimeout`, whose default may drop from 30 s to 10 s, or its own). +- **Slice 6 first decisions (owner, 2026-10-26):** (1) **Durable's internal commit observers run inside `adopt`** + (pi change, upstream first, built and reviewed before the slice): the Session's `observeCommits` listeners (view + mounts, task graph, scheduler, submissions) move from `#publish`, after `Storage.commit` returned, into the `adopt` + callback, so every Durable view changes inside the storage's commit step; with `StoreStorage` Durable's `adopt` runs + inside the host's batch, and Chord follows Durable's views as sources marked in the step (W1). Reason: today a + derive over two Durable views (conversation view and task graph), fed by two watches, computes between their frames + of one Durable commit ("tool call issued, no task running"). `adopt` stays throw-free (a listener's throw is kept and + fails the Session after `Storage.commit` returned, as today); `subscribeCommits` and telemetry events stay in + `#publish`; nothing changes for users of Durable's own storages. (2) **Persisted singletons load sequentially** right + after the factory, all singletons of an object in one read transaction (S4), before its references are bound; a + storage error is `storage_failed`, a fingerprint mismatch, a throwing `migrate` or an invalid migration result is + `broken`. (3) **Durable's ids in the store:** `StoreStorage` keeps a `record_ids` table (Durable's one namespace + across its five tables); the counter at open is `coalesce(max(id), 1) + 1` over it (1 is the root conversation). (4) + **Few statements per commit:** the store keeps seq and head in memory (read at open, the head persisted with each + commit, S9) and folds the documents engine's checks into the writes (a unique `record_ids` key, a partial unique + index on the live address, `UPDATE … WHERE id = ? AND version = ?`), so a delta commit is about three statements + (revision, version, outbox); `StoreStorage` maps constraint errors to the messages Durable's conformance suite + expects. A benchmark compares `SqliteStorage` and `StoreStorage` on the same workload. Still open: the split of slice + 6, and retirement in watches (point 3 of the discussion: no `null` frame, Chord documents that retire). + **Refined (owner, 2026-10-26), after four reviews of the first build:** the first build moved only Durable's internal + observers into `adopt` and deferred host-waking work (`afterPublish`); the reviews found host-visible regressions in + the gap between `adopt` and the end of `Storage.commit` (watch listeners before `subscribeCommits`, `waitForIdle` + early, a late subscriber missing a commit, the active-task metric drifting after a rejection). Under review now, + nothing built: "publication at adoption" (the whole publication, host `subscribeCommits` listeners and telemetry + events included, inside `adopt`; no `afterPublish`; `adopt` runs no task code, so a marked run's `abort()` is + queued). **No Durable special cases in Chord's store** (owner): the store never knows or waits for Durable's + continuation. + **Decided (owner, 2026-10-26), after four checks of the proposal (`/tmp/observers-proposal-*.md`, explainer + `packages/chord/docs/next/durable-observers/`):** publication at adoption in Opus's variant. Inside `adopt`, in + HEAD's order: the trackers, the telemetry events, the queueing of one microtask that dispatches the host + `subscribeCommits` listeners (the listener list read inside that microtask), the internal observers. Microtasks run + in order, so host listeners still run before every wait continuation, abort and `onWake` of the commit (before + every watch delivery too, as first stated: **superseded**, see "Superseded (owner, 2026-10-26)" below), but after the storage's synchronous step, so never inside Chord's batch (Chord calls from them work; Chord + needs only its step and flush to be synchronous). `afterPublish` is dropped. `adopt` runs no task code (a newly + marked run's `abort()` is queued) and is fully throw-free (listeners, reports, metrics and telemetry contained; an + internal observer's error fails the Session after `Storage.commit` returned). Accepted changes from HEAD: abort + listeners fire after host listeners; a commit the storage rejects after `adopt` is published (it is durable; a + replica stays equal to storage; spec invariant 2 changes) and its waits resolve before the Session fails; with a + storage that works after `adopt` (JSONL) the publication runs before `Storage.commit` resolves; a throwing telemetry + call is reported as `host_error` with callback `"telemetry"` and no longer rejects a durable commit. Nothing in pi or + p2 depends on the old orders (checked: p2's three `subscribeCommits` users). + **After the reviews of the second build (owner, 2026-10-26):** one delivery rule for every watch: a watch delivers a + frame in a microtask queued after the frame arrived, also when its listener was busy (Chord's shared `src/watch.ts`: + after a call the drain reschedules instead of taking the next frame in place). Without it a busy watch's + continuation, queued before `adopt`, delivered a commit before its host listeners (Astra, Opus). Cost measured: + about 170 ns per back-to-back delivery with a promise, less with a plain `queueMicrotask`; only busy watches pay it. + Also: a host listener added between `adopt` and the dispatch gets the commit only when a listener existed at + `adopt` (documented, accepted); a rejection after `adopt` keeps the storage error as the failure and still reports an + observer's error. + **Superseded (owner, 2026-10-26), after the third reviews:** the delivery rule did not hold (Astra, Opus: a delivery + already queued folds frames that arrive later, so a watch can still deliver a commit before its host listeners, also + in a shared commit step chained through promises); a complete fix (a late delivery queues itself again) would stall + watchers behind a source that changes in every microtask. Decided instead (option b): **no ordering promise between + host `subscribeCommits` listeners and watch listeners** (independent observers; nothing in pi or p2 depends on their + order); waits and aborts still follow the host listeners by construction; the hop in `src/watch.ts` is reverted. +- **Durable's synchronous observers (owner, 2026-10-26):** `observeConversation(id, observer, ctx)`, + `observeTaskGraph(observer, ctx)` and `observeDoc(token, …, observer, ctx)` become Durable's public primitive: the + observer's `advance(change, ctx)` (the listener bag `{ value, ops, retired }`) runs synchronously inside `adopt`, + `end(failure)` when the Session closes or fails; attaching returns the current value and `detach`, taken atomically + on the line. `watchXXX` stays, built on it (a `Watch` is such an observer), for app code; `observeXXX` is documented + for adapters (record only: no awaits, no Session calls, no throws, quick). Chord's `durableView` (slice 6b) is a + source over it, marked inside Chord's batch in the store and grouped into one batch per Durable commit on the next + microtask outside it. + **After the reviews of its build (owner, 2026-10-26: fix everything to HEAD's level of correctness, performance and + memory; the owner delegated the details):** `onWake` is no longer ordered after host listeners (same cause as + watches); `Watch.endAfterDelivery()` exists (a task document's watch delivers its retirement, then ends `ended`), + detaching only when the watch really ends and never while a listener call runs; the `observeXXX` context covers the + attachment only; observers get `end(undefined)` when their task document's task ends; the change object a listener + gets carries no context (the context is a separate field of the watch); a watch keeps only its address, never a + loaded document; `observeXXX` rejects like `watchXXX`; Durable's file-watcher type in `/env` is renamed + `FileChange` so `WatchChange` names only the listener bag. +- **One retirement mechanism for Chord and Durable, watches follow the address (owner, 2026-10-26; option A):** + supersedes W2a's last frame `[["r", null]]` and the end reason `retired`, and Q11's silent reset as the only meaning + of retiring a singleton. A document lives at an address (Chord: definition and key; Durable: kind, scope and key); + every document, singleton or keyed, Chord or Durable, is watchable; a retirement is signalled to the watch listener + as one last call with the last good value, no ops and a `retired` flag (no `null` anywhere); the watch goes on, and a + later write delivers the new incarnation as `[["r", value]]` on the same watch. A watch ends when its owner ends + (`ended`): a Chord document's object, a Durable task document's task (the retirement call, then `ended`, in the same + commit), the Session; and by `stop`, cancellation, failure or a throwing listener. Durable's watches change from + incarnation to address (breaking; nothing in pi or p2 watches a document that retires). Leaks: none new for task + documents (they end with their task and invocation); a forgotten watch of a session or conversation document that + is retired for good stays until its taker's context, `stop` or the Session ends it (GC tests). Still to decide: the + shape of the flag, the value an absent address shows to members, derives and `.value`, and the per-key source in a + factory. + **Details decided (owner, 2026-10-26):** (1) a watch listener (and an `on` handler) takes one object before the + context, `(change, ctx)`: `{ value, ops, retired }` for JSON watches, `{ value }` for native ones (a breaking change + of the shared `WatchHandle`/`NativeWatchHandle` and of `self.on`, accepted). (2) A retirement call carries the + address's value after the retirement: `initial()` for a Chord document (as Q11 reads an absent singleton), so + `watch.value` equals the source's value for every reader; for Durable, which has no value for an absent document, + the last value with no ops. (4) Watching an absent address gives `undefined` (`notes.watch(key, ctx)` as Durable's + `watchDoc`). The wire carries a `retired` list per hold entry in `state`, like `failed`. (3) **A per-key source in a + factory:** `handle.at(key)` declares a singleton per key: loaded at activation with the object's singletons, it reads + `initial()` while the key is absent, and is a state member, a derive source and an `on` source; its watchers learn + of a retirement through the flag. Telling "absent" from "empty" is `await handle.get(key)` (`undefined`) or the + flag. Not adopted: the key as an activation dependency (fails when absent, ends `gone` on retirement). +- **Text is plain UTF-8, in code point order (owner, 2026-10-26; reviews `/tmp/text-review-{astra,fable,opus,sol}.md`):** + supersedes S16's JavaScript order and W8's UTF-16 BLOB. `str()` columns are plain UTF-8 `TEXT` (`STRICT` tables; + strings bound and read as they are), and text compares by Unicode code point on every backend (SQLite `BINARY`, + Durable Objects, sqlite-wasm; Postgres `COLLATE "C"`, MySQL `utf8mb4_0900_bin` later). Written values and every + query operand (`where` values, `inArray`, prefixes, cursors, `set` operands) of a `str()` column go through + `toWellFormed()`: a lone surrogate becomes U+FFFD (what `TextEncoder` and Node's SQLite binding do), documented; a + value containing NUL fails `invalid/args` (Node 24.4.1 truncates text at NUL on read; Postgres refuses it; content + goes into `json(...)` columns, which escape both). A leading U+FEFF stays allowed: sqlite-wasm's reader drops it, so + its backend reads values starting with `EF BB BF` as bytes and decodes them keeping it. The store exports + `compareText(a, b)`; the memory backend uses it for ordering, ranges, `max`/`min` and `when`; a prefix's upper bound is + the next code point after its last (portable); `substr` offsets count code points; a SQLite file must use the UTF-8 + encoding (checked at open). The guide states the rule: store order differs from JS `sort()` only for U+E000-U+FFFF + against characters above U+FFFF; sort store text with `compareText`, page with the store's cursors, use `prefix()`, + cut strings with `[...s].slice()` or `toWellFormed()`. Durable's strings in `StoreStorage` (`requestId`, kinds) keep + Durable's JSON encoding in indexed columns, so Durable stays lossless. Rejected: CESU-8 in `TEXT` (a read path of + special cases on best-effort SQLite behaviour; Postgres refuses it), CESU-8 in a BLOB, keeping W8. +- **Slice 6 is split (owner, 2026-10-26):** 6a, the store with Chord's persisted documents and tables (backends, + tables, query runner, writes, commit queue, outbox, documents engine, index definitions, migrations, export and + import, `self.table`, persisted `self.state`, P3, the host's store option, `storage_failed`, W12, keyed watches and + retirement); 6b, what builds on it (persisted jobs, `StoreStorage` with Durable's conformance suite, the Chord source + over Durable's views). The API points (the host option's name, how a Chord transaction enters the commit step, the + table API in `self.tx`, the Durable view source, the commit benchmark) are proposed in the 6a and 6b specs. +- **The host's store and how it is opened (owner, 2026-10-26; slice 6a point 1):** the host option is `store` + (`createHost({ …, store })`), not `storage`: it is the store itself, and in 6b Durable's `Storage` is `StoreStorage` + over the same store. One entry point, `openStore(backend, ctx)`, with `sqliteBackend(db)` over any `SqliteDatabase` + (Node through `chord/sqlite/node`, Durable Objects, sqlite-wasm later) and `memoryBackend()` for tests; the backend + interface stays internal; `importStore(backend, archive, ctx)` takes the same values. `openStore` runs the store's + own migrations, checks the file is UTF-8, and reads the head, the definitions and the id counters once. The app owns + the store's lifetime: the host never closes it (in 6b the host and the Durable Session share it); the app calls + `store.close()` after `host.close(ctx)`; on `storage_failed` the store stops, the host closes, and the app closes or + reopens. No `openNodeStore(path)` shortcut. + + ```ts + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, "session.sqlite"))), ctx); + const worker = createHost

({ address: Session({ id }), principal, store, onError }); + … + await worker.close(ctx); + await store.close(); + ``` +- **A Chord transaction enters the commit step as Durable's does (owner, 2026-10-26; slice 6a point 2: "if this is + Durable's proven flow we should use it here too"; checked against Durable's `#runCommit`):** the object's line ~~(and + the facet's)~~ (facet documents removed the same day) is held from the callback until the step returned; an abort or error before the step discards + everything; then every draft is prepared, `checkpointWhen` picks base or delta per persisted document, and + `await store.commit(writes, adopt, ctx)` runs without the caller's abort signal (once admitted, a commit runs to its + end). `adopt(seq)` is one `host.batch.commit(...)` adopting the persisted documents, the in-memory documents and the + staged changes together (Chord's addition: Durable has no in-memory documents; adopting them apart would show half + a transaction, or publish an in-memory change of a commit SQLite then refused). An error before `COMMIT` returned + rejects the transaction with nothing adopted; an error during or after `COMMIT`, or a failed `adopt`, is + `storage_failed` (the commit stays published if it was adopted). A transaction with only in-memory writes never + enters the queue and commits synchronously as before (W12): no extra await or promise. + + ``` + line -> fn -> (abort/error: discard) -> prepare, checkpointWhen + -> await store.commit(writes, adopt) // queue; BEGIN; revision, version, rows, outbox, head; COMMIT + adopt(seq) = host.batch.commit(() => { persisted; in-memory; staged }) + -> release line(s) -> settle staged -> drafts revoked + ``` +- **Facet documents are removed; persisted data stays in objects (owner, 2026-10-26; slice 6a point 5; reviews + `/tmp/facet-persist-review-{astra,fable,opus,sol}.md`):** cut C16 accepted (supersedes its rejection in "Slice 1 + approved"): a facet has no documents (`FacetSelf.state` goes), so no facet line, no lock order across two lines, no + `Documents.facet` and no `Transaction.#facetLine`; a served object's transaction covers only its own object. Shared + plugin state lives in a `local` store object with empty params, reached through its methods (its own failure domain, + R18 recovers its dependents). Point 5 (persisted facet data) is moot: persisted documents and tables stay only in + `local` objects with empty params (P3 unchanged). Why: the four reviewers found no stated reason in the history (the + frozen kernel let facets persist; P3 was written for objects only), but a facet cannot fail at runtime (slice 2: a + facet setup or `initial()` that throws rejects `load`, closing the host; C5 was rejected because a plugin's always-on + parts would share one failure domain), and persisted data fails at runtime (fingerprint mismatch, throwing `migrate`, + a newer version after a rollback, no store on the host): one plugin's stale document would close the worker, or need + a second failable lifecycle for facets. In-memory facet documents had no real use either: none of p2's 46 `self.state` + calls is in a facet; their only ability, one transaction over an object's and its facet's documents, is a + cross-owner transaction, which the design rejects elsewhere (§9 "Rules across owners"). Other findings kept for a + later facet lifecycle: reads of shared tables need the writer's line; an owner string `facet:` collides with + type names. Built in slice 6a (carry-over). +- **The table API in `self.tx` (owner, 2026-10-26; slice 6a point 3):** (a) the first argument is the table handle + from `self.table(Def)`, never the definition (the handle belongs to one object, so `tx` refuses another object's + table, as it refuses its documents); (b) keys are objects with exactly the key columns, `{ path, entry }`, not tuples + (two swapped `str` columns of a tuple are silent at the type level; other or missing fields fail `invalid/args`, a + trust boundary); (c) reads are async (`await tx.get(t, key)`, `tx.page(t, q, limit, { after? })`, `tx.rows(t, q)`), + writes synchronous and buffered like a draft's ops (`tx.put(t, row)`, `tx.update(t, key, set)`, `tx.delete(t, key)`, + `tx.updateWhere(t, where, set)`, `tx.deleteWhere(t, where)`): they check the row and stage it, no I/O, no line, so a + bad row throws at its own call; the buffer goes into the commit step's backend transaction or is dropped on failure, + like aborted drafts (writing at once would hold SQLite's one write transaction for the whole callback). As Durable + (`setTask`, `settleSubmission` synchronous, reads async) and Chord's `tx.start`. (d) No write shorthands on the table + handle (`edits.put(row, ctx)`) until real code repeats the one-liner. Outside a transaction the handle reads + committed state: `edits.get(key, ctx)`, `edits.page(q, limit, ctx, { after? })`, `edits.rows(q, ctx)`, + `edits.count(q, ctx)`. `tx.get` exists for tables only (documents keep `tx.draft`). + + ```ts + self.tx(async (tx) => { + const old = await tx.get(edits, { id }); + tx.put(edits, { ...row, at: now }); + tx.updateWhere(edits, (e) => prefix(e.path, from), { path: replacePrefix(from, to) }); + }, ctx); + ``` +- **Chunks of `rows` migrations (owner, 2026-10-26; slice 6a point 6):** 100 rows per commit, a fixed constant. + Measured on slayer (`node:sqlite`, WAL, 20,000 rows of about 3 KB, a primary-key `UPDATE` plus an outbox `INSERT` + per row): the total is flat from 100 to 1000 rows per chunk (about 15 µs per row); the step's median is 0.94 ms at + 100, 2.4 ms at 250, 9 ms at 500, 14.5 ms at 1000, 32 ms at 2000, and every other owner's commit waits for it. The + store benchmark measures it again with the real store. Can change later (owner), e.g. a table definition naming its + own chunk size; not built until a case needs it. +- **`lastSeq` instead of "head" (owner, 2026-10-26):** the seq of a store's last commit is called `lastSeq`, never + "head" (Durable's "head" is a conversation's head: head markers, `findLatestHeadMarker`). Every "head" of the store + in earlier entries (S3, S4, S5, S9, "Slice 6 first decisions" (4), "The host's store and how it is opened", the + commit-step trace) reads `lastSeq`; a replica's "applied head" is its `lastSeq`. Renamed in design.md, the store + book and the slice 6a spec. +- **Export archives are operations, the encoding is pluggable (owner, 2026-10-26; slice 6a point 7):** + `store.export(ctx)` yields `AsyncIterable`: the state after its `lastSeq`, written in the outbox's + operations as one synthetic batch from 0 to `lastSeq` (a manifest `{ kind, format: 1, lastSeq, schema }`; `table`, + `index`, `put` per row; `document.create` per live record with its base, plus its later `document.change` records + for `history: "full"` ones (Durable, 6b); an `end` record with counts). Retired documents, blobs, the outbox and sync + state are not in it. `importStore(backend, records, ctx)` refuses a non-empty backend, an unknown format, records out + of order and a missing or wrong `end`, and applies everything in one backend transaction with the replica's applier + (import, bootstrap and batch apply are one code path). Export holds the commit queue until the iteration ends, + throws, is closed or `ctx` aborts (a slow consumer stalls commits; the guide: export to a local file, then upload; + snapshot reads on a second connection only if exports get large, S4). The encoding is pluggable, as the link's + codecs: the store yields and takes records only and ships plain JSONL (`encodeJsonl`, `decodeJsonl`) with no size + optimizations; gzip or a Delta-style dictionary encoder come later as formats without touching the applier (owner: + "then it doesn't matter that the first thing we do is plain JSONL"). Sync is not export: it ships outbox batches as + commits happen (S5); a full archive is needed only where no history can be replayed (a session moved as a file, a + split-off session, a replica restored after divergence). Open for the sync layer: a session that never syncs keeps + every outbox record (trimmed only after Radius acknowledges it), so its outbox grows without bound. +- **State and job names are restricted too (2026-10-27, slice 6a fix round, under the owner's delegation):** + `defineState` and `defineJob` names follow the pattern of object type and table names, `[a-z0-9@/._~-]+`, lowercase, + checked at declaration: they become the documents engine's `kind` and keys of job records, where NUL or lone + surrogates would be truncated or replaced by SQLite. +- **Slice 6a build reviews (2026-10-27, under the owner's delegation to the reviewers' majority):** two rounds of four + build reviews (`/tmp/s6a-build-review{,2}-{astra,fable,opus,sol}.md`), merged into two fix lists + (`/tmp/s6a-fix{1,2}-brief.md`, 28 and 16 items) by majority, a point raised by one reviewer taking that reviewer's + recommendation. Rules they settle beyond fixes of the spec: (1) **members across attempts** (**superseded in part** by "References are the only indirection": the reference, not a member source on the object, follows the object across instances): a local object's state + member is followed by its references and by other objects' derives through one member source per object and member + name; when a new attempt goes live every member re-binds by name (one diff frame from the last value, consumers' + derives recompute, a member the new variant lacks fails `invalid/member`), as a remote reference re-holds and gets a + fresh snapshot (Opus B1); supersedes slice-6a §1.2's "derive members' watches end with their attempt". (2) While an + object that was live is failed or activating again, same-host reads and new watches return the last values and get + no frames (was `not_ready` while activating again). (3) A live failure freezes the members only after the commits + admitted before the seal adopted and published once; a retry's factory and hydration wait for that, so no old commit + lands in a new attempt. (4) Re-exposed and forwarded members freeze at the re-exposing object; the provider's source + stays untouched. (5) A transaction cut at the seal while queued for the step rejects `released` at once; line jobs + queued before the seal are refused. (6) The seq, document id and revision counters stop at + `Number.MAX_SAFE_INTEGER` (`storage_failed`). (7) `evolveTable` checks each edit when it is made; import checks + `meta` with the text rules; an export never started rejects `released` after `store.close()`. (8) The migration + yield between chunks: `setImmediate` where the runtime has it, otherwise a `MessageChannel` message, never + `setTimeout(0)` (clamped to 1 ms in Node; a message costs about 0.27 ms per yield there: a 10,000-row migration takes + 70 ms against 100 ms and 191 ms); supersedes round 1's `setTimeout(…, 0)`. (9) Round 1: keyed keys follow the + `str()` rules (the empty key valid); same-host reads of a failed live object return the last value; stuck handlers' + counts, references and presence released at teardown; refusing a new job version on a live object kept for 6a + (slice 7 carry-over); in-memory keyed slots migrate on access. Measured against HEAD (`benchmarks.md`): calls 1-3% + faster (19 promises per call against 20, guarded), watches through a reference at HEAD's level, a document member's + read through a reference 0.25 ns slower, host setup about 3% slower. +- **Derives heal on a re-bind (owner, 2026-10-27: "a all the way, self healing is good"; Opus's F2 in the third + build review):** a consumer's derive that failed on a provider member's earlier error recomputes when that member + re-binds to the provider's new attempt (and every derive downstream of it), so it recovers once its source works + again; watches over it get the recovered value as a fresh frame. Before, a failed derive ignored later changes (as at + HEAD) and recovered only when its own object restarted. +- **Failures are watch calls; derives heal on every change (owner, 2026-10-27: "B", "beautiful, let's do it"):** + extends "Derives heal on a re-bind". (1) A failed derive recomputes on every later change of its sources (and on a + re-bind), so it heals as soon as it gets a value it can compute. (2) A watch stays open through a recoverable failure: + the listener gets one tagged call `{ kind: "failed", error }`, and the next good value as `{ kind: "value", value, + ops: [["r", value]], retired }`; the call is a tagged union (`WatchCall`: `value | failed`; native watches + `NativeWatchCall`: `{ kind: "value", value } | { kind: "failed", error }`), so the compiler makes every listener + handle the failure (an optional `error` field would let `({ value }) => render(value)` render the old value). + Supersedes the listener bags `({ value, ops, retired }, ctx)` and `({ value }, ctx)` of "One retirement mechanism" + (1). (3) Applies to derives, replica members a hold lists as failed, a member a new variant lacks (it comes back with + a later attempt), registry values whose `compose` throws (slice 8). (4) A watch ends only for good: its owner ended, + `stop()` or a cancelled context, a throwing listener, `storage_failed`; Durable's watches fail only for such causes + and only take the new shape. (5) `self.on` handlers are not called for failures (a reconciler acting on an error call + would act on the old value); the routine waits for the next good value; the failure is reported where it happens: + a derive's owner reports `derive_failed { object, member?, error }` once per failure episode (the first failure after + a value or after creation; a heal ends it), a new `HostError` kind (2026-10-27, slice 6a fix round 4: there was no + report for a throwing derive besides a routine's `routine_failed` on its ended watch). One failure call per + episode (a value turning into a failure), not one per source change while failed; a hold keeps only a member's + newest pending failure. Watching a failed source (2026-10-27, slice 6a fix round 5, under the rule "the routine + waits for the next good value"): `watch(ctx)` opens at the last good value and delivers one `failed` call first; + a source that never had a good value rejects `watch()` with its error; `self.on` on a failed source waits for the + first good value instead of ending. + (6) `ref.state` stays a separate watchable of reachability (`loading | live | failed | released`; `ended` since + "One vocabulary, groups 2 and 3"), same words: a + value's watch says whether it can be computed, `ref.state` whether its object is reachable (values stay readable + while loading). No new leaks: an open watch on a failed source holds what it held before and ends with its owner, + reference or context (GC test). Built: the shared `src/watch.ts` upstream in pi first (Durable's listeners adapt), + then slice 6a (points 1-3 except registries; registries in slice 8). +- **One vocabulary, group 1: the shared watch (owner, 2026-10-27: "yes"; vocabulary audit `/tmp/vocab-audit.md`, + A1, A3, A8):** built in the same upstream change as the failure calls. (a) `WatchEnd` is tagged by `kind`, not + `reason` (`end.kind === "failed"`). (b) A watch whose context is aborted ends `aborted`, not `cancelled` (one word + for a requested end, as `AbortSignal`, the `aborted` code and `JobError.aborted`). (c) Durable's synchronous + observers take the watch's shapes: `advance(call, ctx)` with a `WatchCall`, `end(end: WatchEnd)` (Durable never + sends a failure call: its failures are final). (d) A member a new variant lacks behaves the same locally and over a + link: a failure call, the watch stays open (today local `failed`, remote `ended`). (e) Durable's `FileChange` is + tagged: `{ kind: "paths", paths } | { kind: "overflow" } | { kind: "failed", error }`. The audit also found that + "Slice 6a small items" (1) is wrong: Durable's `Storage.close(context)` takes a context (decided with group 2). +- **One vocabulary, groups 2 and 3 (owner, 2026-10-27: "lgtm"; Durable's stored formats stay the same):** + Chord v2 (built in slice 6a): (1) `ref.state` `{ kind: "ended", error }` replaces `released`; the `released` code + gets reasons `ended` (object ended), `closed` (host, store, native), `settled` (`ctx.get` after its call), + `version` (an old handle), `discarded` (a transaction cut at the seal); a holder's own release stays `released` + without a reason. (2) Internal results are `done | failed { error }`; the attempt's `Outcome` is `Activation`, + the jobs' `Ended` is `Settled`. (3) `DocumentSlot.remove` is `retire`; `StoredChange` is `create | change | + retire`. (4) The wire's `cancel` message is `abort`; a native's close reason `stop` is `close`; ~~`job.abort(id, ctx)` + returns `"marked" | "terminal"` (Durable's words)~~ (**superseded**, owner 2026-10-27: Durable's `abortTask` stores a + mark and joins the run, Chord's 6a abort only fires a signal in memory; `job.abort(id)` stays in 6a, and becomes async + and stored with persisted jobs in 6b, its result chosen there). (5) Homonyms: the store's `Staged` is `StagedWrite`, its `Phase` + `WriteOrder`, the host's `KeyedMember` `KeyedBinding`, `KeyedMembers.hold` `pin`, `Presence.origin` `incarnation`. + (6) Context last: `runState.change(fn, ctx)`, `host.route(Kind, options, ctx)`, `store.close(ctx)` (as Durable's + `Storage.close(context)`; supersedes "Slice 6a small items" (1)'s "no context"). (7) `failed` and `gone` carry the + wire error format (`WireError`). (8) `HostError`: `failure` where the value is a `FailureCause`, `error` otherwise; + positions `{ kind: "on" | "onLive" | "native", position }`. (9) Store types as Durable's: `history: "latest" | + "rewindable"`, pages `{ items, next }`, a singleton's address has no key (`""` only in the column). (10) The + listener parameter is `call`; holds `served | relayed`; `put.v` is `version`. Durable's public API (pi upstream, + with group 1): (11) `onReport(report)` with `report.kind` and `report.error`; `session_failed` is `storage_failed`. + (12) `subscribeCommits` is `observeCommits` with the observer shape (`{ value, detach }`); a file watcher stops with + `stop(ctx)`. (13) `CompactionFailure.reason` and `CompactionStatus.reason` are `trigger` (`pi.live`'s definition + bumps its version with a `migrate`). (14) ~~One word for a definition that cannot take the stored state: + `incompatible`~~ (**superseded**, owner 2026-10-27: `task_too_old` is stored in orphaned outcomes and settlements, and the + report's `incompatible_task` covered two cases): the `task_outdated` report uses the scheduler's own words, + `missing_task | task_too_old | migration_failed` (newer without `migrate`: `migration_failed`); Chord's jobs name + their blocked reasons in slice 7. + (15) `TaskInspection` uses `TaskState`'s words (`ready` is `pending`). (16) `DocumentCommitChange` is tagged + (`{ kind: "retire" }` instead of `value: null`). Not done: Durable's stored tags (`status`, `type`; group 4), Durable + throwing Chord's codes (6b), one retirement value ("One retirement mechanism" (2) stands). + Added (owner, 2026-10-27, "lgtm"): Durable's `SessionEnd` and the `ReportFrom` argument of `runtime.report()` are + tagged by `kind` too (neither is stored). +- **Chord v2 and its docs live in pi upstream (owner, 2026-10-27: "we can put chord v2 in upstream", "the docs can + all be public, and in upstream"):** `packages/chord/src/next`, `src/next-node`, `test/next`, `test/next-node` and + `docs/next` moved to pi `main` (`93029a0f1`), not exported until v2 has an entry point (experimental, noted in the + changelog); the decision log moved from `apps/p2/docs/redesign-decisions.md` to `packages/chord/docs/next/decisions.md`; + the root `AGENTS.md` carries the "Chord v2 Code Style" section. From now on every slice and every design doc change + goes to pi first (worktree of `upstream/main`, check, tests, push, CI, merge into `private`); only `apps/` stays + private. +- **Slice 6b D1: `StoreStorage` and `durableView` live in Durable (owner, 2026-10-27: "they are only used if durable + is used"):** in Durable's own `src` (an ordinary pi module, now that Chord v2 is upstream), importing + `@earendil-works/chord/next` (Chord's `./next` export comes with them). No package cycle: Durable already depends on + Chord, Chord imports nothing of Durable (S13). Durable's conformance suite runs in Durable's tests and CI. +- **Slice 6b D2: Durable's records are ordinary declared tables of the store (owner, 2026-10-27: "a, that's the whole + purpose of this slice"):** `StoreStorage` declares conversations, entries, tasks, submissions and `record_ids` as + `defineTable`s, writes them with `put` (Durable replaces complete records), and reads them with the store's queries; + the outbox, export, import, the applier and later `evolveTable` versions handle them as every table. The partial + head index becomes a table `entry_heads` (one row per entry with a head; build, 2026-10-27, faster than an index on + `(conversation_id, has_head, id)`); indexed strings that must stay lossless keep + Durable's JSON encoding. +- **Slice 6b D3: one id counter per store (owner, 2026-10-27: "a"):** the store's counter (`next_document_id`, renamed + `next_id`; the archive manifest's `nextDocumentId` becomes `nextId`) numbers every id in the store: Durable's + `mintId` (conversations, entries, tasks, submissions, documents) takes from it in memory, explicit ids in Durable's + writes raise it (an entry committed with id 100: the next mint is 101), and Chord's commit step takes its documents' + ids from it; it is persisted as a high-water mark with every commit. `record_ids` stays for Durable's type check of + an id ("already belongs to entry"); supersedes "Slice 6 first decisions" (3)'s `coalesce(max(id), 1) + 1` and 6a's + "document ids monotonic with seq". Durable's own ids still rise in its creation order (its Session line serializes + its commits and mints inside them), which its scans, transcripts and windows rely on; Chord's ids are gaps between + them, as the five record types already leave. A minted id never committed may come again after a reopen (S10). +- **Slice 6b D4: the store has Durable's Storage semantics (owner, 2026-10-27: "agreed"; supersedes in part "Store + errors: Durable's rule" and slice-6a rule 10's "an expression result outside it fails the step, which is + `storage_failed`"):** Durable has two levels: its `Storage` rolls a failed commit back and stays usable (the + conformance suite tests it), its Session fails on any commit error (`GuardedStorage`). "Store errors: Durable's + rule" had put the Session's level into the store. Now: a step that fails before `COMMIT` rolls back and rejects that + commit only (the step's code runs in the transaction callback, so an error there is known to come before `COMMIT`); + the store stops only when the outcome is unknown (`COMMIT` or `ROLLBACK` failed), when `adopt` threw, or on a read's + backend error. Owners react as Durable's Session: Durable's Session fails (unchanged); a Chord object whose commit the + step refused fails live (`object_failed`, `broken`: memory and disk disagree); an `int` `CHECK` failure (a plugin's + value out of range) fails only that transaction (`invalid/args`). `StoreStorage` runs Durable's commit checks in the + step, where `SqliteStorage` runs them: no special case. Built in 6b (the store's step and object failure). +- **Slice 6b D5: persisted run records are documents of their object; one harness per store (owner, 2026-10-27: + "sure"):** each unsettled run's record is a keyed persisted document of its object in the engine's namespace + `chord.jobs` (owner the object's owner string, kind the job name, key the run id); Chord sets that namespace, a plugin + never does, so nothing collides; the state document stays an ordinary document of the run; `finish` deletes the + record and retires the state (Q9), so only unsettled runs are stored and in memory; activation loads them through the + engine's key listing and a load per run (no new engine read). Durable's tables are named under the type name + `pi.durable` (`"session:pi.durable:entries"`; type names are unique per app). One `StoreStorage` per store: a second + one on the same store is refused (`invalid/args`), as one host per store (rule 7). Rejected on the way: a table per + job, a store-wide runs table, `$`-marked internal names (owner: "if our abstractions can't cleanly support this + without $ magic, it's a bad abstraction"). +- **Slice 6b D6: persisted jobs restart as Durable's tasks (owner, 2026-10-27: "yes, same rule for chord"):** at + activation a run recorded `running` gets `restarts + 1` (stored) and resumes per its `resume`; a clean close writes + its stopped runs back so the next activation counts nothing (as Durable's `writeAtClose`); a `ctx.state.change` + resets the count; at five in a row the run ends `failed { kind: "restart_limit", restarts }` (a new `JobError`), + `finish` runs, `job_interrupted` is reported. Supersedes design §15's `blocked { restart_limit }` (no parking, as + Durable since #11139). +- **Slice 6b D7: `job.abort(id, ctx): Promise<"marked" | "terminal">` (owner, 2026-10-27: "yes"):** Durable's words + and behaviour (`abortTask`): a running run's abort is stored and its signal fired, resolving `"marked"` once `run` + settled; a finishing run is only marked; a blocked run is dropped (`"marked"`, as Durable's orphan); no unsettled run + with that id resolves `"terminal"` (Chord keeps no settled runs, Q9). Supersedes "job.abort stays until 6b". +- **Slice 6b D8: deferred persistence of a run's state, per change (owner, 2026-10-27: "agree"):** `ctx.state.change(fn, + ctx, { persistence: { kind: "deferred", withinMs? } })`, Durable's `CommitPersistence` shape (default 1,000 ms): + adopted and published at once, stored with the next immediate change, before the outcome is recorded, at close, or + after `withinMs`; a crash loses what is not stored. Only run state documents for now. +- **Slice 6b D9: `self.follow(attach)` follows Durable's views (owner, 2026-10-27: "sounds good"):** a generic Chord + primitive, no `durableView` module in Durable: `attach(observer, ctx)` returns `Promise<{ value, detach }>` + (Durable's `Observation`; its observers have Chord's watch shapes since pi `5f312dac4`). Chord makes the observer, + attaches during activation (early calls held, `value` the start value), detaches at the attempt's end; calls inside a + batch mark the source in the step, calls outside one are grouped into one batch on the next microtask; a rejected or + `undefined` attach fails the activation as a failed dependency; Durable's `failed` end ends the watches `failed`. + Returns a `Watchable` (state member, derive source, `on` source). Supersedes the carry-over's `durableView` and D1's + `durableView` part. +- **Slice 8 is "Registries and `host.root`"; the harness object is p3's (owner, 2026-10-27: "we want to get chord done + this week. p3 needs design and is next week. just need all chord primitives in"):** Chord builds the primitives the + harness needs (registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root`, `chord(fn, + context)`); the harness object (`Harness.open` over `StoreStorage`, `resume()` after the start barrier, the models, + environments and extensions registries) is p3's base code, shown in Chord only as an example or test. Plan: Chord v2 + (6b, 7, 8, 9) this week, p3's design next week. +- **Slice 6b R1: retirement per record by `history` (owner, 2026-10-27: "sgtm"; spec review, all four reviewers + found that Durable keeps retired records):** the engine's `history` gets three values, set by the adapter at + creation and applied at retirement: `none` (Chord's documents and job records: older revisions dropped at a new base, + row and revisions deleted at retirement, 6a's behaviour, until now called `latest`), `latest` (Durable's session, task + and latest conversation documents: older revisions dropped, the row kept with `retired_at` at retirement, so reads at + a point find it), `rewindable` (everything kept). `retireDocument` reads the record's `history`; no flag, archives + and replicas carry it. Rejected: keeping every retired record (one rule, but a dead row per finished job run, two + with its state, and per retired keyed document, that Chord never reads). +- **Slice 6b R2: no host kind in store names (owner, 2026-10-27: "what use is host in table names anyways? seems + stupid. no host"; supersedes the host-kind part of W7, "Restricted names" and "Slice 6a's three code questions" (3)):** + tables are `":"` (`"notes.store:edits"`, `"pi.durable:entries"`), document owners + `""`; a store belongs to one host (rule 7), so the host kind told nothing, and `StoreStorage` and stores + used before or without a host get the same names. W7's rule stays: no host instance in a name. Changed in 6a's code + in 6b; the applier checks two parts. +- **Slice 6b: owner names and claims, no `pi.durable` (owner, 2026-10-27: "seems reasonable"; supersedes D2's and D5's + `pi.durable` type name and D5's "one `StoreStorage` per store" rule):** every table and document in the store belongs + to an owner name, claimed once per store. A `local` object with empty params (P3) claims its type when it first + declares persisted data (`self.table`, persisted `self.state`, persisted `self.job`); `StoreStorage.open(store, owner, + ctx)` claims `owner`; a second claim of a name is refused (`invalid/args`): a second harness, or a plugin type that + meets the harness's name. `StoreStorage` declares its own tables (`entries`, `tasks`, …) through the store, the path + `self.table` takes, named `":
"`, and writes its documents under `owner` in the engine's namespaces + (Durable's scopes; corrected 2026-10-27, it was "`durable`"). In p3 the harness object passes its own type (`StoreStorage.open(store, Harness.name, ctx)`; base code gives + the store to the core facet); Durable's conformance suite passes a test name. The owner name plays the role of + `SqliteStorage`'s `prefix`, checked by the store. Objects never see the store: their API writes to the host's store + under their type; a host without a store fails an object declaring persisted data (`broken`, 6a). Rejected: an owner + handle (`self.storage`), an indirection that only added protection against trusted code. +- **Slice 6b R3: a seal is a clean stop for persisted runs (owner, 2026-10-27: "a"):** a persisted run cut off by + its attempt's seal while the object continues (a live failure, then a retry) is written back stopped, and the next + live attempt resumes it per its `resume` (rerun or abandon) without counting a restart, as after a clean close (D6). + In-memory runs keep 6a's rule (`failed { aborted, reason: "restart" }`, finished on the next live attempt). +- **Slice 6b R4: a follow source's end (owner, 2026-10-27: "yes"):** Durable ends an observer `failed` (its Session + failed: the object that opened Durable fails `transient`, its retry reopens its natives, Durable included, and attaches + again, as Durable's own recovery is to reopen from Storage) or `ended` (the observed owner ended normally, a task + document's task: final, the member keeps its last value, its watches end `ended`). +- **Slice 6b: claims per table and per document namespace (owner, 2026-10-27: "agree"; refines "owner names and + claims"; Fable's second review N2):** what can collide is claimed once per store: a table by its SQL name, documents by + `(owner, namespace)`; so the object that opens Durable can also declare its own persisted data under the same owner + name, and two harnesses can share a store under two owner names (owner: "seems like they could"); a second + `StoreStorage` on the same owner is refused by its tables. "One harness per store" is no rule, only how p3 uses it. +- **Slice 8 D1: a registry's token is shared, its provider serves it (owner, 2026-10-27: "b", "yes"):** the token + `defineRegistry(name, { version })` carries identity only (bundled by every contributor, as a contract); the + defining plugin's facet provides it with `host.serve(R, { compose, accepts? })`, as it serves object types. One rule + for versioned things: a shared definition (object types, registries) keeps its adapters with the provider; a + definition of one owner (documents, tables, `json(...)` columns, jobs) keeps them in itself. Refines Q8's "one + authoritative declaration" (the serve is it) and design §17's example. +- **Slice 8: registry failures (owner, 2026-10-27: "ok", "agree"):** contributions never fail a registry: a + watchable contribution that fails is left out of the table and reported (`contribution_failed`), back once healed; + a throwing `compose` (the owner's bug) is reported (`registry_failed`), its watches get a `failed` call, and `.value` + keeps the last good composition (before any, it throws). Derives keep `.value` throwing while failed: state must match + its sources, an extension point must stay available. +- **Slice 8 D2 and D3 (owner, 2026-10-27: "yes"):** D2: a `retryOn` source retries a failed object only on a `value` + call; a `failed` call does nothing. D3: `host.root(fn, context, { principal? })` gives `fn` a handler's call + context (the host's principal or the given one, this host's incarnation, `context`'s telemetry and signal linked + until `fn` settles, `ctx.get`); at settle or abort the context is sealed (`ctx.get` rejects `released`, its references + released, their calls reject), so an uncooperative `fn` fails at its next use of Chord; before `load` its gets park; + after `host.close` it rejects `released`. `chord(fn, context)` is p3's helper. Closes design §26 #25. +- **Slice 8 S1: a Session close fails Durable's observers (owner, 2026-10-27: "a"; spec review, all four reviewers):** + Durable ends an observer `failed` (error "session closed") when its Session closes, and `ended` only when the + observed owner ended (a task document's task); refines R4 (Durable change, upstream). So a follower of a harness + reached through a registry fails `transient` when that harness closes, and its retry follows the new one. A + `retryOn` wake that arrives while the object activates is kept and retries it at once if that attempt fails. +- **Slice 8 S2: registry readers of an older version (owner, 2026-10-27: "a"):** as contract state, the provider maps + back: `host.serve(R, { compose, accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } } })`; + a reader bundling an older token gets the composed value mapped to its version (computed once per composition and + version, only while such a reader exists); without a `value` mapping, or for a reader newer than the provider, its + value is failed `contract_mismatch` (reported); such a reader fails `broken`, not `transient` (nothing heals it before + a reload, which retries broken objects; build review 2026-10-27). +- **Slice 8 S3: `self.registry` is a dependency, as `self.get` (owner, 2026-10-27: "isn't self.registry meant to wait + like self.get?"; my spec had it as a plain read, a mistake against design §6, §7 and §17):** activation waits until + the registry has a value (served and composed once; an empty table composes too, at the serve); after `load`'s + setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` + (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not + served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Slice 7b's open points, as recommended (owner, 2026-10-29: "yes"; brief `slice-7b/`):** (1) the API is + `host.reload(facets, ctx)`, by plugin name (added, changed; a plugin missing from the list is removed), resolving with + a report of what each object did, rejecting with the code error and nothing changed; `load` stays the first call. (2) + A `writable` across a pure swap goes back to its initial value (the new factory makes a new one; no positional + matching). (3) The rollout across hosts is the app's: it calls `host.reload` on each host in order (server, workers, + clients); Chord guarantees per host (staging, commit, `installing`, rollback). (4) No install watchdog: `reload` + takes a context and the app passes its deadline. (5) The loader lives in Chord (`chord/next/loader`, slice 7c). (6) + Chord ships the app kit as optional subpaths: `loader` and `dev` (7c), `http` and `workers` (slice 10), `web` and + `tui` (slice 11); telemetry (slice 9) last. +- **Slice 7b's build decisions (owner, 2026-10-29, answering the builder; `slice-7b.md` §13):** a loaded plugin passed + as the same facet object (`===`) is unchanged and untouched, any other facet value for a loaded name is changed; + activating (not parked) and backoff-waiting objects of a reloaded plugin are retried; the report's `parked` covers + swaps and retries (their first dependency check), restarts are reported at once; a swapped-out attempt is sealed + once its calls settled (bounded by `closeTimeout`) and the swap ended, its job invocations finishing on their old + code; a stuck old routine call is reported `drain_timeout` and fenced by the old routine calls' shared token; a swap + to another variant sends holds a fresh `bound`; a native restart ends in-memory runs `aborted { reload }` while + persisted ones follow 6b R3 (stopped, resumed per `resume`, no restart counted); staging is synchronous + (`installing` tested from a staged factory); a removed object's member sources and pinned keys are handed to the next + object of its identity (fix round 3: one hand-over per identity for removal, version refusal and an object ending + while requests wait, `slice-7b.md` §13 item 11); keyed members' versions are built in 7b (L1's keyed edits and an accepts `keyed` section, + decision "Keyed state members, local contracts only"). The builder's own choice, accepted by the owner (2026-10-29): + adapted state members are derives per object and version, not per binding. (**Superseded in part** by "References + are the only indirection": the hand-over of member sources and the per-version sources that outlive an object go.) +- **References are the only indirection (owner, 2026-10-29: "go"; after seven fix rounds of 7b found one edge case after + another in sources handed between objects):** supersedes in part 6a's "members across attempts" (one member source + per object and member name) and slice 7b decisions 8 and 11 (member sources and pinned keys frozen and handed to the + next object of the identity; per-version outward sources owned by (identity, version); `host/handover.ts`). The + problem: a consumer's watch or derive over `ref.count` was attached to the referenced object's own document or derive + (the reference was bypassed), so every change of what the reference points at needed a stable node somewhere else + (`MemberSource` on the object, then sources that outlive the object and move to the next one). The rule: (1) a + consumer's watches and derives over a referenced object's member attach through the reference's member, which passes + the subscription straight through to the object's own document or derive (no extra hop on commits). (2) When what the + reference points at changes (a new object instance after a factory re-run: reload, retry, restart; a new object + after the type was removed and served again; another variant), the reference moves its subscriptions to the new one + and sends one frame, a diff from the last value. (3) While it points at nothing, its members keep their last values + (a pointer, no copy): while loading (a restart, a retry, a swap whose new instance is not live yet) its watches get + no call; while failed (a live failure, its version refused, its type removed, a failed dependency) one `failed` call + with the cause's error, and they stay open (replaces 6a's "no frames while failed"); a reference that never had a + value reads `not_ready` while loading and its failure's error while failed, and its watches wait for its first values + (a member the new variant lacks fails `invalid/member`, as before). Keyed watches and pins re-pin on each new instance the same way (the + cross-attempt key sources of 6a go; a key keeps its last value until the new instance loaded it; a dependent's + factory pin is still loaded before the dependent goes live). Once hydrated, a reference has last values for every state member of its bound variant, read or not: when it + detaches it takes them at its version from the outgoing instance (through that instance's mapping for an older + version), keeping no node of it. After a `failed` call, the next good value arrives as `[["r", value]]` even when + unchanged; a watch opened while failed starts at the last value and gets the failure once. No source, mapping, + document handle or pin of an instance outlives it through a reference: a detached reference keeps only plain last + values and its failure; releasing it drops them and its bindings; stopping a keyed watch drops its pin. (4) The same rule over a link: a replica keeps its last values + readable while its hold is failed. (5) An older consumer (`accepts`) attaches to the object instance's shared mapping + for its version: per member one derive `(current) => old`, made on first use, dropped when no reference of that + version is left, gone with the instance (one map and one diff per commit per version, whatever the number of old + references); a version the instance does not accept gives no member (unbound, last values kept). A reference knows + its version from the contract token it was made with. Nothing outlives its object instance; nothing is handed over. + Kept: requests never end on a code change, healing on rebind, `accepts` for arguments and results. A simplification + audit of slices 6a to 7b follows 7b's commit (owner: "how much other such bullshit was added in the last slices"). + Refined after the gap checks (Astra, Sol, Opus) and the build (owner, 2026-10-29: "i accept the recommendations"): + a detaching reference follows the old instance until it stops changing (its seal after admitted commits, a restart + after its drain), then takes its last values for every member of its bound variant, read or not, never from an + instance it never followed; after a `failed` call the next good value is `[["r", value]]`; a watch on a reference that + never hydrated resolves at its first binding (rejects on release, revocation, close, abort); release drops last + values and bindings; registry values and job state are not affected. **Derives over a failed reference member compute + on its last value** (G2 (A), as 6a): a contribution built on it stays in its table; only the reference member's own + watches get the `failed` call, and `ref.state` shows the failure. **Activation churn +9.6% accepted** (CPU 22, 104.7-107.9 + → 114.7-118.3 µs, allocation equal): an object whose derive reads another object's member makes one pass-through + node per reference member per instance; calls, commits, links and memory are within the bar. Two references to one + key hold two pins (counted on the key's one source; the document is loaded and kept once). **Binary append commit-to-frame + median +10% accepted** (owner, 2026-10-29; CPU 10, 0.031 → 0.034-0.035 ms): equal with the young generation pinned, fewer + scavenges over the whole benchmark (V8's young-generation sizing, slice 7b fix round 3). +- **Per plugin at startup and reload (owner, 2026-10-30: "yes", "1. lgtm 2. yes"; analyses + `/tmp/per-plugin-{opus,fable}.md`):** supersedes "Rollback is all-or-nothing per host" (2026-10-2x, 7b §2, design §21) + and 7c's Q2 ("a facet throwing while evaluated fails its host's whole step"). Why all or nothing was chosen: per plugin + was decided first and p2's plugin reviewers found counter-traces (a dropped argument field written into a transcript; a + copied Durable document written by two versions; a provider's failed factory leaving the harness hanging; mixed copied + code and registries); v2 has since closed them (closed argument schemas for every caller, Q12, Q15; version binding + with waiting, 7a; ownership of documents; registry versions with `accepts`, Q8; copied cross-plugin files limited to + contracts without dependencies, 7c). Rules: (1) **at startup** a plugin whose build, evaluation or setup fails is left + out; `load` resolves with a report `{ failed: [{ plugin, error }] }` and never closes the host for it (the core facet + included: base code decides whether that is fatal); its consumers wait `failed { waiting }` as for any missing plugin. + (2) **At reload** a plugin whose new code fails (build, evaluation, setup, a factory in staging, a declaration error, + `checkAccepts`) keeps its old code (its loaded facet, unchanged), and every other plugin of the reload applies; the + report lists it (`failed: [{ plugin, error }]`, as `load`'s; an added plugin that fails is left out); staging retries without it. (3) **The closure rule** (Opus): after + staging, a changed plugin whose recorded requests would not bind because the plugin they need was kept old or left + out is kept old too, so a consumer edited together with a broken provider keeps working on its old code. Its "missing" + branch (owner, 2026-10-30: "keep", against narrowing it to names served before): a request nobody serves keeps its + plugin old whenever any plugin of the reload failed, since a failed new setup may be what would have served it. (4) What no + rule detects: a consumer relying on new behaviour of a kept provider whose contract (version, fingerprint) did not + change: the same risk as a rollout across hosts, documented. (5) `rollout` keeps a plugin dirty while it was kept old + on any host. Built with slice 7c (the kernel's `load` and `reload` change with it). +- **Slice 7c's points (owner, 2026-10-30; `slice-7c.md` §5):** (1) esbuild is an optional peer dependency of Chord, + imported only by the plugins component. (2) **No dev/prod split** (owner: "chord apps are supposed to not have a + prod/dev split. they are supposed to be self modifying applications"): watching, rebuilding and rolling out are core + machinery; one component `chord/next/plugins` (discovery, build, evaluation, `pluginFile`, the watcher, `rollout`) in + `src/next-node/plugins/`, exported with v2's entry point; it turns V8's compilation cache off when created; reloads + stay explicit (the watcher marks, never applies). Supersedes the separate `loader` and `dev` subpaths of "Slice 7b's + open points" (5), (6). (3) `installDependencies(plugin, ctx)` as coding-agent installs package dependencies (`npm + install --omit=dev --legacy-peer-deps --ignore-scripts --no-audit --no-fund` in the plugin's real directory, the + command overridable). (4) Cross-plugin imports: any file of a discovered plugin, copied, documented as contracts and + side-effect-free modules only, listed by the build, no static check yet. (5) `pluginFile` answered per facet by the + `require` evaluation passes. +- **Slice 7c build: an abort after the commit names what it applied (owner, 2026-10-30: "yes"; fix round 1, item + 11, Astra):** `host.reload` still rejects `aborted` when its context aborts after the commit (the commit stands, only + the wait ends), and the error carries `details.committed`, a `CommittedReload { added, changed, removed, failed }` + (plugin names; `failed`: kept on their old code). Why: the plugins component keeps per host the facet each plugin + runs and adopts it when the reload resolves; on this rejection it would keep the old facet and pass it at the next + reload, swapping the committed plugin back silently. Rejected: resolving with the report after the commit despite + the abort (it would change the rule that an abort after the commit rejects). Examples for 7c (owner): full coverage, + each example a folder with the host code and real plugin directories (35-39). +- **Slice 10's points (owner, 2026-10-30; `slice-10.md`):** (D1, "a"; `httpServer` **superseded** by `createHttp`, decisions "The HTTP core is a fetch handler" and "Settings are arguments") the HTTP component defines the registry `Http` + (`"chord.http"`) and serves it through a facet of its own (`http.facet`, plugin name `chord.http`) that the app loads + with its plugins; its `compose` builds the router; the server reads it with `host.registry(Http, ctx)` per request. + (D2, "a") Wraps are built in slice 10 as designed (decision "HTTP routes and wraps", design §23): a contribution's + `wraps` run around routes in the component's phases (`observe`, `headers`, `transform`, outermost first), within a + phase in contributor order, by default around their own plugin's routes (`scope: "own"`; `"all"` explicit; `route` + narrows to one, a missing one a reported conflict). The server's `headers` option stays for every response, also + those no route makes (a 404, a refused request). + (D3, "a"; **superseded** by "The HTTP server is a native object": the `Authenticators` registry) Authentication is one app function, `authenticate(request, ctx) => P | undefined`, called once per request + and per upgrade; no Chord registry of resolvers (p3 composes its own from sharing, devices and the relay, with its + own precedence rules). + (D4, "k": a) Request bodies: `request.json(limit?)` (parsed JSON, 1 MiB by default, malformed: 400) and + `request.body(limit?)` (an async iterable of byte chunks, for uploads); nothing is read until a route asks; past its + limit the read stops and the request is answered 413; a disconnect aborts `ctx` and ends the iteration. + (D5, "a") Responses are descriptions: `json`, `html` (a full document; replaces the design's `page { title, body }`), + `bytes` (`mime`, `size`, `read(offset, length, ctx)`, `download?`, `ranges?`; 206/416, `HEAD`, safe + `Content-Disposition`, 64 KiB reads with backpressure, stopped on disconnect), `redirect`, `error`; `headers` on every + kind (cookies, a preview's own CSP); `fileResponse(path, options)` over a file; a throwing route is a 500 without its + message. + (D6, "a") Plugin WebSockets are `sockets` in the `Http` contribution: `{ "/terminal": (socket, request, ctx) => … }`; + the component checks the origin and runs `authenticate` (no principal: refused, 403, before plugin code), upgrades + and hands the plugin a `ws` WebSocket for its own protocol; patterns as routes; wraps do not apply; the links path is + the component's (`host.accept`). The design's channel component (acknowledged byte channels) waits for a second user + and would be built on top of it. + (D7, "a") `ws` is an optional peer dependency of Chord (pinned exactly), imported only by `next-node/http`, as + esbuild for plugins. + (D8, "a") Route patterns: exact segments, `:name` (one non-empty segment, percent-decoded, `params.name`) and a + trailing `/*` (the rest, possibly empty, `params["*"]`); the most specific match wins, segment by segment from the + left (exact before `:name` before `*`); one pattern from two contributions is a reported conflict, the first by + plugin name wins; `HEAD` falls back to the `GET` route. + (D9, "a") Worker keep-alive: the child reports `host.activity` and the app's `busy` counts (a JSON watchable of + numbers, e.g. Durable's active tasks) whenever they change; the server stops a worker whose counts were all zero for + `idleMs` (default 5 min, checked every second), never while it starts; a stop sends `shutdown` (the host closes), + the process tree killed after 5 s. + (D10, "a"; **superseded** by R2 below: no automatic restarts, no route backoff) A failed worker start uses the kernel's route backoff (1 s doubling to 60 s while holds exist, + `resolve_failed` reported); no sticky failure; `workers.state` shows the error and when it is tried again. + (D10b, "b") `host.route(Kind, options, ctx)` returns a `RouteHandle` with `retry(address)`: it ends that address's + backoff, so `resolve` runs now if the address has holders (a Retry button for a worker that failed to start, wired + by the app to the handle; the workers component has no `retry` of its own). + (D11, "a") Workers never build: the server's plugins component discovers and builds every plugin; a worker's start + message carries its kind's current scripts (`Built`, plain data) and a reload ships new ones (D12); the child's + `plugins` only evaluates (no esbuild, no watcher). So plugins are defined where the server can read them, one set for + every host (owner: "no cwd specific plugins, which might be alright; that way workers can live anywhere"); project + plugins (design §26 #14) would be discovered and built by the server too. A worker on another machine needs the + plugin directories there only for `chord.external` packages and `pluginFile`. + (D12, "a") A rollout step returns one report per host, `{ hosts: [{ host, report }] }` (7c's `RolloutStep` changes; + the in-process server step returns a list of one); the workers step reloads every running worker in parallel; a + worker that closed during it is listed with its error and does not stop the rollout; a plugin's dirty mark clears + only when it applied on every host of every step. + Spec review (Fable, Opus): (R1, "a") `RouteHandle.suspend(address)`: while suspended the route does not resolve the + address, its holders stay `loading`; `retry(address)` ends the suspension; `workers.stop` suspends, then stops (an + archived session stays stopped while a tab still holds it). + (R2, owner: "take it"; **supersedes D10 and the kernel's route backoff**) **Workers start on demand and never + restart by themselves** (owner: "if it crashes, chances are it had a good reason to crash, and restarting it won't + fix that"; D10 as recommended was a restart loop whenever a consumer holds). A worker starts for a hold of an address + that has not failed (the first, or one after an idle stop or `stop`). A crash or a failed start marks the address + failed with its error until `workers.retry(address)` (`RouteHandle.retry`) or a rollout that reaches workers (new + code may fix it, as a reload retries failed objects); its holders and new holds see `failed` with that error, nothing + forks. Kernel: a route has no backoff timer; it resolves only for a hold, on `retry`, or after its link dropped while + held, and a rejected `resolve` fails the route's holders with its error (no endless `loading`); a network route that + wants to reconnect keeps that policy in its own `resolve`. The workers component owns its route + (`createWorkers(host, Kind, options)` calls `host.route` and keeps the handle) and answers a resolve of a failed + address with the stored failure at once. A start finding the lock still held by a draining old worker waits for it in + the child (bounded by the start timeout). R1's `suspend` and D10b's `retry` stay. + Other timed retries (owner, 2026-10-30: "keep them as is for now"): objects' transient failures (1 s to 60 s while + held), `self.on` handlers that throw (1 s to 60 s while live) and `self.onLive` restarts (the same, reset after a + minute's run) stay; they heal real transient failures (a network down for a while) and cost at most one try a minute + each; revisit if one shows up as a loop in practice. + (R3, "a"; refines D9) The child reports only the flip: `{ idle: true }` when every count (`host.activity` and `busy`) + becomes zero, `{ idle: false }` when one becomes nonzero; the server's idle clock starts at `true` (a status message + costs about 6 µs on each side, a local call 2.9 µs: no report per call). + (R4; **superseded in part** by "allowed hosts only through contributions" and "The HTTP component's security baseline": no `allowedHosts` option, provenance by the socket; "a if configurable at runtime so allowed domains can be added and removed by a plugin") Before `authenticate`, the + component refuses (403) a `Host` that is neither loopback nor allowed, and an upgrade (`/chord`, a plugin socket) + whose `Origin` is not the request's own origin; `request.origin` comes only from an accepted `Host` (`http://` on + loopback, `https://` for an allowed host); `headers(request, principal)` gets the principal. Allowed hosts are the + server option `allowedHosts` plus every `hosts` list in `Http` contributions: a plugin adds and removes its domains + at runtime by contributing a writable (`self.writable`, design §17), the router recomposed on each change (a tunnel or + relay plugin names its public host). + **The app kit is malleable by plugins** (owner, 2026-10-30: "all of this should be malleable via plugins"; "yes"): + in `http`, `workers` (and slice 11's components) every option that is not bootstrap is either a contribution to + `Http` (routes, wraps, sockets, hosts; writables where it changes at runtime) or a function the component calls per + use (`authenticate`, `headers`, a worker's `start`, `idleMs`), which the app backs with its own registries; nothing is + copied out of a contribution or an option at startup. Bootstrap stays base code, changed by a restart: the listen + address and port, the links path, the worker entry module with its `execArgv` and `env`, the host kind workers serve. + (R5, "agree") `workers.state` per address key: `{ status: "starting" | "running" | "stopping", pid, startedAt }` or + `{ status: "failed", error: { message, code?, reason? }, at, stderrTail? }`; no entry when stopped or never started + (a clean stop removes it); `failed` stays until Retry or a rollout; plain JSON (a core object may serve it). + (R6, "b, common enough") `bytes` may omit `size`: chunked transfer encoding, no ranges (no `Accept-Ranges`), `read` + called with growing offsets until it returns an empty chunk (a zip built on the fly, a growing log, an export paged + from a worker). + Spec review round 2 (Fable, Opus, Astra, Sol): (Q1, recommended) holders of a failed worker get `failed { waiting }` + with the stored error as cause and stay on the address's path: a later link of the address (Retry, a rollout) binds + them without a re-hold; a `get` waits its `readyTimeout` meanwhile. + (Q2, "a") A rollout that reaches workers clears the failures from before it, then calls `RouteHandle.retry` for the + cleared addresses that have holders (they start with the new code and are listed in the step); an unheld one starts + at its next hold; a worker that fails during this rollout stays failed (no loop: one start per held failed address per + rollout a person triggers). + (Q3, "a"; refines R1) A `stop`'s suspension lasts while the address has holders; they see `failed { waiting }` + ("suspended"); with the last holder the record goes, and a later hold starts the worker on demand (as after an idle + stop); whether an archived session may be opened is the app's (`authorize`, its views). Memory stays bounded. + (Q4, "a") The start timer (60 s) covers fork, lock and connect (`started`) only; after `started` a slow `load` is + objects showing `loading`; `ready` sets `state` to `running` and lets idle stops begin; a `load` that never ends + shows `starting` until `stop` (no kill, no retry). + (Q5, "a"; refines R3) A worker's idleness is `references`, `jobs`, `lingering` and the app's `busy`, not `calls`: a + remote call runs on a hold (counted), local calls come from routines (not counted, §14), jobs, Durable tasks (`busy`) + and handlers under a hold; no message per local call, no timer. The first report goes with `ready` (`ready { idle }`), + so a worker idle from its start is stopped too. + (Q6, owner: "if sockets are a contribution, why keep them when the contribution is removed?": design §17's rule, an + entry lives as long as its contributor and consumers keep nothing beyond a use) A plugin socket is closed (1012, + "service restart") when the contribution that handed it out leaves the table (a reload changing or removing it); a + `/chord` link or socket whose `Host` is no longer allowed closes (1008) when the allowed hosts change. +- **Slice 11's points (owner, 2026-10-30; `slice-11.md`):** the surfaces are generic plumbing; the app keeps + everything visible (owner: "this one needs to be generic, so apps can do whatever they want, css, tui components, + themes"). (P1, "a") `chord/next/web` ships serving the surface (the base bundle, the plugins' web scripts, + stylesheets and assets, the page through the app's template, all as `Http` contributions), the browser's client host + with its connection loop, and loading and applying the plugins' web facets; not HTML, CSS, components, themes, + navigation, slots or layout. + (P2, "script tag") A plugin's web script is served as a file that registers itself + (`chordFacet("", "", function (require, module, exports) { … })`) and is loaded with ` + diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md new file mode 100644 index 000000000..d763cd78c --- /dev/null +++ b/packages/chord/docs/next/dependencies/model.md @@ -0,0 +1,330 @@ +# Dependency resolution in Chord v2: one model (decided, 2026-10-29) + +Status: decided (owner, 2026-10-29); **built for slice 7a** (audit A1, 2026-10-30: `installing` is cut, staging being synchronous; A5: retry dedupe by a visited set and a hop budget; A11: `parked_long` for any long park) (§1-4 and §6-11 without what installs and reloads add: +§1's `installing`, §4's install-end and reload wakes, §3.9's reload handover and §5's removal, versions and rollout +are slice 7b's; folded into `design.md` §3, §5, §6, §7, §17, §21's start barrier, §22; `slice-7a.md`). The draft +history below is kept as it was. Sources: the owner's direction +("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin +unloading"), the reviews (`/tmp/dep-mechanism-*.md`, `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md`, +`/tmp/model/review{1,2}-*.md`), the code at pi `358f7d46a`. **Owner** marks an open decision. + +Draft 7 (round 6): (A) also retries in lockstep without finding the cycle (Fable); the probe's recovery after a +suppressed search becomes a choice between (B1) a re-search backoff for probes only (recommended) and (B2) a full +search lifecycle without timers, since `dead` notices alone need recovery state that eviction may not drop (Astra, Sol). +Causes are compared by value on wakes (Opus). +Round 7: a returned search is accepted while its initiator is parked, whatever its round (Astra: re-searches must not +invalidate searches under way); detection is eventual, not within one backoff period; the local walk stays (Opus). + +Draft 6 (round 5): cycles across hosts become an owner choice between (A) the owner's slice 4b decision (simultaneous +starts across hosts time out, no probe message), narrowed to one deadline, and (B) the probe, completed with edge +tokens, no revisits, and `dead` notices back to the initiator (Astra: a check that dies silently can lose the only +search that survived dedupe). (A) turned out to either poll chains across hosts again or miss a ring of `waiting` +attempts, so (B) is recommended, pending its review. + +Draft 5 (round 4, Sol's phantom cycle): the second round is replaced by a check along the recorded trail of +`(attempt, edge generation)` hops; a reported ring existed at one moment; probes grow with their path. + +Draft 4 after round 4 (probe only): round numbers per attempt, forwarded only when newer than the stored one; probes +honoured on parent and route links, dropped on child links; completeness stated under an eventually stable graph; the +repair caveat broadened to any concurrent edge removal; a new edge generation resets confirmation; a bounded display +trail. + +Draft 4 changes (round 3): the probe starts a fresh round on every new or changed wait edge (a ring closes at its last +edge, whose source finds it), keeps one seen entry per initiator, accepts only its current round, and states its +cost honestly; every host drops probes from untrusted links; local forwarding is iterative; no-placement remote +requests are re-placed when a host's first parent link connects; a cohort object that ends before its first outcome +settles the barrier; the last-good refinement is marked for the owner. + +Draft 3 changes (round 2): remote types without placement keep today's fallback (here if served, else the parent); +hydration stays before parking; no copied `retryAt`; a dependency `ended` breaks its dependent; a reload's handover +binds new requests before releasing old ones; retry passes only to `broken` and `transient` blockers, once per +operation; `installing` re-evaluated at every install end (commit or rejection); a registry's last good value dropped +when one of its contributors leaves; the probe with round ids, bounded seen sets, edge-generation triggers, attempt +ids on every non-live answer and relays dropping probes from untrusted links; one cycle mechanism, local and remote. + +## 0. The problem + +The same situation, "what I need is not there", has five outcomes today, depending only on how it is declared: + +| Declared as | Provider missing | Today | +| --- | --- | --- | +| `self.get(LocalType)` | no facet serves it | `self.get` throws, the factory throws: `broken` | +| `self.get(RemoteType)` | the worker does not serve it | the hold is refused `broken`; the dependent `transient`, polled 1-60 s forever | +| `self.registry(R)` | no provider | `transient`, woken at R's first value only | +| `self.registry(R)`, reader newer | | `broken` | +| a reload removes the type | | holders get `gone`: ended for good | + +And the startup barrier deadlocks an acyclic graph across two hosts (Sol, `/tmp/dep-graph-sol-barrier.log`): S's eager +A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which waits for A. Only `readyTimeout` ends it. + +## 1. Requests and one resolver + +- **A request** is what a reference stands for: type (or registry), params, principal, the consumer's contract version. + Its holder makes it and owns it: an attempt (`self.get`, `self.registry`, keyed pins), a call (`ctx.get`), outside + code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the + attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the + holding host closes), never because its provider is missing, went away or does not fit. +- **The destination by address** (round 1): a `local` type goes here; a `remote` type goes to the address its + `hostAddress` (computed from the consumer's own contract) gives: here if that is this host's address, whether served + here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares + its `hostAddress` (owner, 2026-10-29: the optional `placement` of slice 3a is removed and the field renamed, and with it the fallback "here if + served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or + refused (`invalid/host_address`) as today; a relay never captures a hold because it serves the same name. +- **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the + provider's own host-address check). Version checks never look at a same-name registration elsewhere. +- **The publication gate:** a request made on this host, or arriving on its links, before the host's first + publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the + synchronous setups, never for activation, so it cannot deadlock. +- **Outcomes of the check:** + + | The check finds | Request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | + | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params | `failed { broken, forbidden \| invalid }`, as today, never indexed | + + The unbound reasons that wait for a name (`unpublished`, `not_served`, `version`, `compose`) are indexed under it on + the host that decided them: the consumer's host for a local request; the provider's host for a hold that arrived + over a link (it already keeps such holds as refused: `remote/hold.ts`). `installing` is the provider host's own, + re-evaluated at its install's end; `no_link` lives on the path, where the next link holds it again. Indexes drop an + entry when its request ends; GC-tested. + +## 2. States + +`ref.state` keeps its four kinds; `FailureCause` gains one kind, `waiting`: + +```ts +type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } + | { kind: "ended"; error: ChordError }; + +type FailureCause = + | { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // the object's own code, a cycle, or authority: retry() or a reload + | { kind: "waiting"; error: ChordError }; // not the object's fault: no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event +``` + +| The request is | `ref.state` | Calls | +| --- | --- | --- | +| bound, its object activating or parked with no failed blocker | `loading` | park, bounded | +| unbound `unpublished` or `installing`, or its path has no link | `loading` | park, bounded | +| unbound `not_served`, `version` or `compose` | `failed { waiting }` | reject at once | +| bound, its object failed | `failed` with the object's cause, as today | reject at once | +| bound, its object live | `live` | run | +| ended by its holder, revoked, or its host closed | `ended` | reject | + +**Owner:** `waiting` as a new cause kind (Opus, Fable; recommended: the cause says "not your fault, an event will wake +you" and a UI can say what is missing) or no new kind: `transient` with no `retryAt` (Astra; today's wire already +carries a `transient` cause without `retryIn`, `remote-object.ts`), or `loading` with a diagnostic (Sol; parks every +call on a missing plugin for its full `readyTimeout`). + +## 3. Dependents: parked until their dependencies resolve + +A **dependency** is what an attempt declares and needs before it goes live: `self.get` (local or remote), +`self.registry`, keyed pins (`at(key)` on another object's keyed member). `self.follow` is the attempt's own step (its +attach runs Durable's code), not a dependency; a follower declares `retryOn` on the registry it reads the harness +through, or recovers by its own backoff. + +1. **Order, as today up to the check:** the factory runs; persisted singletons hydrate (the object's documents, kept + across attempts); the requests are made. The attempt is **parked** while any dependency is not `live` (hydrated): + + | Its dependencies | The dependent shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting, "waiting for X: " }`, `retryAt` undefined; the message keeps the nearest blocker and the root cause within the 500-character rule | + | one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | + | one failed with `forbidden` (an `authorize` refused it: authority, §3.6) | `broken` with the same error: only `retry()` or a reload helps (slice 7a fix round 1) | + | one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable) | + +2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own + steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This is + R18's rule for live objects applied from the check on, not a rule of its own (owner, 2026-10-29: a dependency can + go away at any point of an object's life; the object stays live, its calls through that reference meet its state). So a parked attempt holds no + natives, and an attempt that opened natives is never parked. +3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as + eager migrations pause it) and resumes with the budget left. +4. **Wakes, one rule:** the dependent re-evaluates on every state change of any of its dependencies (live, loading, + failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and + round), updating what it shows; it leaves parking when all are live. The factory never re-runs + while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the + attempt's own steps or its teardown is remembered. (As built in 7a, with the parent's note 5: a change while the + parked attempt checks or loads its pinned keys is remembered and checked again; after the dependency check, during + the own steps, a change is ignored, §3.2's R18 rule.) +5. **Kept while parked:** its documents (the object's), its declarations and requests, its seal token; no natives. + Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes). +6. **`broken` only for the object's own code** (a throwing factory, a derive throwing at activation, hydration, + `migrate`, a document fingerprint mismatch, an `open` throwing `broken(...)`), **cycles** (§7) and **authority** + (`forbidden`, an ended dependency). +7. **Retry passes on:** `ref.retry()` on a `failed { waiting }` object retries its `broken` and `transient` blockers, + hop by hop (the existing `retry` message over links), once per retry operation (an id dedupes diamonds); `waiting` + blockers pass it on; nothing else is touched. Retry of a `broken` or `transient` object starts a new attempt, as + today. +8. **`retryOn`** wakes failed objects as decided in slice 8 (a value call retries a `transient` or `broken` object); + while the object is parked `waiting`, a value call does nothing: its dependencies decide. +9. **Own-code reload of a parked object:** its plugin's reload replaces the parked attempt with the new factory's, + as one handover: the new requests are made and bound first, then the old ones released (round 2, Astra N4: an + unchanged dependency held only by this object must not end in between; today's `#activate` binds before + `#releasePrevious`). +10. **After live, R18 unchanged:** a dependency leaving `live` leaves its dependent live. + +## 4. Wakes: two events + +| Event | Re-evaluates | +| --- | --- | +| **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | +| **a name becomes usable, or an install ends, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | + +No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause +kind `waiting`. + +## 5. Removal, versions, rollout + +- **A reload that removes a type:** its objects end as today (natives close, reason `end`; callers rejected at once; + handlers drain). Their requests are not ended: each becomes unbound `not_served` and binds again if a later commit + serves the name. + - **Locally:** a `Binding` whose target ended by removal becomes unbound and accepts a second `bind`: member sources + re-bind by name (one frame from the last value, as on a restart), keyed pins re-pin, parked calls flush or + reject, presence re-lists. The one new piece of machinery. + - **Over a link:** a `failed` on a bound hold unbinds it (Fable: one message, one rule; the consumer applies it as + `#startOver` plus `failed`); the provider detaches the hold's sources and drops its unsent pending ops and queued + `bound`; the ack slots of `bound`/`state` messages already sent stay counted until their `ack` or the link's loss; + calls in flight on it are rejected and free their slots once. +- **A reload that stops accepting a version** unbinds only the requests on that version (`version`); the object stays + (its own plugin's reload still restarts it if it has natives, §21). +- **A type moved to another host kind:** the request carries the address its own consumer code computed; the old host + answers `not_served`; the consumer's own reload makes a new request. No redirect. +- **A consumer whose code moves to a new version** releases its old request and makes a new one (today). +- **Rollout, without revision fields:** each host knows only its own install. A provider host that installs keeps + holds that do not fit its current code `installing` (`loading`) and re-evaluates them when the install ends, by its + commit or by its rejection. A provider host that is queued but not installing yet answers a mismatch `failed + { waiting, version }` until its install commits and wakes it: a visible blip of at most one install, accepted (Opus). + An install never waits for activations (a reverse-order rollout would deadlock); a host whose own install was + rejected fails its own requests at once with the reason (design §21). The loader keeps its own duties (the desired + code set, digests, serialized installs). +- **Old calls across a reload** follow §21 unchanged: a pure swap lets them finish on old handles; a native restart + drains them first; a removal rejects their callers at once and drains the handlers. Calls are never resent. + +## 6. Startup: publication, then first outcomes + +1. **Publication:** `load` runs every facet's setup, then publishes all at once in one batch (types served, + registries served and composed once, contributions in their tables), then wakes (§4) after that batch. Nothing + activates inside it. +2. **No gate beyond it:** holds and `host.hold` wait only for the publication, never for the barrier. This ends the + startup deadlock: W's hold for C binds as soon as S's setups published; B goes live, then A. +3. **The barrier** (**Owner:** (a) or (b); recommended (a), three reviewers of four): + - **(a):** `load` resolves when every host-lifetime object of this `load` has a first outcome: + `firstOutcome(o) = o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b)))`, + where `local` means executing on this host (an object here, a keyed pin still loading here), not the contract's + kind. So `CodeMode → Harness` waits for the harness's outcome; a wait on another host or on a failed dependency + ends the first pass. One `firstOutcome` signal per object, separate from the readiness dependents wait for (Sol), + counted once, never reopened by retries. An object of the cohort that ends before its first outcome (its plugin + removed, the host closing) settles its entry as ended; `host.close` settles the waiting `load` (Astra R3). + - **What (a) promises:** every host-lifetime object on this host had its first try, and the contributions of those + that went live are in their tables. **Not:** every tool that will arrive (a parked or slow contributor, tools found + in `onLive`, MCP; Sol's probe `/tmp/barrier-sol-first-outcome.log`). What a first turn needs is p3's explicit + admission check. + - **(b):** `load` resolves after the publication; Chord adds `host.startup`, a watchable of the host-lifetime + objects' first outcomes with their causes; the app decides (Fable). The same tracker, awaited by the app. + - p2 today resumes its harness without waiting for all host objects (`apps/p2/src/hosts/worker.ts`), because its + worker objects that need server types are reachable only after `control.ready()`. + +## 7. Cycles (decided 2026-10-29: no probe) + +On one host, cycles are found as today: at `self.get`, following the waiting edges of activating and parked attempts +(exact). Across hosts, a hold carries the activation chain it was made in, which finds every ring whose objects start +one after another. The remaining gap, a ring across hosts whose objects start at the same moment, is not detected: +without a dependency deadline it stays parked. It is made visible: an attempt parked longer than its `readyTimeout` on +a remote attempt that is itself only `loading` is reported once per parking episode as `HostError` `parked_long +{ object, waitingFor, since }`; nothing fails or retries. A cycle is a plugin bug; the guide gives plugin authors and +agents four rules: (1) a factory declares only what the object needs, never a type that needs it back; (2) +back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data flows up through state and +registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare worker types in their factory. + +Rejected (owner, 2026-10-29): the probe (edge-chasing searches with a check along the recorded trail; seven review +rounds, `/tmp/model/review*-*.md`; complete only with a re-search timer or a full search lifecycle, the most intricate +code in Chord, for a plugin bug) and the slice 4b timeout narrowed (it polls chains across hosts again, misses a ring of +`waiting` attempts, and retries in lockstep without finding the ring). + +## 8. Getters, holds, calls: one rule + +- **`host.hold`** returns a reference at once; only invalid params throw (also before `load`). A missing local type + gives `failed { waiting }` after the publication, `loading` before it. +- **`host.get`, `ctx.get` and `host.registry`** (owner, 2026-10-29, decision 2): resolve when live; wait while `loading`, + `failed { transient }` or `failed { waiting }`, up to their `readyTimeout` (default 60 s, the caller's choice), then + reject with the last cause; reject at once only on `failed { broken }`. On + success, rejection or timeout their temporary wait goes; `host.registry` returns the `RegistryValue` with its + ordinary watch ownership (no `release()`). A UI holds and shows `ref.state`; code that needs the object now gets it, with the + patience it passes. +- **Calls** on a `loading` reference park, each bounded by the reference's `readyTimeout` from when it was made; on a + `failed` one they reject at once with the cause; sent calls are never resent. + +## 9. Registries + +- The same resolver on one host: a reader's request for R at its version is bound while R is served and has a usable + value at that version (composed; mapped by `accepts[old].value` for an older reader), otherwise unbound + `not_served`, `version` or `compose`, or `loading` before the publication. Woken at every transition to usable + (`#valued`'s first-value latch goes). S2's `broken` and S3's backoff go. +- **Version skew within one host:** a reload of the registry's plugin alone leaves other plugins' contributors and + readers on their bundled versions; `accepts[old].contribution` and `.value` keep them working, unsupported + contributions are left out and reported, unsupported readers are `waiting` (`version`). +- A registry waits for a value, never for a particular contribution (an empty table composes). After live, a later + `compose` failure changes nothing for the reader: `.value` keeps the last good composition, **but only while every + contribution it was built from is still in the table** (round 2, Astra N2): when one of them leaves (its + contributor ended or was reloaded), the last good value is dropped and `.value` throws until `compose` succeeds, so + it never hands out a native whose contributor ended. `compose` represents a missing contributor as a value. +- Registries stay host-local (contributions are in-process values). + +## 10. Keep-alive, presence, revocation, jobs + +- A request over a link counts once in `host.activity.references` from its arrival to its end, bound or unbound; + presence (`host.references`) lists only bound references. +- Revocation ends the requests of a revoked principal, bound and unbound, at every hop; nothing revives them. +- `gone` stays only for revocation (a provider host closing closes its links and sends nothing; its consumers see a + drop, `loading`, and hold again on the next link). +- Persisted job runs of a parked host-lifetime object are loaded with its documents; `run` and `finish` start only + once it is live; the run count keeps the object as today. + +## 11. Timeouts after the model + +| Timeout or backoff | After | Why | +| --- | --- | --- | +| `readyTimeout` of an attempt | only its own steps; paused while parked | a slow dependency is bounded where it runs | +| `readyTimeout` of `host.get`, `ctx.get`, `host.registry` | kept | a caller gets an answer | +| `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | +| identity backoff 1-60 s | only for the object's own `transient` failures | nothing else says the disk or network is back | +| dependency backoff, polling of refused holds | removed | recovery is an event | +| dependency deadline across links | removed | the simultaneous-cycle gap is a plugin bug, reported by `parked_long` | +| `parked_long` report | new: once per parking episode after `readyTimeout` on a remote attempt that is only `loading` | a diagnostic; nothing fails or retries | +| route backoff, heartbeat, `closeTimeout`, routine and `finish` backoffs, `linger` | kept | not dependency waits | +| rollout install timeout (not built) | a watchdog on the install only | never a deadline for dependencies | + +## 12. What changes in the design + +R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, +wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's +last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the +simultaneous-cycle gap: no probe, `parked_long`), §7 (`host.hold`, getters, calls), §21 (no `gone` on +removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision +fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `parked_long`. Unchanged: what an object's own code reload does (§21), R18 after live, +`retryOn` outside parking, `self.follow`'s R4, the call window and flow control, revocation, no client providers. + +## 13. Decisions (owner, 2026-10-29) + +1. The barrier: (a), `load` resolves at the first outcomes of this host's host-lifetime objects (§6.3). +2. Getters wait through `waiting` until their `readyTimeout`; calls on `failed` still reject (§8). +3. `waiting` is a new `FailureCause` kind (§2). +4. A pure swap whose new factory declares a dependency that is not there parks the object (its gate closes; holders + see `loading` or `failed { waiting }`; calls park or reject); no overlap of old and new code (§21). +5. The last good registry value is kept only while every contribution it was built from is still in the table (§9). +6. Cycles across hosts: no probe; `parked_long` and four authoring rules (§7). + +1, 3, 4 and 5 are the recommendations, taken when the owner said "just build this" (2, 6 decided in discussion). diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md new file mode 100644 index 000000000..4c56352f2 --- /dev/null +++ b/packages/chord/docs/next/design.md @@ -0,0 +1,2627 @@ +# Chord v2: target design + +The design of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) and of how apps (the +playground, p3) use it: **the one current statement of every rule** (owner, 2026-10-24). Every decision is recorded in +the decision log `packages/chord/docs/next/decisions.md` (the history and the reasons, kept complete) and folded into this +document in the same commit. Where a rule is built, the slice is named (`slices.md`); the code follows this document. +It replaces `apps/p2/docs/target-design.md`. + +Marks: everything is decided unless marked **Decision needed** (problem with an example, then options; nothing in such +a block is settled; section 26 lists them all). **(carried over)** marks behaviour of today's Chord that v2 keeps +without a new decision. API names are working names. + +Contents: 1 goals · 2 concepts · 3 hosts and links · 4 principals · 5 object types · 6 objects · 7 references · 8 ctx · +9 documents · 10 store and storage · 11 kinds of state · 12 derives · 13 natives · 14 routines · 15 jobs · 16 the facet +· 17 registries · 18 start data and platform bindings · 19 presence · 20 input into running work · 21 reload and rollout +· 22 wire · 23 optional components · 24 Durable integration · 25 apps: ownership and porting · 26 decisions needed · 27 +versions and adapters · 28 building: slices · 29 telemetry · 30 cost model · 31 rejected + +--- + +## 1. Goals and principles + +1. **Reload without restarts.** A plugin's new code takes over inside the running app: hosts keep running; data, open + connections and running work survive where they can. Reloads are explicit (a button or a command), never on save. + Base code (Chord, the app runtime) restarts the hosts. +2. **Ownership.** Every thing has exactly one owning object, the smallest it belongs to. The owner holds the data, + enforces its rules in its methods and offers the operations others need. Nobody reaches past it. +3. **Factories declare, Chord activates.** A factory is synchronous and only declares; Chord performs activation. + Handles a factory gets are valid for the object's whole life. +4. **Natives only in objects.** OS-level values are opened and closed by the object that owns them, and reach other + plugins only through that owner's methods, state or registry entries. +5. **Errors look like disconnects.** No request-id framework; consumers handle a failed call as they handle a dropped + link. References re-hold by themselves. +6. **Tagged unions, always.** States, results and outcomes are discriminated by `kind`; each state carries exactly its + own fields. +7. **Portable kernel.** A host is a logical identity plus an incarnation; facets may come pre-imported; values are + JSON plus bytes independent of the transport. Node first; Cloudflare Durable Objects and similar stay possible. +8. **Context last.** Parameters are the required arguments, then the context (`ctx` or a Durable `Context`), then + optional arguments: `self.tx(fn, ctx)`, `handle.change(key, fn, ctx)`, `chord(fn, context)`, `job.start(args, ctx, + { key })` (owner, 2026-10-21). +9. **Few concepts, no ceremony.** No special directories, no `defineNative`, no lints standing in for design. +10. **Aligned with Durable** (owner, 2026-10-27): where Chord and Durable solve the same problem (commits, lines, + loads, storage errors, watches), they use the same rules and mechanisms. + +--- + +## 2. Concepts + +| Concept | What it is | +| --- | --- | +| host | one participant (a process, a tab, a DO): a logical identity plus an incarnation | +| link | a connection between two hosts | +| host kind / address | typed description of addressable hosts / one host of a kind (`Session({ id })`) | +| facet | a plugin's declarations for one host kind: the types it serves, its contributions; no state or behaviour of its own | +| object type | a contract: name, params (which instance), host address, members, version | +| object | one instance per (type, params, principal key); `local`: per (type, params) | +| principal | who calls; stamped by links, typed by the app | +| reference | what `get` returns; keeps its object alive; live and hydrated when awaited | +| document | a named JSON value an object owns, written in transactions | +| derive | a value computed from watchables | +| native | an OS-level value opened with `self.open`; only in objects | +| routine | standing behaviour of an object: `self.on(source, fn)`, `self.onLive(fn)` | +| job | owned work a call starts that outlives it: intent, run, finish; optional state document per run | +| registry | an owner's extension point: one contribution per contributor, composed by the owner | +| shared store | a `local` object holding shared data once, behind per-principal `remote` views | +| `ctx` | per call: cancellation, principal, call-scoped references | +| revision | (server incarnation, sequence): the complete code set a host runs | + +--- + +## 3. Hosts and links + +### Roles + +A Node app has one shape: + +``` +client client client browsers, TUIs: not addressable + └──────┼──────┘ + server exactly one, addressable + ┌──────┼──────┐ +worker worker worker addressable by a typed address, started on demand +``` + +| Role | Addressable | Principal comes from | Lifecycle | +| --- | --- | --- | --- | +| client | no | authentication of its link | connects by itself | +| server | yes, one | its own declaration | runs | +| worker | yes, many | stamped by the parent | started on demand by the app | + +Clients serve only `local` types (they are not addressable): a host without a principal or without an address refuses +to serve a remote type (a declaration error: `load` leaves that plugin out and reports it, a reload keeps it on its +old code; a remote object is owned by a principal of the host serving it, owner, +2026-10-24; and placed by its host address, owner, 2026-10-29). No client providers for now (owner, +2026-10-28): client-side capabilities (a camera, WebMCP tools) go through an owner's requests that clients answer +(section 20, the Questions service). A client's local +calls have no principal; what a client calls on the server is stamped with the client's authenticated principal. +Clients reach workers only through the server, which relays; workers reach each other through the server too. A +detached process (e.g. a terminal host) would be another child kind of the server. Things that are not hosts: a code +mode sandbox (a worker thread inside a worker), remote environment daemons (their own protocol, owned by an object), +the TUI sandbox behind its launcher (the launcher only passes frames). A Durable Object deployment is a mesh routed by +the edge (section 24 and the research note). + +### Host kinds and addresses + +```ts +export const Server = hostKind("server"); +export const Session = hostKind("session", { id: str() }); // params like a contract's + +export const Notes = remote("notes", { + version: 1, + params: { session: str() }, // flat record of required scalars + hostAddress: ({ session }) => Session({ id: session }), // a typed function of the params + variants: { … }, +}); +export const Catalog = remote("catalog", { hostAddress: () => Server(), … }); +``` + +- `hostKind(name, params?)` declares a kind of addressable host; its params follow the params policy of contracts + (section 5). `Session({ id: "S1" })` is an **address**: `{ kind, params, key }`, the key built like an identity key + (`session|S1`); two addresses are equal when their keys are. No host-name strings appear in app code. +- **`hostAddress`** (owner, 2026-10-21 as `placement`, renamed 2026-10-29) is a typed function of the params returning + the address of the host that serves the object, required on every `remote` contract (owner, 2026-10-29); `local` + contracts have none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). +- **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its + identity (section 5). +- **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and + never needs the contract or parses params. **The provider computes it again** from the params: a hold naming this + host whose params the provider's code places elsewhere (code skew) waits unbound `version` (`failed { waiting }`, + `invalid/host_address`, dependency model §1; slice 7a); a hold naming another host's address is relayed, never served + here (misrouted holds are never captured by a relay that serves the same name). +- **Every reference is a request** (dependency model, slice 7a): it has one holder (an attempt, a call, outside code, + a link's peer) and ends only when its holder ends it (release, a call settling, a getter giving up, the peer's link + or origin closing, revocation, the host closing), never because its provider is missing, went away or does not fit. +- **Where a reference made on a host, or a hold arriving on one of its links, goes**, by address alone (slice 4a; + slice 7a: no same-name check on the consumer's host): + + | Case | Goes | + | --- | --- | + | a `local` type | here | + | a `remote` type whose host address (computed here from the params) is this host's address, served here or not | here | + | an arriving hold naming this host's address | here | + | the address (computed, or named by the hold) is of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | + | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | + | no parent link: the host never had one | `unavailable` | + | no parent link: it is down (a drop) | waits on the parent path, `loading`, until the next link (design audit, 2026-10-27) | + + A `hostAddress` or `authorize` that throws fails the request broken with that error; an address whose params fail + the kind's check is broken `invalid/host_address`. +- **The publication gate** (slice 7a): a request made on this host, or a hold arriving on its links, before the + host's first publication (`load`'s batch of declarations) is not placed yet: it is `loading`, and the publication + places it. The gate lasts only for the synchronous setups, never for activation, so it cannot deadlock. +- **Then the selected provider is checked** (here: served, the version and fingerprint, the params, the provider's + own host-address check; over a path: at the provider host). Version checks never look at a same-name registration + elsewhere. What the check finds: + + | The check finds | The request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address (or serves it as the other kind) | unbound `not_served`: `failed { waiting }` (`unavailable` "X is not served here") | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere | unbound `version`: `failed { waiting }` (`contract_mismatch`, `invalid/host_address`) | + | a registry served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` (§17) | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params, a cycle | `failed { broken }`, never indexed | + + The unbound reasons that wait for a name are indexed under it on the host that decided them: the consumer's host + for a request made there, the provider's host for a hold that arrived over a link (kept refused there; a `retry` + re-runs its start). A publication serving a name (`load`'s, or a reload's commit, which wakes only the requests of + the names it serves, slice 7b) wakes its requests, which run their check again + (a refused hold sends `loading`, then `bound` or `failed`); no new message kind. Index entries go when their request + ends (GC-tested). The consumer never re-holds a request by itself: no backoff, no polling. + +### The low-level host API + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +server.route(Session, { + authorize: (p, address) => p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === address.params.id), + resolve: async (address, context) => { + const child = fork("worker.ts", [], { serialization: "advanced" }); // no `signal`: see below + child.send({ id: address.params.id, init: await initFor(address.params.id) }); // the app's own start message + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, +}); +await server.load([{ name: "@app/core", facet: coreServerFacet(config) }, ...serverPlugins], ctx); + +wss.on("connection", (ws, req) => { // an entry point: app or adapter code + const principal = authenticate(req); + if (!principal) return ws.close(4401); + server.accept(webSocketLink(ws), { principal }, ctx); +}); +radius.onChannel((ws) => server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }, ctx)); +``` + +```ts +// worker.ts: connect first (W6), then load; start data through the core facet (section 18) +const { id, init } = await startMessage(process); +const worker = createHost

({ address: Session({ id }), principal: { kind: "worker", session: id }, onError }); +worker.connect(ipcLink(process), { principal: { kind: "server" }, trusted: true }, ctx); // stores may reference server types +await worker.load([{ name: "@app/core", facet: coreWorkerFacet(init) }, ...workerPlugins], ctx); // serves SessionInfo +await (await worker.get(Harness, {}, ctx)).resume(ctx); // tasks run only after every plugin loaded + +// a browser tab +const tab = createHost

({ onError }); +await tab.load(webPlugins, ctx); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +const notes = await tab.get(Notes, { session: "S1" }, ctx); // tab → server → route(Session) → worker +``` + +```ts +// a test: everything in one process +server.route(Session, { + authorize: () => true, + resolve: async (address) => { + const [up, down] = memoryLinkPair(); + const principal = { kind: "worker", session: address.params.id } as const; + const worker = createHost

({ address, principal, onError }); + worker.connect(up, { principal: { kind: "server" } }, ctx); // W6: connect, load, resume + void worker.load([{ name: "@app/core", facet: coreWorkerFacet({ id: address.params.id }) }, ...workerPlugins], ctx); + return { link: down, principal }; + }, +}); +``` + +Do not pass `context.abortSignal` to `fork`: `host.close` aborts it, which kills the worker and makes Node emit an +`error` event that crashes an app without a handler. Chord closes the link at `host.close`; the worker exits when its +parent link closes. `resolve` returns the link before the worker has loaded (the worker connects first, W6): made +later, the link would leave the worker's `hello` unread and the heartbeat would close it during a long `load` (slice +10's spec review; the deadlock once feared here does not happen: 7a's barrier ends its first pass on a wait on another +host). The workers component (§23) does this. + +| Call | Meaning | +| --- | --- | +| `createHost

({ address?, principal?, ...options })` | with an address: addressable, serves only objects placed there, acts as `principal` on its own host; without: a client. `P` is the app's principal type | +| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, publishes all of them in one batch (types and registries served, registries composed once, contributions; requests waiting for the publication are placed after it), then waits for the start barrier (section 21: the first outcomes of its host-lifetime objects, slice 7a); a plugin whose setup throws is left out and reported (`load` resolves with `{ failed: [{ plugin, error }] }`, decision "Per plugin at startup and reload"); runs once. Loaders are adapters; base code passes its core facet the same way | +| `reload([{ name, facet }], context)` | the complete new plugin set after `load` (a facet passed as the same object is an unchanged plugin): staging, then one commit, then every object of a changed or removed plugin takes its fate; resolves with a report (a plugin whose new code fails keeps its old code, listed in `failed`); one at a time (section 21, slice 7b) | +| `connect(link, { principal, trusted?, codec? }, context)` | the link to the parent, at most one open; once it closed (a drop), `connect` takes the next, and every remote reference over the parent is held again on it. `principal` = who the parent is | +| `accept(link, { principal, trusted?, codec? }, context)` | a link from a child; holds and calls on it act as `principal` | +| `route(Kind, { authorize, resolve }, context)` | how to reach hosts of a kind (below); called before `load`, once per kind; work arriving on its links runs under `context`; returns a `RouteHandle`: `retry(address)` resolves the address again (D10b), `suspend(address)` stops resolving it until `retry` (R1) | +| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live, waiting through `loading`, `failed { transient }` and `failed { waiting }` up to its `readyTimeout` (section 7) | +| `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | +| `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | +| `revoke(r, context)` | revocation (section 4; slice 4b) | +| `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI, an HTTP request): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8); an omitted `principal` acts as the host, an explicit `undefined` runs anonymous (slice 10) | +| `registry(R, context, { readyTimeout? })` | registry `R`'s composed value for code outside objects once it is usable; waits for the publication, then while it has none up to its `readyTimeout`, then rejects with the last error (slice 8, 7a, section 17) | + +**Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), +`telemetry` (section 29), `onError`, `closeTimeout`, `readyTimeout`, `heartbeat: { ping?, close? }`; later `revokedBy` +(section 4, slice 4b) and `store` (section 10, slice 6a: the store, opened by the app with `openStore(backend, ctx)` +and closed by the app with `store.close(ctx)` after `host.close`; the host never closes it). **`onError`** receives everything that fails in the +background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery +threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, +native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), the registries' `registry_conflict { registry, message, plugins }`, `registry_failed { registry, error }`, `contribution_failed { registry, plugin, type?, error }` and `contribution_refused { registry, plugin, type?, version }` (each once per episode, slice 8), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6), `parked_long { object, waitingFor, state, since }` (an attempt parked longer than its `readyTimeout`, whatever its blocker, once per parking episode: a diagnostic of a ring across hosts started at the same moment or a long wait, slice 7a, audit A11, §6) and `storage_failed { error }` +(slice 6a), which every app must handle: Chord never ends the process, +the app exits with its own code for its supervisor or reopens. + +**Timeouts** are host options; no fixed numbers in the code. Two host timeouts (C13, owner, 2026-10-22): + +| Timeout | Bounds | Host default | Override | +| --- | --- | --- | --- | +| `closeTimeout` | a native's `close` (then reported as a possible leak; the rest continues); the drain of an ending object's running handlers (callers are rejected at once) | 30 s | per native: `self.open(open, close, { closeTimeout })` | +| `readyTimeout` | an activation attempt's own steps (its wait for the previous instance's teardown included; paused while the attempt is parked on its dependencies, slice 7a); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`/`registry`: `{ readyTimeout }` | + +The park deadline's per-type override is not built (only if a real case needs it). Timeouts of later slices belong to +their mechanisms: the routine handover uses `closeTimeout`, a job `finish` backs off 1 s to 60 s, provider-side +`linger` is per type; a reload has no install timeout (it takes the app's context, slice 7b); a per-type drain +override only if a case needs it (section 26). A route's `resolve` is bounded by the app. + +- **Incarnations** are minted by Chord (uuidv7, code copied from pi-ai, no dependency), never passed in, and carried + in the link handshake (`hello`): the other side sees a restart as a new incarnation of the same identity; + references to it re-hold. +- **Entry points vary** (loopback or tunnel WebSocket servers, Radius dialing out and delivering channels, a DO edge, + memory), so the kernel has no `listen`; each entry point authenticates and calls `accept`. Radius no longer needs to + pose as a loopback request with a secret header. +- **App code, not Chord:** starting processes, init data, authentication, keep-alive decisions. Helpers come only when + apps repeat the same code. +- Compared with today's Chord: same shape (`createHost`, `connect`, `accept`, `route`, `load`); gone are string host + names and prefix parsing, app-managed generations, the duplicate `upstream` option, `opaque` runtime objects and the + loader inside the kernel. +- **Worker keep-alive** works as in p2: the worker counts, the app's supervisor decides. **`host.activity`** (slice 4b; + owner, 2026-10-24) is a JSON `Watchable` of `{ references, calls, jobs, lingering }`: holds arriving over links, + method calls running on this host, running jobs, provider-side lingering objects. Counters are kept where the work + happens; a change marks it dirty and it is published at most once per microtask turn with changes. A hold placed + here counts from its arrival to its end, bound or not (one waiting for a missing provider keeps the worker alive; + slice 7a); a hold this host relays counts at its destination; `host.references` lists bound references only. Core code + reports it plus Durable tasks (the core `Harness`); the server's supervisor stops the worker when all are zero for + the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle logic. + +### Routes + +```ts +host.route(Session, { + authorize(principal: P, address: Address): boolean; + resolve(address: Address, context: Context): + Promise<{ link: Link; principal: P; trusted?: boolean; codec?: Codec }>; +}); +``` + +- **`authorize`** runs for every hold routed over the route, with the hold's principal (section 4), before anything + starts; `false` fails the hold broken `forbidden`. +- **`resolve`** runs at most once at a time per address: the first hold starts it, later holds wait for the same + link. The returned principal is stamped on that link; `trusted` and `codec` as for `accept`. Its `context` is + aborted only when the host closes; a link it returns after that is closed at once. A `resolve` whose holds were all + released meanwhile finishes and its link is kept (a worker start that produced a link finishes even if its first + requester left). A `resolve` that never settles is the app's (it bounds its own `fork`); getters still time out. +- **Failure and re-resolve** (slice 10 R2, owner, 2026-10-30; replaces the per-address backoff): a route has no + timer. A rejected `resolve` (reported `resolve_failed`) or a returned link already closed fails the route's holders + with its error (`failed`, not an endless `loading`); the route resolves again only for a new hold, on + `RouteHandle.retry(address)`, or once after the address's link dropped while holds exist (a drop of a healthy link + recovers at once; whether a crashed worker comes back is the `resolve`'s answer). `RouteHandle.suspend(address)` + (R1) stops resolving an address (its holders stay `loading`) until `retry`. A network route that wants to reconnect + keeps its own policy in its `resolve`. The workers component answers a failed address's `resolve` with its stored + failure at once: workers never restart by themselves (§23). +- Chord closes a route link itself only at `host.close`, on the heartbeat and on a protocol error; stopping idle + workers is the app's (`host.activity`, the workers component, §23). +- Work arriving on a route's links runs under the context given to `host.route(Kind, options, context)` (slice 4b). + +### Links + +- **A link is delivery only:** an ordered, bidirectional channel of `Uint8Array` messages; adapters only move bytes + and close both ends. Session recovery is the adapter's policy. Implementations (slices 3a, 4a): + - `memoryLinkPair()`: delivers with `queueMicrotask` (no Node APIs); closing one end closes the peer a microtask + later. + - `ipcLink(port)` over a Node IPC channel (`fork` with `serialization: "advanced"`, or `process` in the child), + typed structurally (no Node import in Chord). A `Uint8Array` crosses as one message; other messages on the + channel (the app's start message) are skipped. It listens from the moment it is made and queues until its + listener is set. `send` returning `false` is backpressure, not failure; a thrown `send`, an error callback or + `disconnect` closes the link once; listeners are removed on close. + - `webSocketLink(socket)` over a WebSocket (`ws` in Node, the browser's), typed structurally: one binary frame per + message, `binaryType = "arraybuffer"`; messages sent while the socket connects are buffered until `open`; a socket + closing or closed already gives a closed link; `close`, `error` and a text frame close it. The transport's + `maxPayload` stays its hard cap (code 1009 closes the connection). +- **One link object per link carries both directions** (slice 4a): the channel (handshake, codec, checks, + heartbeat), a **provider side per origin** (holds and calls the peer sent, its own or those of a relay's downstream + link, by the peer's ids) and a **consumer side** (holds and calls this host's own references sent, by its own + counters). Message kinds are directional, so each goes to its side. A link carries nothing until a hold or call + (W6). +- **Roles:** `parent` (made by `connect`), `child` (`accept`), `route` (`resolve`). The role decides whose principal a + hold acts as (section 4). **Trusted links** are an explicit option (`trusted: true` on `accept`, `connect` and in + `resolve`'s result; default untrusted; never inferred from the link kind; owner, 2026-10-23): they skip only the + `MESSAGE` schema check; framing, the codec's decode, `checkParams`, `checkArgs` and Delta decoding still run. Trust + never grants the principal fields (they are read only on the parent link). +- **Remote objects belong to a path:** the parent, or a route address. Each new link of a path holds every remote + object of that path again (with new hold ids); a closed link drops its tables and the objects go `loading`. +- **Heartbeat on every link kind, in the protocol layer** (owner, 2026-10-23): each link keeps a `lastReceived` + timestamp (every received message; sends do not count); the host runs one interval timer for all its links (none per + message). A link silent for 15 s sends `ping` (after its handshake); the peer answers `pong` at once, never queued + behind the window. A link silent for 30 s, the handshake included, is closed (its references go `loading`). Both + numbers are host options (`heartbeat`). The heartbeat detects a silent transport (TCP does not notice a sleeping + laptop), not a peer that answers pings but withholds acks (bounded by the window and the pending cap instead). A + handler that blocks the event loop longer than 30 s gets its links closed (documented). +- **Flow control: an ack-clocked window per origin** (owner, 2026-10-23; per origin 2026-10-24). Per (link, origin), at + most **K = 2** of the provider's own `bound` and `state` messages are unacknowledged; only these two kinds count + (everything else is control, never queued behind them). The consumer sends `ack` (with the message's origin) after + applying each, after its batch; an `ack` crossing nothing in flight is ignored. While the window is full: + - each touched hold member keeps its ops in its **pending queue** (section 11: raw, never mutated, folded through + the leaf index, at most 10,000 ops, then only "replace with the current value"); when an `ack` frees a slot, one + `state` message carries every waiting hold's entry, its members' pending ops encoded once (or `[["r", value]]` + after an overflow), all reflecting the latest commit, still atomic for the consumer. So a slow consumer gets + coarser updates, never stale ones (measured: a `bufferedAmount` cap alone left state 9.4 s old behind kernel and + proxy buffers; acks kept it 0.0 s old); + - a hold that binds waits for a slot; its snapshot and subscription are taken in one synchronous step when its + `bound` is sent; bindings and waiting state take turns when both wait, so neither starves; + - a hold whose object leaves `live` while its `bound` waits goes back to pending (D1, owner, 2026-10-24; built in + slice 7b with the reload, which makes a live object leave `live`); so does a bound one, unbound (§7, §22). + Not built, by decision: a byte window, credit messages, snapshot admission or debt, an adapter `backlog()` API, + `reset` from anyone. +- **Calls: a call window N** (default 256, the same on every host) announced in `hello`: how many concurrent calls a + side accepts **per origin**. A side sends no call before the peer's `hello`. A host sends at most N of its own calls + per link in flight; further calls wait in the link's **call queue**, one FIFO, sent in order as replies free slots; + nothing is refused for being "too many" (K16). A slot is freed once per call: by its `result` or `error`, its + `abort` (when sent, at the caller; when received, at the provider), its hold's `release`, `gone`, its origin's + `closed` or the link closing. A call still queued when its hold rebinds (`loading`) is rejected `unavailable` (it may + name a method of the old variant). A provider seeing more than N running calls of one origin closes the link + (protocol error); a call naming a hold it does not know is ignored before it is counted (Q1). A per-link N, if ever + needed, must bound a relay's forwarding too. +- Holds are not limited by count (authorization and params checks protect them); size limits are a separate check + (section 22). Byte-heavy traffic (terminals) uses separate channels (section 23), so a large state message never + holds up keystrokes. + +### Relays (slice 4a; decision "Relay flow control per origin", owner, 2026-10-24) + +A host that receives a hold it cannot serve (the table above: a route or the parent) **relays** it: it holds +it again on that link (upstream) with the arriving hold's principal and incarnation, and forwards between the two +without parsing payloads. The provider keeps the window, the pending queues and the calls per origin; the relay never +drops, folds or resets. + +- **Origins:** the relay gives each downstream origin (a downstream link, and the origin its messages carry when that + link is itself a relay's) a number, one counter per host, never reused; 0 is a host's own work. Every message + upstream for that origin carries `origin`; the provider keys holds, calls, windows and acks by (link, origin, id), and + its answers carry the origin back. `origin` is read only on parent and route links, where a relay's messages + arrive (a worker may relay a child's hold to its parent); on a child link it is a protocol error (decision "Origins + only from relays", 2026-10-31); a per-origin record goes once it has no holds, calls or messages in flight; + principals and incarnations stay parent-only. +- **Ids are not renumbered:** an upstream hold has the downstream hold's id under the origin's number; calls the same. + Two tabs (two holds) of one principal are two upstream holds, each with its own encoders (R1). +- **Nothing is parsed:** the relay routes by the hold's address and authorizes it; params, arguments, results, + snapshots and ops travel as opaque parts. Cost per message: decode the envelope, swap the origin, encode the + envelope; parts copied once into the outgoing buffer. +- **What the relay keeps:** per relayed hold, its upstream link and its original `hold` message (params copied once), + which routes `call`, `abort`, `retry` and `release` upstream at once and holds again on the next link; per + downstream origin, the FIFO of the sender of each `bound`/`state` message in flight and the set of upstream links it + relayed to. +- **Forwarding up:** `hold`, `call`, `abort`, `retry`, `release` with the origin's number; a hold waiting for its + route is sent when the link exists; a `release` before that drops it. A call is forwarded at once; with no upstream + link it is answered `unavailable` (it crossed the `loading` notice). Each downstream consumer is bounded by its own + N, so the upstream sees at most N per origin. +- **Forwarding down:** a message from upstream with an origin goes to that origin's provider side, which forwards it + with its own origin (none for a tab), ids and parts unchanged. **A link answers only for what was relayed over it** + (owner, 2026-10-24): every entry of a `bound` or `state`, every `failed`, `loading` and `gone` by its hold, every + `result` and `error` by its call is checked against the relay's table; an answer for a hold or call not relayed over + the link it came from is dropped (Q1: also one crossing a release). A `bound` or `state` left with some entries is + forwarded with those; left with none, it is acknowledged to that link at once. A message for an origin from a link + the origin relayed nothing to is ignored whole. +- **Acks:** the relay forwards each upstream `bound`/`state` at once and returns its `ack` to the upstream link that + sent it when the downstream acknowledges it (the senders FIFO). A tab served by W workers may have 2W messages in + flight (plus the relay's own 2), each worker's window bounding its own. A slow tab stalls only its origin at each + worker; other tabs are other origins. Measured: a slow tab through the relay catches up like a direct one (a 60 s + stall: 0.38 s, 23.7 KiB at 64 KiB/s; before the rework, 24 s and three full snapshots). +- **Upstream drop:** the relay sends `loading { hold }` downstream for every relayed hold on the dropped link, fails + their relayed calls in flight `unavailable` (never resent), re-resolves and holds again upstream from the kept + message (same id, same origin); the new `bound` is forwarded. Acks for messages of the dropped link still in flight + downstream go nowhere. +- **Downstream drop:** the origin's provider side ends: its served holds are released, its relayed holds forgotten, + and every upstream link it relayed to gets one `closed { origin }`, which releases the origin's holds and aborts its + calls there; the origin's number is never used again; messages crossing it are dropped. +- **`gone`** from upstream is forwarded; the relayed hold is forgotten. Relays never make a `gone`. + +### Recovery + +- After a drop the consumer re-holds every hold of the path over the new link and receives snapshots (a `bound` on a + bound hold replaces its replicas in one batch, another variant allowed); calls in flight fail like on a disconnect; + calls made meanwhile park (section 7). +- A slow connection never ends an object: nothing releases a hold for falling behind (the reference count is unchanged). +- A link drop is a departure (section 19). + +### Values + +- **JSON plus `Uint8Array` leaves**, with a bytes schema type, defined independently of the transport. Every link + carries the same `Message` through its codec (section 22); parts are encoded at their origin; byte segments travel + raw, never base64. Same-host calls (no link) pass values by reference. +- Schemas are plain JSON Schema built with `@earendil-works/pi-schema` (decision "The schema switch") and must + describe data (no callbacks, no native types, no runtime codecs); `format` is an annotation, never checked. Method arguments are + validated at the provider for every caller against closed schemas (section 5, R7, Q12, Q15); results and state are + never validated (section 30). Values are immutable by convention once handed over (Delta's rule): nobody mutates a + value it got, and nothing is copied or frozen to enforce it. There is no development mode (R9): one behavior + everywhere. + +--- + +## 4. Principals + +- The app defines its principal type `P`: JSON, every variant with a `kind`, a type alias (not an interface), + unversioned (owner, 2026-10-22): Chord never persists principals, base code changes every host at once, links stamp + them. Objects persist a projection with its own schema, never the principal (p2: chat's `Author { kind, id, name }`). + p2/p3: `owner` (`access: "local" | "relay" | "device"`), `guest` (`session`, `mode`, grant revision), `server`, + `worker` (`session`). +- **Links stamp principals.** A caller's own data never sets its principal. +- **The principal key** maps a principal to its object identity (`principalKey`, default canonical JSON; cached per + principal object; the same function on every host). It should include connection provenance and grant revision, so + a revoked grant's objects are distinct from a new grant's **(carried over)**. +- **Principals on holds** (slice 4a; owner, 2026-10-23): a `hold` carries `principal?` and `incarnation?`. The receiver + uses them **only on its parent link**; everywhere else they are ignored and the link's principal and peer + incarnation apply (a tab cannot claim to be the owner; a relay overwrites what a client sent). Calls carry neither: a + call runs as its hold's principal and with its hold's incarnation (refines R1/Q13, which stamped calls too). What a + host sends in those fields when it holds over a route link (towards a child that honours them): + + | The reference was made by | `principal` | `incarnation` | + | --- | --- | --- | + | `host.get` / `host.hold` | the host's own | this host's | + | `ctx.get` in a handler | the handler's `ctx.principal` (the caller's) | the handler's `ctx.incarnation` | + | `self.get` in a remote object | that object's principal | this host's | + | `self.get` in a local object or a facet | the host's own | this host's | + | a relayed hold | the arriving hold's principal | the arriving hold's incarnation | + + So a guest's call on the server whose handler `ctx.get`s a worker type reaches the worker as the guest and gets the + guest's variant; two references with different principals are two remote objects. +- **Upward is a boundary:** a worker's hold of a server type goes over its parent link, and the server ignores + principal fields from a child, so it acts as the worker, whoever the worker's handler serves. A worker that acts for + a user towards the server passes the user as data and the server object decides. +- **The caller's incarnation** (R20, Q13): `ctx.incarnation` is the incarnation of the host where the call started + (from the handshake, or from the hold's field on a parent link), next to `ctx.principal`, not part of the principal + or its key, never sent as data by apps; it is this host's own for local calls (section 25 uses it as a fence). +- **Revocation** (slice 4b) is a typed app function, not field matching: + + ```ts + type Revocation = { grant: string } | { device: string } | { session: string }; + const server = createHost({ + address: Server(), principal: { kind: "server" }, onError, + revokedBy: (p): Revocation[] => + p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" ? [{ device: p.device }] + : [], + }); + server.revoke({ grant: "g1" }, ctx); + ``` + + `host.revoke(r)` ends every object and closes every child link whose principal `p` has `r` in `revokedBy(p)` (compared by + canonical JSON of each value `revokedBy` returns, computed once per principal; never the host's own principal), and propagates `r` to the children as a `revoke` message on its route links, + honoured only on a child's parent link, where each applies its own `revokedBy` (the same app code; nothing is + replayed to a child connected later). Holders of a revoked object get `gone` with `forbidden/revoked` (owner, + 2026-10-24: a reason of its own, so a UI can say "access revoked"); holds still waiting for `load` are ended too. + Order: the `revoke` message goes out on the route links before the revoked child links close, so a worker's objects + end `forbidden/revoked`, not `released` by the tab's `closed`. Revocation ends a principal's objects at + once, whatever references or jobs remain (R11); their jobs get the abort signal and the normal `closeTimeout`. + There is no re-stamping: a changed grant (mode `write` → `read`) is a revocation of the old one; the client + reconnects and authenticates anew, and its references re-hold. Reconnecting never restores revoked authority. Work + started on behalf of a revoked principal in a shared store stops when its reconciling routine runs (R10's window, + accepted; section 25); there is no revocation hook. +- `local` objects have no principal in their identity; their handlers and references act as the host (section 5). + +--- + +## 5. Object types + +```ts +export const Notes = remote("notes", { version, hostAddress, params, variants }); +export const NoteStore = local("notes.store", { version, params, members }); +``` + +| | `remote` | `local` | +| --- | --- | --- | +| reachable from | any host, across links | its own host only | +| identity | (type, params, principal key) | (type, params) | +| principals, variants | yes | no | +| factory | `(params, principal, self)` returning `{ variant, members }` | `(params, self)` returning the members | +| arguments | validated once at the provider for every caller (compiled closed schema, R7, Q12); unknown fields rejected; passed by reference on the same host | validated the same way (Q15), never copied | +| results | not validated; passed by reference on the same host | not validated, not copied | +| members | state (JSON) and methods (JSON plus bytes) | plain-data state and methods; no `opaque`, no callbacks | +| `ctx.principal` in methods | the caller | the host's principal (none on a client); references a local object makes act as the host | + +```ts +variants: { + editor: { state: { notes: NotesSchema }, methods: { add: method(AddArgs, NoteId), remove: method(RemoveArgs) } }, + reader: { state: { notes: PublicNotesSchema } }, +}, +// the factory chooses one variant per object (slice 3a, C15): +host.serve(Notes, ({ session }, principal, self) => + principal.kind === "owner" ? { variant: "editor", members: { notes, add, remove } } : { variant: "reader", members: { notes: publicNotes } }); +``` + +- **Names** (owner, 2026-10-27): object type names, host kind names, document (`defineState`) and job (`defineJob`) + names use `[a-z0-9@/._~-]`, lowercase, checked at declaration (they become SQL table names in the store, joined with `:`). +- **Keyed state members** (owner, 2026-10-26; `local` contracts only): `members: { notes: keyed(NoteSchema) }`; the + provider returns a keyed document handle; a consumer on the same host takes `ref.notes.at(key)` in its factory (a + counted pin: the key stays loaded while the consumer lives) or `ref.notes.watch(key, ctx)`/`get(key, ctx)` anywhere; + an absent key reads `initial()`, a retirement arrives as the `retired` flag (a derive over it carries the flag on). + Across links, a view per entity (`NoteView { session, id }`) is held as an ordinary remote object. +- **Methods:** `method()` (no arguments, no result), `method(args)`, `method(args, result)`, `method(undefined, + result)`. A method without arguments omits them in the API (`ref.bump(ctx)`, implemented as `bump: async (ctx) => + …`); `null` is their internal and wire form (owner, 2026-10-22). +- **Implementations** (owner, 2026-10-22): a state member is a JSON `Watchable` the factory returns (a document handle, + a derive, a reference member, `constant(...)`), never a getter; a method is a function `(args, ctx)` or `(ctx)`. + Activation checks what the factory returned (plugin code): exactly the variant's members, methods as functions, + state members as JSON watchables (a `NativeWatchable` or anything else fails the object `broken` naming the member, + Q14). +- **Reserved member names:** `then`, `release`, `state`, `retry` and `variant` are refused at declaration (a reference + has them; `then` would make an awaited reference thenable). +- **Compatibility (strict, like documents; R6):** a hold carries only the consumer's contract `version` and + fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7b) when the + version is one it accepts and the fingerprint equals the one computed from its own history (section 27). Every + contract change, additive ones included, bumps `version`. Anything else leaves the request unbound `version`: + `failed { waiting }` with `contract_mismatch` (slice 7a: not the consumer's fault; a publication that fits wakes it, + §3). The fingerprint covers params, variants and members, not the host address. A consumer whose contract does not + fit waits on its own request; the provider object and its other bindings are not affected. +- **Variants:** a member name has the same kind (state or method) in every variant that has it, and a method the same + signature (arguments and result) in every variant; state schemas may differ per variant (redaction). Enforced at + the type level in `remote(...)` and at declaration at runtime. A remote factory returns one variant, `{ variant, + members }`; `ref.variant` is the discriminant (section 7). Calls are checked against the binding's variant + (`invalid/member` outside it). +- **Params** (owner, 2026-10-21/22): a flat record of required fields, each `str()`, `int()`, `bool()` or `oneOf(...)` + of string literals, constraints allowed (`str({ pattern })`, `int({ min: 0 })`); no defaults, optional, nested or + unknown fields. Checked for every caller with one compiled closed-object pi-schema check (a missing or extra field + fails `invalid/params`). The identity key is the field values in declared order, each escaped, joined with `|`. + Params are never copied or frozen: the object keeps the caller's object, immutable by convention. +- **Arguments (R7, Q12, Q15):** always validated with the provider's schema, whoever the caller is (trusted code is + not type-checked code: plugins are bundled without type checks), never with a schema a peer sent; object schemas are + closed (`obj()` sets `additionalProperties: false`), so an unknown field fails `invalid/args` instead of being + stripped; nothing is copied or mutated; arguments have no defaults (optional fields are `opt()`, handlers write + `args.limit ?? 50`). Same-host arguments are not checked for plain data (no prototype or getter checks): callers pass + plain data by convention. Provider schemas compile lazily at first use, cached in `WeakMap`s keyed by the schema + owner; accepted versions' validators also on first use. Bytes are leaves (`instanceof Uint8Array`, optional length + bound), never walked. Results and state are not validated. +- **Providers** (`bindProvider(Type, accepts)`, L1): runs `checkAccepts` once per load and reload and keeps the + validated mappings; per call `checkArgs(provider, method, raw, boundVersion, context)` returns a tagged value, + `{ kind: "method"; args; result? }` (current-shape arguments and, for an old-version caller, the result mapping back) + or `{ kind: "call"; args; call }` (the provider's own replacement of a removed method, invoked by Chord, slice 7b). The + binding is held only by the served type's registration and by calls in flight (GC-tested; no global strong cache). +- **Errors** (L1, slice 3a): `ChordError` with a code and, for some codes, a reason, a closed union that is part of the + link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): + `invalid` (`args`, `params`, `member`, `host_address`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` + (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, + `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names pi-schema's JSON Pointer path ("unknown field", "missing", else + pi-schema's message; `~` and `/` in keys escaped as `~0`, `~1`). Plugins throw `PluginError(code, message, + details?)` for codes of their own; anything else crosses a link as the plugin code `internal` with its message, never + a stack or a cause; plugin errors never use Chord's codes. An error rebuilt from the wire is not copied and only its + envelope is checked; a malformed envelope is a protocol error. +- **Methods return data,** never objects; an object is handed out as its address and the receiver does `ctx.get`. +- **No callbacks as arguments,** on either kind: behaviour goes through registries, multi-step work through jobs, + input through flows, runtime services through core objects. Real schemas instead of `Type.Unsafe`. + +--- + +## 6. Objects + +### Lifecycle + +``` +(get) → activating (parked on its dependencies → own steps) → live → ending → ended + └→ failed (backoff) → activating … + live → failed (a runtime failure of the owner, slice 6a: broken, by reload or retry only) +``` + +- **One object per identity.** The first `get` runs the factory; later gets share the object. +- **Lifetime follows from kind and params (Q-A, owner, 2026-10-21):** there is no `lifetime` option and no lazy flag. + A `local` type with empty params lives as long as the host: started eagerly at host start, never ended by reference + count (it may hold persisted data, P3, and contribute to registries, R21). Every other `local` type and every + `remote` type lives while referenced (`linger` keeps it briefly after the last reference). No remote object has host + lifetime: its identity includes the principal, and long-lived parts belong in a local shared store. Below, + "host-lifetime object" means a `local` object with empty params. Running and finishing job runs keep their + object alive as a count, not as references, while an attempt is live or activating ("Slice 6a small items" (12)); when + the count reaches zero with no holders, the object ends (§15). +- **Activation** (one fresh attempt each time, bounded by `readyTimeout`, its wait for the previous instance's + teardown included): the factory runs; singleton documents take `initial()` and, on the object's first activation, + persisted documents load and its tables' definitions are checked and migrated (slice 6a; the deadline paused during + eager migration work); its requests are made (`self.get`, `self.registry`, keyed pins), and the attempt is + **parked** until every one is live and hydrated (below; the deadline paused); then its own steps run under the + deadline: natives open one after another, followed sources attach, the returned members are checked (section 5), + derives compute; routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its + **gate** until then. +- **The seal fence** (owner, 2026-10-26): every attempt has a seal token (a job run invocation its own, for its state + changes and outcome). After the seal, new work through the attempt's handles is refused at use (`self.tx`, `ctx.get`, + a job or routine start); commits are checked when they enter the store's commit step (sealed: rejected `released`, + nothing written) and at adopt (sealed during the SQL: durable, adopted into the object's documents, its staged job + changes adopted into the object too: decision "Second spec review: the majority" (3)); at the seal a transaction not yet in the step is aborted and the line released, a started step + finishes before teardown completes; cleanup (references, presence, activity counts) still runs. A pure swap is not a + replacement. +- **Ending** (slice 1, 2): new calls are rejected; every caller of a running call gets `released` at once; the + handler keeps running in the background under an aborted context and its `self.tx` throws `released` (a running + method can change nothing after the end). Each call runs under a context aborted by its caller and by the object's + end (one controller per attempt, linked until the call settles; a caller's cancellation aborts only its own call); a + call whose caller context is already aborted is rejected before its handler runs. Then the **drain** waits for the + handlers' own promises, bounded by `closeTimeout` (`drain_timeout` reported), natives close in reverse order, + documents are dropped, derives detached, and references released last (closes may use them). Routines stop and jobs + follow their rules (slice 5). +- **A live object can fail** (owner, 2026-10-26): a runtime failure of the owner (a `migrate` failing on access, a + stored version newer than the code) seals the live attempt and makes the object `failed` (`broken`): calls reject + with the cause, state members keep their last values, natives close, documents stay on the object; recovery by + reload or `ref.retry()`; live dependents stay live. Its references keep following every member (its own and + re-exposed ones) until the commits admitted before the seal adopted and published once, then take their last values + (audit A10, 2026-10-30: one rule, as restarts), and a retry's factory waits for that. +- **Parked on its dependencies** (dependency model §3, slice 7a; supersedes R18's dependency deadline and backoff): a + dependency is what an attempt declares and needs before it goes live: `self.get` (local or remote), `self.registry`, + keyed pins (`at(key)` on another object's keyed member); `self.follow` is the attempt's own step, not a dependency. + While any dependency is not live (and hydrated), the attempt is parked and its object shows: + + | Its dependencies | The object shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting }`, `retryAt` undefined, error `unavailable` "waiting for X: " (X the first failed one in declaration order, the root cause found through its message's "waiting for …: "; always these two, so a ring of waiting attempts settles after one round; within 500 characters, X's label clipped first, fix round 1) | + | one failed with `invalid/cycle` (a dependent off the ring) or `forbidden` (an `authorize` that refused it: authority, M§3.6) | `broken` with that error: nothing heals it but `ref.retry()` or a reload | + | one `ended` (revoked, its host closed) | `broken` with that error: the request cannot come back | + + **Nothing times out while parked:** no dependency deadline, no backoff (only `parked_long`'s diagnostic report, below); the attempt's `readyTimeout` deadline is paused + (as eager migrations pause it) and resumes with the budget left; the identity backoff stays for the object's own + `transient` failures only. **Wakes:** the parked attempt checks again at every change of a dependency (live, + loading, failed, a new cause compared by value, ended, and its first outcome, below), coalesced per object after the + batch that caused them (a change while it checks or loads pinned keys is remembered); it never re-runs its factory. + A parked attempt keeps its documents, declarations, requests and seal token, and holds no natives. **The dependency + check is the boundary:** once all are live (and the pinned keys loaded), the own steps run; a dependency that leaves + `live` afterwards changes nothing (R18 after live: the object goes live; its calls through that reference meet its + state). **Retry passes on:** `ref.retry()` (or a `retry` message) on an object parked `failed { waiting }` passes on + to its failed dependencies (audit A5, 2026-10-30): on one host one retry's walk keeps a visited set, so a diamond + retries a shared dependency once; across links the `retry` message carries a hop budget (`hops`, 16 from the first + host, one less per host, dropped at 0), so a waiting ring across hosts passes it round until the budget runs out + (starting no attempt) and a diamond across hosts may retry a shared dependency twice (harmless). A `broken` or + `transient` one starts an attempt, a parked one passes it on, a request waiting unbound in an index is left as it is + (an event wakes it), a refused hold over a link re-runs its start. A `retryOn` wake while parked does nothing. When + a parked object's last reference goes, it ends at once (it is not live: no `linger`) and releases its requests. + `broken` is only for an object's own code, cycles and authority; no failure keeps references (the R18 retry trigger + is gone: parking replaces it). +- **Dependency cycles are rejected (Q5):** every object instance has a uuidv7 instance id, minted at its first + attempt; an object that restarts gets a new one, so an old chain never matches (slice 4b, the first user; D4, + owner, 2026-10-24). On one host, at + `self.get(B)` during A's activation Chord follows the waiting edges of activating objects from B; reaching A is a + cycle. Across hosts, a hold made during an activation carries the activation chain of instance ids in its header + (bounded; ignored from untrusted links): the provider rejects the hold when the instance being activated is in the + chain (slice 4b; on one host slice 1; at most 32 entries; locally created objects inherit the activating chain). A + cycle fails the activation `broken invalid/cycle` with the cycle as cause ("A → B → A"), and every other activating + object on it the same way; no references are kept. A dependency failing with `invalid/cycle` makes its dependent + **broken** with the same error (owner, 2026-10-24: it would wait on a ring that never heals); dependents that are not + on the cycle break too, and a reload or `ref.retry()` recovers them. The walk on one host follows the waiting edges + of activating and parked attempts (exact). Checked again after a reload. Cost: O(chain) per declared reference, + only during activation. + **The gap: rings across hosts started at the same moment** (owner, 2026-10-29: no probe): a chain is fixed when its + object is created, so a cycle entering an object already activating is not detected. Example: A on the server and B + on a worker start at the same moment, A declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds + the other, and without a dependency deadline both stay parked. It is made visible: an attempt parked longer than its + `readyTimeout`, whatever its blocker (audit A11, 2026-10-30), is reported once per parking episode as + `parked_long { object, waitingFor, state, since }` (`waitingFor` the blocker's label, `state` `loading` or `failed`); nothing fails or retries. A cycle is a plugin bug; **the guide's four + rules** for plugin authors and agents (host README): (1) a factory declares only what the object needs, never a type + that needs it back; (2) back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data + flows up through state and registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare + worker types in their factory. +- **Failed:** a broken part fails the whole object: a throwing factory, document hydration or `migrate`, a failed + open, a derive that throws on its first computation during activation. After the object is live, a throwing derive + fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it failed: + + ```ts + cause: { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // its own code (factory, derive, hydration, migrate), a cycle, authority + | { kind: "waiting"; error: ChordError } // not its fault (slice 7a): no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event + ``` + + Only an `open` may choose `broken` itself (`throw broken(error)`, e.g. invalid configuration). **Transient** causes + get a **backoff per identity** (1 s doubling to 60 s; a map holding no object, so a failed object is collected after + its last reference while its window runs; concurrent gets share one attempt; success resets). **Broken** ones are + retried only by a trigger. Triggers (for both): `ref.retry()`, a plugin reload (slice 7b, §21) and wake signals the factory + declares with `self.retryOn(...readables)` (slice 8; anywhere in the factory: the factory specifies, Chord installs + the watches before natives open, so a failed open still has them). Every trigger works like `ref.retry()`: the + backoff window is skipped, an attempt starts now, holders are told `loading`. `retryOn` sources must live independently of the + attempt (registry values); a reference's member is a type error. A facet cannot fail + after `load`: its setup only declares (a throwing setup leaves its plugin out of `load`, reported in `load`'s + `failed`; decision "Per plugin at startup and reload"; owner, 2026-10-26, 2026-10-30). + +### The factory principle + +```ts +host.serve(NoteStore, ({ session }, self) => { + const notes = self.state(NoteDoc); // section 9 + const index = self.state(Index); + const count = self.derive([index], ([i]) => i.ids.length); // section 12 + const watcher = self.open(async (context) => watch(dir, { signal: context.abortSignal }), (w) => w.close()); // section 13 + self.onLive(async (ctx) => { for (;;) { await sleep(HOUR, ctx); await cleanup(ctx); } }); // section 14 + const imports = self.job(ImportJob, { run, finish }); // section 15 + self.contribute(Http, { routes: { … } }); // section 17 + const info = self.get(SessionInfo, {}); // start data, section 18 + const catalog = self.get(Catalog, {}); // section 7 + return { index, count, add: …, startImport: … }; +}); +``` + +- Synchronous and declarative: no awaits, no side effects; async work happens in activation (opens), in calls, in + routines and jobs. +- Handles stay valid for their attempt's life (owner, 2026-10-26: documents and the line belong to the object; each + attempt's handles carry its seal token and definition version); nothing is "read at use" or re-fetched. +- On a reload the factory runs again: in staging against a recording `self` (declarations only), then for real; pure + objects swap in place (new methods and derives, same documents), native objects restart after their drain, + objects of a removed type end and their holders wait (section 21). + +### Pattern: shared store, per-principal views + +``` +NoteStore {S1} local shared store: the stored notes, once + ▲ self.get ▲ +Notes {S1} as owner Notes {S1} as read-only guest + variant "editor" variant "reader": derives a filtered view +``` + +Heavy or shared parts (a Durable watch, a prompt watcher, a sandbox, a store) live once in a `local` shared store; +per-principal `remote` views get it and expose what their principal may see. Authority is the view's; the store is +trusted and acts for the host. Views of the same variant read one value from the store (a redaction is computed once, +not per guest; section 12). Any plugin uses this pattern with the same primitives; base code is not special. + +**Worker-wide native owners:** the facet cannot `self.get` a type it serves, so an owner of a per-session native (an +inbox watcher, a sandbox) is a `local` object with empty params (host lifetime) that reads its session from the core's start +data (`self.get(SessionInfo, {})`, read in its opener). Paths are derived by the plugin from generic start data (the +data directory), never provided per plugin. +Such an object cannot `self.get` a session-scoped object in its factory (the session is known only after activation): +it gets one with `ctx.get` in a routine or method (R22). A `get` with params computed after activation is added only +if a real case needs it. + +**Natives that follow a relocation:** the native is a stable manager (`self.open` once); a routine on the session +location switches its internal resource (the watched directory) and bumps a generation, so late events from the old +location are dropped. + +--- + +## 7. References + +| Where | API | Released | +| --- | --- | --- | +| a factory | `self.get(Type, params)` | when the object ends | +| a call | `await ctx.get(Type, params, { readyTimeout? })` | when the handler settles | +| outside | `await host.get(Type, params, context, { readyTimeout? })` / `host.hold(Type, params, context, { readyTimeout? })` | `release()` | + +- **`host.get`** (and `ctx.get`, `host.registry`) resolves with a reference once its object is live and **hydrated** + (every state member of its variant has a value). It waits while the reference is `loading`, `failed { transient }` + or `failed { waiting }`, up to its `readyTimeout` (default 60 s, the caller's choice), then rejects with the last + cause; only `failed { broken }` rejects at once (owner, 2026-10-29, decision "Getters wait through `waiting`"). A + getter that gives up releases its reference. **`host.hold`** returns a reference at once that follows its object + until released (loading, live, failed and retried as its object is, loading again while its link is down) and never + rejects; only invalid params throw (checked at once, also before `load`: design audit 7, 2026-10-27). A type not + served where it goes is `loading` before the host's publication and `failed { waiting }` after it, before and after + `load` alike (§3; slice 7a). Its `variant` is `undefined` until its first binding (slice 3b). A UI holds and shows + `ref.state`; code that needs the object now gets it, with the patience it passes. There is no `state: false` option + (C8) and no `optional` flag. +- **Narrowing by variant:** `ref.variant` is the discriminant (C15: no `inVariant`/`expectVariant`); `RefIn` + narrows to some variants (strict unions, never weakened). +- **References are the only indirection** (owner, 2026-10-29; supersedes 6a's member source per object and 7b's + hand-over): a consumer's watches and derives over `ref.member` attach through the reference's member, which passes + the subscription straight through to the object's own document or derive (no extra hop, no copy: the commit's value + and ops by reference). When what the reference points at changes (a new object instance after a factory re-run: + reload, retry, restart; a new object after its type was removed and served again; another variant), the reference + moves its subscriptions to the new one and sends one frame, a diff from the last value (consumers' derives recompute, + a failed one heals; a member the new variant lacks fails `invalid/member`). While it points at nothing, its members + keep their last values (a pointer, no copy): while loading (a restart, a retry, a swap whose new instance is not live + yet) its watches get no call (`ref.state` says `loading`); while failed (a live failure, its version refused, its type + removed, a failed dependency) they get one `failed` call with the cause's error and stay open (decision "Failures are + watch calls"); a reference that never had a value reads `not_ready` while loading and its failure's error while failed, and its + watches wait for its first values. A member the new variant lacks is not a failed reference: it fails `invalid/member` + (reads throw it, watches get one `failed` call) until a later variant brings it back. Keyed watches and pins + re-pin on each new instance the same way (a key keeps its last value until the new instance has loaded it); a + dependent's factory pin is still loaded before the dependent goes live. Once hydrated, a reference has last values for every state member of its bound variant, read or not: when it + detaches it takes them at its version from the outgoing instance (through that instance's mapping for an older + version), keeping no node of it. After a `failed` call, the next good value arrives as `[["r", value]]` even when + unchanged; a watch opened while failed starts at the last value and gets the failure once. No source, mapping, + document handle or pin of an instance outlives it through a reference: a detached reference keeps only plain last + values and its failure; releasing it drops them and its bindings; stopping a keyed watch drops its pin. Derives over a failed reference member compute on its last value (a contribution built on it stays); only the + member's own watches get the `failed` call, and `ref.state` shows the failure (owner, 2026-10-29). A re-exposed member is + the re-exposing object's reference member, followed the same way. Over a link the same rule: the replica keeps its + last values readable while its hold is failed. Nothing outlives its object instance and nothing is handed between + objects. +- **References across a reload** (slice 7b §6): a reference is never ended by a code change. Its object swapped in + place or restarted: it follows the new instance (above; a hold over a link gets a fresh `bound`, §22). Its version no longer accepted, or a member of it refused (`contract_mismatch`): it leaves the object and waits + unbound (`failed { waiting }`) with its last values; the object and its other holders stay. Its type removed: it + waits unbound `not_served` with its last values; a publication that serves the name again binds it to the next + object (a new object: its in-memory documents start over from `initial()`, persisted ones load again). On a removal + the references take their last values at the seal: a commit admitted at that moment is stored and loaded by the next + object, but not in the last values they show while they wait (accepted: deferring it would need new state on the + waiting path, owner, 2026-10-29) (one frame, watches and derives heal, keyed pins re-pin, parked calls flush or reject, presence lists it + again). A reference of an older version a provider accepts reads its state members through the object instance's + shared mapping for its version (and keyed members per key), and its calls are mapped forwards and their results + back, or run the provider's replacement of a removed method (`checkArgs`). A reference knows its version from the + contract token it was made with; one made by outside code keeps the token its caller passed (Chord reloads no caller + code). +- **Members:** each state member of a reference is a watchable of the provider's member (a replica over a link); + watches taken through a reference belong to it and end when it is released. Methods are functions (`ref.add(args, + ctx)`, `ref.bump(ctx)`). +- **`ref.state`** is a `NativeWatchable` (slice 3b) of a tagged value; watches get each new state: + + ```ts + | { kind: "loading" } | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt: a transient cause's backoff; + // undefined for broken and waiting + | { kind: "ended"; error: ChordError } // released by its holder, or its object ended + ``` + +| Situation | Calls | Values | +| --- | --- | --- | +| link drop | park until reconnect | readable (`loading`) | +| object loading or restarting | park until live | readable (`loading`) | +| unbound (not served, another version) or parked on a failed dependency | reject at once with the cause | `failed { waiting }` (slice 7a) | +| object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | + +- A failed reference stays alive: while held, Chord retries a `transient` failure per the backoff and wakes a + `waiting` one at the event it waits for; `ref.retry()` tries at once (a Retry button; over a link the `retry + { hold, hops }` message, which also re-runs a refused hold's start: the host address, binding; design audit 2026-10-27; + on a parked object it passes on to the failed dependencies, §6); success goes `loading`, then `live`. A provider + tells a pending hold's consumer each failure (`failed`, with the rest of its backoff window for a transient one), + each new cause of an object parked on a failed dependency, and the next attempt (`loading`). +- **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the + reference's `readyTimeout` from the moment it was made (never renewed), then fail `unavailable/park_timeout`; they + are sent in order when it goes live (a new call never overtakes them) and rejected with the cause when it fails. + Otherwise a call ends only through its caller's signal; there is no per-call deadline. +- Calls already sent when a link drops fail `unavailable` and are not resent; calls running on an instance that ends + fail and are not resent. **Releasing a reference rejects the calls made through it** (`released`), locally and over + links (slice 3a). A consumer receiving `gone(h)` rejects its own pending calls on `h` (Q1). +- References survive link drops: one owner of a host's remote references holds them again on every new link of their + path; a re-hold may bind another variant (slice 3b). +- **Consumers** handle a failed call as they handle a disconnect (error shown, user retries); Chat's own submission + `requestId` stays an app concern. + +- **Optional dependencies** are registries owned by the dependent, not optional references: ports contributes its + forwards to `radius.forwards`; without ports the table is empty and radius works. + +- **Authority checks** live in the per-principal view's methods (and in `route.authorize` before a host starts); + there is no async activation guard. An object for an address that carries authority (`LoginFlow { id }`) may exist; + what a caller can see and do is decided by its view. + +--- + +## 8. `ctx` + +Every handler gets a `ctx` (a `Context`): `telemetry` (the current span, section 29), `abortSignal` (the caller +cancelled or the object ended; there is no per-call deadline), `principal` (the caller as its link stamped it; on a +local object's handler the host's principal, none on a client), `incarnation` (the calling host's, section 4), and +`get(Type, params, { readyTimeout? })` (references as the caller sees them, released when the handler settles; a +`get` after that throws `released`). Later: `ctx.trace(name, attributes, fn)` gives `fn` a child `ctx` under a child +span (slice 9, cut C11 to decide); in a job's `run` also `jobId` and `state` (slice 5). Pass `ctx` to everything you wait +on. Code outside objects gets a `ctx` from `host.root` (slice 8). Durable's `Context` is the same module +(`../context`); bridging is explicit. No code versions, no request ids. + +**Contexts are required** (owner, 2026-10-22): every function that takes a context requires it, as the parameter after +the required ones (context last); optional only on error-reporting paths. A getter that cannot take one (a lazy derive +read) uses its owner's. + +**Call-scoped natives** (a native that lives exactly as long as one call: a temporary SSH session for a directory +listing, a file handle for a download) are opened and closed inside the method with `try/finally`; there is no +`ctx.open`: + +```ts +list: async ({ path }, ctx) => { + const session = await ssh.open(config, ctx.abortSignal); + try { return await session.readDir(path, ctx.abortSignal); } + finally { session.close(); } +}, +``` + +--- + +## 9. Documents + +### Definition and handles + +```ts +const Settings = defineState("settings", { + schema: SettingsSchema, + version: 1, + initial: () => ({ theme: "dark" }), + persist: true, // default: in memory + checkpointWhen: () => true, // required for persisted documents; small: a base per write +}); +const Draft = defineState("draft", { + schema: DraftSchema, + version: 2, + migrate: (old, from) => ({ body: old.text }), // pure + initial: () => ({ body: "" }), + keyed: true, // many per object + persist: true, + checkpointWhen: (value, ops, info) => info.deltasSinceBase > 50, +}); + +host.serve(DraftStore, (_, self) => { // local, empty params: may persist (P3) + const settings = self.state(Settings); // THE settings document + const drafts = self.state(Draft); // all Draft documents of this object + return { + settings, // a singleton handle is an exposable Watchable + get: ({ conversation }, ctx) => drafts.get(conversation, ctx), + save: ({ conversation, body }, ctx) => + self.tx(async (tx) => { + const d = await tx.draft(drafts, conversation); + d.body = body; + return d.body.length; + }, ctx), + drop: ({ conversation }, ctx) => self.tx(async (tx) => tx.retire(drafts, conversation), ctx), + }; +}); +``` + +- **Ownership:** a document belongs to exactly one object identity, and so does its line (owner, 2026-10-26: across the + object's attempts, retries, failures and native restarts; an attempt keeps only its natives, routines, derives and + seal token) (persisted ones across restarts; in-memory ones for + the object's life, across reloads and native restarts). Facets have no documents (owner, 2026-10-26: C16). + Others see it only through exposed members. References to other objects are stored as data (addresses). +- **Singleton** (no `keyed`): loaded at activation, kept in memory while the object lives; `handle.value` is + synchronous; the handle is an exposable `Watchable`. +- **Keyed:** addressed by a short key local to the object, loaded on demand; `await handle.get(key, ctx)` reads (creates + nothing); `for await (const key of handle.keys(ctx))` pages keys (both take a context, owner, 2026-10-26) (indexed per owner and definition, also in memory). No + live key list: a collection shown live is a singleton (an object holding a list or map); large collections are + methods returning pages. +- **Values are immutable by convention** (Chord Delta's rule, as in Durable): `handle.value` and published values + are shared, never mutated by their readers, and never copied or frozen; a draft belongs only to its transaction. + +### Writes and transactions + +- **Only `tx.draft`** (as Durable's `tx.doc`): `await tx.draft(handle, key?)` inside `self.tx(fn, ctx)` + returns a draft of the current document, or a draft over `initial()` if it is absent. Reading is drafting (Delta's + overlay: no copy). An unchanged draft (empty ops) commits nothing, so an absent document is never stored by being + read. `tx.retire(handle, key?)` removes a document. `keyed` makes the key mandatory at the type level. There is no + `tx.get` for documents (tables have one, section 10). +- Drafts record ops (assignment, `push`, `Object.assign`); the same document returns the same draft within one + transaction. **Drafts are revoked when the transaction settles (R12, as in Durable):** a transaction writes and + returns at most plain values (an id, a count), never anything taken from a draft. To return data, read after the + commit from the document (`handle.value`, `handle.get`), which then holds the committed revision; a later commit + may slip in between, and the reader gets that newer value. +- **No existence check inside a transaction (R13, as in Durable):** `tx.retire` of an absent document is a no-op (it + never creates); `tx.draft(handle, key)` creates a missing document when the draft changes; + whether a keyed document exists is read outside the transaction (`handle.get(key, ctx)` gives `undefined` for an absent + one). **A singleton always has a value (Q11):** `handle.value` is the written value or `initial()`, never + `undefined`, so an exposed singleton is typed `T`; singleton existence has no API (unlike Durable's `snapshot()`). + + Patterns: + + ```ts + // Return data: write in the transaction, read after the commit. + await self.tx(async (tx) => { (await tx.draft(notes)).items.push(note); }, ctx); + return notes.value.items; // the committed revision (or a newer one) + + // Remove if present: retire is a no-op for an absent document. + await self.tx((tx) => tx.retire(drafts, conversation), ctx); + + // Create if missing: the draft creates it (only if it changes). + await self.tx(async (tx) => { + (await tx.draft(drafts, conversation)).text = text; + }, ctx); + + // Branch on existence (keyed documents): read first, outside the transaction. + if ((await drafts.get(conversation, ctx)) === undefined) { … } + ``` +- Document roots are `JsonObject` (as in Durable; lists live in a field); documents hold JSON only (bytes are a value + and wire type). A singleton that does not exist yet reads `initial()` through its handle; retiring resets it to + `initial()`. Handles may be used only in transactions of their own object. +- **Commit sequence** (Durable's): wait for the object's line (per object, never across objects; reads outside a + transaction do not take it exclusively); run the callback; an error or an abort before the storage write discards + everything; prepare every draft (Delta copies and checks only the placed values); write persisted documents in one + storage batch through the store's commit step, `await store.commit(writes, adopt, ctx)` without the caller's abort + signal (deltas, or a base when `checkpointWhen` says so; the line held until the step returned, as Durable's + `#runCommit`, owner, 2026-10-26); adopt by pointer swap and mark dependents in the step, persisted and in-memory + documents and staged changes in one `host.batch.commit` (errors as Durable's Storage, owner, 2026-10-27: a step that + fails before `COMMIT` rolls back and rejects that commit only, nothing adopted, the store goes on; plugin mistakes are + refused when a write is staged; a seal-fenced commit is rejected `released` before `BEGIN`); then the owner's synchronous flush recomputes dirty derives, sends one message + per link and queues watch callbacks (W1); drafts then throw if touched. In-memory-only commits take no store seq and never + enter the commit queue (W12). No + revalidation of the whole document, no copy, no freeze. +- **Nothing leaves a transaction before it commits (S2):** ids and data reach other owners only after the commit; + reference what is committed (a reference to a later commit dangles on every replica that stops in between). +- **Durable's rule:** inside the callback only `await tx.*`: no other objects, models, processes or network, no second + `self.tx`, no reads past `tx` (otherwise a hang, not an error). Fetch first; decide and write inside. +- A transaction covers only its own object's documents and tables: one line, no transactions across owners (facet + documents were removed, owner, 2026-10-26; shared plugin state is a `local` store object's, reached by methods). +- **Shorthands:** `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (one document, loaded before the synchronous `fn`), + `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`); several steps atomically: `self.tx`. + +### Versions + +- A shape or meaning change raises `version` with a pure `migrate(old, from)`. A document migrates on access by the + new definition (in memory too); the migrated base is persisted with the next commit. +- **Schema fingerprints instead of value validation:** `defineState` computes the fingerprint of its structural schema + once (descriptions and titles ignored) and keeps it on the token; every document record stores the fingerprint it was + written with (at creation, and with the new base after a `migrate`). On access Chord compares the token's fingerprint + with the record's: the same version with a different fingerprint fails the object `broken` ("schema changed without + a version bump"). Always on, O(1). Every structural schema change needs a version bump (an additive change gets a + trivial `migrate`). Values are not walked against the schema in production, with two exceptions (R8): + - the result of `migrate` is validated against the new schema once, when it is produced (per document per version + change, off the commit path); a failure refuses the access, keeps the old data and fails the object (slice 6); + - `initial()` is checked once per definition, at its first value (D3, owner, 2026-10-24: `initial()` takes no input, + so a wrong shape is wrong at its first call; checking every call would cost one validation per created document + and catch nothing more). A failing singleton `initial()` fails its object's activation (`broken`); a failing + keyed one fails only the transaction that drafts the absent document, naming the definition (slice 1). +- Old code whose definition is older than the document throws on access. Without `migrate` the access is refused and + the data kept. A changed meaning without a bump cannot be detected. +- Singleton documents migrate during activation and after reload (hydration counts as access). A hydration or + migration failure keeps stored data and fails the whole owning object, not only the exposed member. Keyed + documents migrate on their next access; a migration failure there also fails their owner. A migrated base is written + back by the next commit that changes the document (owner, 2026-10-27); until then a load migrates again (once per + object, since documents stay on the object). + +### Lifetime, persistence + +- **No tags (owner, 2026-10-21):** deleting a session deletes its store and directory; instance data on the server that + refers to a session is a table with a `session` column, cleaned up by its owner (`deleteWhere`, reconciling from the + catalog). +- **In-memory documents are instance state (P2, owner, 2026-10-21):** attached to the live object, with no storage, no + address, no seq and never in the outbox; their commit is prepare and adopt (inside the store's commit step when the + transaction also writes persisted data). +- **Lifetime:** in-memory documents end with their object (last reference released, linger expired, a params change on + reload) and survive reloads and native restarts of a live object; long-lived objects creating many keyed in-memory documents retire + them themselves. Persisted documents stay until retired. +- **Persisted data only in `local` objects with empty params (P3, owner, 2026-10-21):** persisted documents and tables + live in `local` objects with empty params (one per host and type); their address is (object type, name, key), never a + host instance such as the session id (W7; no host kind either since slice 6b R2: a store belongs to one host), + so a params change never orphans data. Per-entity data is keyed documents or table rows there, with an app-chosen + key; objects with params keep only in-memory state and read persisted data by key. Renaming a persisted store's type or + definition, or changing how keys are formed, is an explicit one-off migration in the store. + Why not elsewhere: a `remote` object's identity contains the principal key (provenance, grant revision), so its + persisted documents would scatter across access paths and be orphaned after a reconnect; a shared storage key + (`storeKey`) would put two objects with two lines on one document, so it does not exist; params would orphan data + on a params change. Enforced at the type level (`self.state(…, persist)` and `self.table` only in factories of + `local` types with empty params) and as a declaration error at runtime ("persist needs a stable identity: put the + documents into a local shared store"). +- **Durable plugin state:** whatever must survive a host teardown (crash, restart, a DO eviction) is a persisted + document; everything else is rebuilt on activation (in-memory documents, natives, derives, routines, presence, + references re-held by clients; job runs end `lost` unless their records persist). + +### Rules across owners + +- **No plugin reads or writes another plugin's documents** (Chord or Durable). Data crosses plugins only through + state and methods. Data core enforces lives in a format core owns and is handed to core through its API when the data is + created (`createConversation` with `init`), never written into core's documents directly. +- **Invariants across owners** are enforced by the owner of the dependent data, which reconciles from the other + owner's state with an idempotent routine. No cross-owner transactions, no "notify the other owner" calls that can + be lost: + + ```ts + // ports' ForwardStore: "a forward of an archived session is closed" + const catalog = self.get(Catalog, {}); + self.on(catalog.sessions, async ({ value: sessions }, ctx) => { + for (const f of forwards.value.list) if (sessions[f.session]?.archived) await close(f.id, ctx); // idempotent + }); + ``` + + After a crash the routine runs again with the current value; sessions does not know ports exists. + +--- + +## 10. Store and storage + +> **Superseded (owner, 2026-10-21):** persisted data lives in one transactional store per session, designed in +> `packages/chord/docs/next/store/book.md` and decided in the log ("The store", S1-S17, W1-W14, P2-P4, "Flat columns", +> "Query runner", "Index definitions are recorded", "Blobs and derived data"). In short: backends SQLite (Node, Durable +> Objects, sqlite-wasm in the browser; tests on SQLite `:memory:`, no hand-written memory backend, owner, 2026-10-27); real tables owned by objects with empty params (P3), flat +> columns (scalars, every `str()` as plain UTF-8 `TEXT` in code point order with lone surrogates replaced and NUL refused (owner, 2026-10-26, superseding W8), `json(...)` with its own version and on-access migration), a small +> query language, writes `put`/`update`/`delete`/`updateWhere`/`deleteWhere`; one documents engine for Durable +> (`StoreStorage`) and Chord; one commit queue per store that writes, adopts and marks (S1, W1), with `Storage.commit( +> writes, adopt, context)` in Durable; an outbox of ordered operations in the same transaction (no compaction at first); +> eager migrations by scan with row `v` for eager steps only. In-memory documents are instance state with no storage +> (P2). The text below describes the per-object lines, Delta trackers and commit sequence, which stay; its storage +> interface is replaced by the store. + +- **What stays from Durable's store design** (`durable/src/session/transaction.ts`, `session.ts`): one Delta tracker + per document held by its object, prepare and adopt by pointer swap, ops per commit, lines per object, migrate on + access, `checkpointWhen` (required for persisted documents, R14), retire, and the documents engine's invariants (create before change, no + change after retire, one live incarnation per address, `deltasSinceBase`). +- **Where persisted documents live:** rows of the store's documents engine (S13) with address `(namespace: "chord", + owner: "", kind: , key)` (key `""` for a singleton), never a host instance (W7); ids from the store's + counter (S10). Chord ships no storage of its own: no `DocumentStorage` interface, no Chord SQLite or memory document + storage and no Chord storage conformance suite (the store's conformance suite covers the engine); the frozen `next/` + storages are not ported. The store is passed to the session's host by base code (owner, 2026-10-26): + + ```ts + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, "session.sqlite"))), ctx); + const worker = createHost

({ address: Session({ id }), principal, store, onError }); // tests: SQLite :memory: + … + await worker.close(ctx); + await store.close(ctx); // the app owns the store; in 6b the Durable Session shares it + ``` + + `openStore` runs the store's own migrations, checks the file is UTF-8 and reads `lastSeq`, the definitions and the + id counters once; `sqliteBackend(db)` takes any `SqliteDatabase` (Node, Durable Objects, sqlite-wasm later); the + backend interface is internal; `importStore(backend, archive, ctx)` takes the same backends. No `openNodeStore`. +- **Loading at activation (owner, 2026-10-26):** persisted singletons load right after the factory, sequentially, + all singletons of the object with plain reads (owner, 2026-10-27: the object holds its line, so no read transaction + is needed), once per object, before its references are bound: find, load (base plus + deltas), fingerprint check, `migrate` with its result validated once (R8). A storage error is `storage_failed`; a + fingerprint mismatch, a throwing `migrate` or an invalid migration result fails the object `broken`. +- **Statements per commit (owner, 2026-10-26):** the store keeps `lastSeq` (the seq of its last commit) in memory (read at open, + persisted with each commit, S9) and folds the documents engine's checks into the writes (unique keys, a partial + unique index on the live address, conditional `UPDATE … WHERE id = ? AND version = ?`): a delta commit is about three + statements (revision, the record's counted conditional `UPDATE`, `lastSeq`), plus one `INSERT` with an outbox. Durable's ids in the store (slice 6b D3): one id allocator per store + for every id (Durable's `mintId` and Chord's document ids; it starts at 2, saturates at `Number.MAX_SAFE_INTEGER`, each + commit persists it, outbox commits and archives carry it as `nextId`); `StoreStorage`'s `record_ids` table only checks + an id's record type. Every table and document namespace in the store is claimed once (slice 6b "owner names and + claims"): a table by its SQL name `":

"`, documents by `(owner, namespace)`; an object claims under its + type, `StoreStorage.open(store, owner, ctx)` under `owner` (two harnesses on one store under two names are fine). +- **Tables in a transaction (owner, 2026-10-26):** the table handle first, keys as objects with exactly the key + columns, reads async, writes synchronous and buffered like a draft's ops (checked and staged, no I/O; written in the + commit step or dropped on failure); no write shorthands on the handle: + + ```ts + self.tx(async (tx) => { + const old = await tx.get(edits, { id }); // overlays own put/update/delete (W10 after bulk) + const { items, next } = await tx.page(edits, q, 50, { after }); // committed rows, stable for the transaction + tx.put(edits, row); + tx.update(edits, { id }, { at: now }); + tx.delete(edits, { id }); + tx.updateWhere(edits, (e) => prefix(e.path, from), (e) => ({ path: replacePrefix(e.path, from, to) })); + tx.deleteWhere(edits, (e) => lt(e.at, cutoff)); + }, ctx); + await edits.get({ id }, ctx); // outside: committed state, ctx last + ``` +- **The outbox (owner, 2026-10-26):** optional, `openStore(backend, ctx, { outbox: true })` (an omitted option keeps the + persisted setting; switching needs an explicit `true` or `false`), persisted in the store's + metadata (turning it on starts sync from an archive at the current `lastSeq`); a table in the store's backend, + written in the commit step's transaction (atomic for one `INSERT`); exposed as `store.outbox.read(afterSeq, limit, + ctx)` -> `{ seq, writes }[]` and `store.outbox.trim(throughSeq, ctx)`, no pluggable encoding; versioned by the store's + schema migrations (its layout) and a `format` per outbox commit (the write vocabulary, the archive's `format`; older + formats stepped forward on read, newer ones refused). +- **Writes, commits, batches, archives (owner, 2026-10-26):** a *write* (`StoreWrite`, tagged by `kind`) is one change + of the store (`put`, `update`, `updateWhere`, `table`, `index`, `document.create`, …); a *commit* the writes of one + seq in the outbox; a *batch* the commits `from..to` that sync uploads and a replica applies; an *archive* a manifest, + the writes from 0 to `lastSeq` and an `end`. `lastSeq` is the seq of the store's last commit. +- **Export and import (owner, 2026-10-26):** `store.export(ctx)` yields the archive's parts, the state after its + `lastSeq` as writes (manifest, `table`, `index`, `put`, `document.create`, `end`), holding the commit queue + until the iteration ends or `ctx` aborts; `importStore(backend, parts, ctx)` applies them into an empty backend in + one transaction with the replica's applier. Lossless (owner, 2026-10-26): every `put` write carries its versions (`$v`, + each `json(...)` column's) in the outbox and in archives, a `table` write its state, a `latest` document its current + value; the applier never infers a version. The encoding is pluggable; plain JSONL ships first. Sync is not export: + it ships outbox batches as commits happen. +- **`json(schema, { version, migrate? })` columns (owner, 2026-10-26):** schema and version required, `migrate` + optional and pure, on access (W11), its result checked once (R8); without it (or failing) the read is refused and + the owner fails `broken`; each column's version and schema fingerprint are recorded in the table definition and + compared at activation (same version, other fingerprint: `broken`, D9); a read never writes back. +- **Table versions: `evolveTable` (2026-10-27, the second spec review's majority):** a table's history is a base + (`defineTable`) and one `evolveTable(prev, (t) => t.rename(…).add(…).rows(…).drop(…).index(…))` per version, each + materialized and fingerprinted at declaration, as contracts' `evolve`; the store replays the recorded version's + successors in order (DDL and `migrating`, the data step, `ready`); key columns are immutable in `set`. +- **Eager migrations and addresses (owner, 2026-10-26):** the activation attempt's `readyTimeout` deadline does not + run while an eager migration step runs (getters still time out by their own). Store names carry no host kind (slice + 6b R2, owner, 2026-10-27): tables are `":
"`, document owners `""`. +- **Eager `rows` migrations** commit in chunks of 100 rows (owner, 2026-10-26; a fixed constant: about 1 ms of commit + step per chunk, the total time is the same for larger chunks, and every other owner's commit waits for the step). + Between chunks the migration yields one macrotask without `setTimeout`'s 1 ms clamp: `setImmediate` where the + runtime has it, otherwise a `MessageChannel` message (2026-10-27, slice 6a build reviews). +- **Eviction as in Durable,** for keyed persisted documents only (persisted singletons and `at(key)` documents stay + in memory while their object lives: `handle.value` is a synchronous read; owner, 2026-10-26): one not read, written or watched for the retention + period (default 10 minutes) leaves memory and reloads on the next access. **Store errors (owner, 2026-10-27; supersedes in part "Store errors: Durable's rule"):** the store has Durable's + `Storage` semantics, its owners react as Durable's Session does. A step that fails before `COMMIT` rolls back: that + commit rejects, nothing is adopted, the store goes on. The store stops (`storage_failed`: later calls reject + `unavailable/storage_failed`, the host closes, `onError` receives `{ kind: "storage_failed" }`, the app decides, + section 3) only when the outcome is unknown (`COMMIT` or `ROLLBACK` failed) or `adopt` threw, and on a backend + error of a read whose `ctx` was not aborted. A Chord object whose commit the step refused (memory and disk disagree: + the counted conditional `UPDATE`, the live-address index) fails live (`object_failed`, `broken`); an `int` result out + of range (`CHECK`) fails only that transaction (`invalid/args`). Durable's Session fails on any commit error + (`GuardedStorage`), as today. + +--- + +## 11. Kinds of state + +| Kind | For | API | +| --- | --- | --- | +| document | data the object owns | `self.state`, `self.tx` | +| derive | computed values | `self.derive(sources, fn)` | +| Durable's views | the conversation view, task graph and Durable documents | `Watchable`s through Durable's watches; feed derives and state members | +| paged method | large or queryable collections | a method returning pages (chat history) | +| table | a plugin's own records with queries (owner, 2026-10-21) | `self.table`, the store's query DSL | +| plugin database | derived data only: full-text search, caches (rebuilt, never synced) | a native opened by its owner | + +App state lives in Chord documents; only transcript-bound data (entries, documents that fork and rewind with a +conversation) lives in Durable. + +### Watchables (owner, 2026-10-21; log "Watchable and one watch API", W1-W4) + +There is no `replicatedState` in v2. Everything observable is one of two kinds, sharing one lifecycle: + +```ts +interface Watchable { // JSON only (no bytes); documents, derives, reference members, constant(v), Durable's views + readonly value: T; + watch(ctx: Context): Promise>; // listener (call, ctx), call: WatchCall (below) +} +interface NativeWatchable { // self.writable(initial) with set(), composed registry values + readonly value: T; + watch(ctx: Context): Promise>; // listener (call, ctx), call: NativeWatchCall; a change is a new value (!==) +} +type WatchCall = // owner, 2026-10-27: a recoverable failure is a call, the watch stays open + | { kind: "value"; value: T; ops: Op[]; retired: boolean } + | { kind: "failed"; error: unknown }; +type NativeWatchCall = { kind: "value"; value: T } | { kind: "failed"; error: unknown }; +// const w = await notes.watch(ctx); send(w.value); +// w.start(async (call, ctx) => { if (call.kind === "failed") return showError(call.error); render(call.value); }); +// … await w.stop(); +``` + +- **Failures are calls, watches end only for good (owner, 2026-10-27):** a source that fails and can recover (a + derive whose compute throws, a replica member a hold lists as failed, a member a new variant lacks, a registry whose + `compose` throws, slice 8) gives its watches one `{ kind: "failed", error }` call and keeps them open; its next good + value arrives as `[["r", value]]` (folded with the failure for a busy listener). A failed derive recomputes on every + later change of its sources and on a re-bind (self-healing). A watch ends only when nothing can come back: its owner + ended (`ended`), `stop()` or a cancelled context, a throwing listener (`listener_error`), `storage_failed`. `self.on` + handlers are not called for failures: a routine waits for the next good value (the failure is reported where it + happens: the derive's owner reports `derive_failed { object, member?, error }` once per failure episode, the first + failure after a value or after creation, not for a derive failing only because a source failed). `ref.state` stays + a separate watchable of the object's reachability (`loading | live | failed | ended`), with the same words. A + replica member that vanished with a variant is kept and comes back with a later binding; while failed it accepts + only `[["r", value]]`. +- **Watch handles are Durable's, moved into Chord** (Durable imports them): `watch()` atomically takes the current + value and buffers later commits; `start(listener)` installs the one listener; commits are delivered in order, async + listeners awaited; a listener still busy with an earlier frame gets the commits made meanwhile as one call, from + the **pending queue** (owner, 2026-10-23): ops stay raw and are folded through a leaf index (a map from path to + the last pending append or primitive set; a new one on that path merges into it; any structural op clears the + map; `r` drops everything before it); past 10,000 pending ops only "replace with the current value" is kept (no + size, text or time caps; no folding of list appends or trims; ops come only from Chord's tracker and + `diffRevisions`, so there are no defences against hand-written ops). `PendingOps` is Chord's shared Delta code. The + same queue holds a link hold's ops while its window is full (section 3). Natives: every value in order, past 100 pending values one call with the newest (W4). Durable's + event stream keeps its own queue (events are not state); `stop()` is idempotent, drops pending frames, never aborts or awaits a running call; `stop()` and + `closed` return the end, tagged by `kind` (owner, 2026-10-27): `stopped`, `aborted` (its context aborted), `ended`, + `listener_error { error }`, `failed { error }` (a final failure; a recoverable one is a call). Durable's synchronous + observers take the same shapes: `advance(call, ctx)`, `end(end)`. + **Retirement (owner, 2026-10-26, option A; replaces W2a's `[["r", null]]` frame and the end `retired`):** one + mechanism for Chord and Durable documents, singletons and keyed alike. A watch follows its document's address; a + retirement is one last call with a `retired` flag (a Chord document: `initial()` with `[["r", initial()]]`, which a + wire replica applies; a Durable document: the last good value, no ops); the watch goes on, and a later + write delivers the new incarnation as `[["r", value]]`. A watch ends `ended` with its owner (the object of a Chord + document; for Durable the task of a task document, right after its retirement call, or the Session). Listeners take + one object before the context (owner, 2026-10-26; tagged since 2026-10-27, above): `(call, ctx)`, a `WatchCall` or + `NativeWatchCall`. + A retirement call carries the address's value after the retirement: `initial()` for a Chord document, the last value + (no ops) for a Durable one, which has none while absent. Watching an absent address gives `undefined`. On the wire a + `state` entry lists retired members like failed ones. A per-key source in a factory, `handle.at(key)`, is a singleton per key: loaded at activation, `initial()` while absent, usable as a state member, derive source and `on` source (owner, 2026-10-26). A + stopped watch releases every reference it held (W3). +- **Only `Watchable` can be a contract state member or go on the wire**, by type and by a kind check when the + object is served (Q14); a `NativeWatchable` never can, even holding JSON. Mutating a native value in place notifies + nobody. +- **Publication (W1):** the commit step writes, adopts (pointer swaps, Durable's touched views dispatched by id) and + marks (dirty derives; ops appended by reference to each touched hold's batch and each watch's queue). Right after + the step, synchronously, the owner's flush recomputes the dirty derives someone holds or watches (dependency order), + encodes one entry per touched hold into one `state` message per link and origin, and queues watch callbacks. A + throwing derive fails only its member; reentrant `self.tx`/`watch`/`release`/`get`/`hold`/`load`/`close` throw + `invalid/reentrant`; a delivery or flush that throws is reported (`delivery_failed`) and the others still run. The + same boundary runs without a store seq for in-memory commits, `writable.set`, registry changes and inbound wire + batches (a `bound` or `state` message is applied as one batch before listeners run). There is **one batch per host**, + passed to its sources (no module-level singleton); a dirty derive nobody holds or watches recomputes lazily on its + next read. +- **Consumers:** derives and the wire use the internal flush; app code uses watches; `self.on` is a watch owned by + the object (first call with the current value, then every commit; Chord awaits a running call on reload); clients + get ordered ops over the link, folded by the provider's pending queue when they fall behind (section 3); a re-hold + gets a fresh snapshot. + +--- + +## 12. Derives + +`self.derive(sources, (values) => value)`: a synchronous pure function, recomputed when a source changes and +diffed into ops for the wire. Sources are `Watchable`s and `NativeWatchable`s (documents, other derives, reference +members, Durable's views, registry values); the result is always JSON (state is JSON only; bytes only in method arguments and results). A derive is marked dirty in the +commit step and recomputed in the owner's flush right after it, so its change travels in the same message as its +source (W1); a dirty derive nobody holds recomputes on its next read. A throwing derive fails only its member +(`broken`). There is no `previous` (W14): `mapShared` keeps unchanged elements identical. + +- **Built on what exists:** derives sit on Chord's internal publication primitive (not today's `cell.ts` or + `replicatedState`); the diff is + Delta's `diffRevisions` called directly (identity short-circuit: unchanged shared subtrees cost one `===`; bytes are + atomic leaves compared by identity; equality never by stringify). +- **Container width (R24):** structural sharing is O(change) along a path, not across a container's width: changing + one item of a flat array copies the array's slots, and `mapShared` visits every element (one identity comparison + each). Large collections are keyed documents or bounded windows; a benchmark of long arrays (allocations, element + visits per change) sets the documented limit. +- **The footgun:** a derive that builds new objects for unchanged data makes every commit cost O(size of the value), + per view, because the diff must compare contents. Unchanged parts must be the same objects as last time. +- **`mapShared(f)`** (from `@earendil-works/chord/next` state; it moves into Delta when v2 goes upstream): an identity cache per source element: `f` runs only for new or + changed source elements, unchanged ones return their previous result object. With it a list derive costs pointer + comparisons, not content comparisons: + + ```ts + const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); // a mapper over the whole array + self.derive([notes], ([n]) => ({ list: strip(n.list) })); + // one appended note: one new object, 2,000 identical references, diffRevisions emits one insert + ``` + +- **Cookbook** (documentation with examples per case): select a field, look up by key, small aggregates (just compute; + the value is small); filter or map a list (`mapShared`); redaction (preferably separate documents per visibility, + otherwise `mapShared`; computed once in the shared store, not per view); joins across two sources (`mapShared` plus + the looked-up source's revision); a window over a large list (take a section of the source elements without + rebuilding them, cache per-element measurements by identity, keep the window's start stable so changes arrive as + appends). Large combined views from several sources re-path the sources' ops (as Durable's `ConversationView`); + application code never interprets ops. + +--- + +## 13. Natives + +```ts +const watcher = self.open(async (context) => watchDir(path, { signal: context.abortSignal }), (w) => w.close()); +self.onLive(async (ctx) => { for await (const f of watcher.value.files(ctx.abortSignal)) await importFile(f, ctx); }); +``` + +- `self.open(open, close, { closeTimeout? })` (slice 2): `open(context)` gets a context aborted when the attempt is + abandoned (a timeout, a cycle, its end); `close(value, reason, context)` gets one aborted at the close timeout, after + which a possible leak is reported (`close_timeout`) and the rest continues. It returns a handle with `.value`, not a + proxy: reading it before its open resolved throws `not_ready`, after its close started `released`. A value that + arrives after its attempt was abandoned is closed at once. + Natives open during activation (the object goes live after all opens) and close in reverse order when it ends. + The opened value never changes during that attempt: no `self.resource`, versions, `replace`, `deps` or stale + handles. To change a live native, the object restarts. A partly failed activation closes already-open natives + with reason `end`; a later attempt opens them all anew. `retryOn` may appear anywhere in the synchronous factory. + Type names: `Native` (the handle), `NativeCloseReason`. +- **Order:** natives open one after another in declaration order (a later opener may use an earlier handle's + `.value`) and close in reverse. A throwing `close` does not stop the others: on `host.close()` all errors are + collected and it rejects with them; an object ending in the background reports them to `onError`. +- Behaviour around a native goes into routines; listeners on natives outside routines are forbidden. +- The facet may not open natives. +- What an old native does after its owner restarts is its owner's `close` (a model runtime keeps working until + collected; a closed SSH connection fails its in-flight `exec`). +- An open that fails makes the object `failed` (`transient` with backoff, or `broken` if it says so). + +- **`close` gets a reason:** `reason: { kind: "reload" } | { kind: "close" } | + { kind: "end" }` (the plugin is reloaded and the object is native / the host shuts down / the last reference ended or + the type was removed). Chord aborts only the object's own calls and jobs; other work using the native (Durable tool + executions on a code mode sandbox) is the owner's: `close` may drain instead of tearing down, e.g. seal the sandbox + (no new scripts) and end it when its running scripts finish or hit their deadline. +- **Restart stays sequential (R16):** close the old native, then open the new one. Two patterns for natives whose + resource has work in flight: + - **Drain in the background:** `close` detaches the old resource, returns at once, and disposes of it when its work + has finished or after `closeTimeout` (30 s at most, Q7: a longer script is aborted); Chord then opens the new one + immediately. Code mode's sandbox does + this; its tool entry looks up the current sandbox at `execute()`. + - **Swap through a registry:** the resource is a registry contribution; readers always find the current one, and a + reader that must wait for one uses a cancellable "wait until present" with the park deadline. + +--- + +## 14. Routines + +Two kinds, declared in the factory (owner, 2026-10-24; C4 decided; slice 5): + +```ts +self.on(catalog.sessions, async ({ value: sessions }, ctx) => { // reacts to a value: first the current one, then every change + for (const f of forwards.value.list) if (sessions[f.session]?.archived) await close(f.id, ctx); // idempotent +}); +self.onLive(async (ctx) => { // called once when the object goes live + for await (const file of watcher.value.files(ctx.abortSignal)) await importFile(file, ctx); +}); +self.onLive(async (ctx) => { // periodic: a loop with sleep + for (;;) { await sleep(HOUR, ctx); await dropOld(ctx); } +}); +self.onLive(async (ctx) => { // set up, wait for the end, clean up + const timer = setInterval(tick, 1000); + try { await untilAborted(ctx); } finally { clearInterval(timer); } +}); +``` + +- **Lifecycle (both kinds):** declared in objects, never in facets (owner, 2026-10-26: everything is done through + objects); they start when the object goes live (in the same turn) + and stop when it ends (their `ctx` is aborted, then + the drain waits for running calls, bounded by `closeTimeout`). Each call gets its own `ctx` (the object's principal + for a remote type, the host's for a local one; the host's incarnation; `ctx.get` released when the call settles). +- **On reload** the old routines' `ctx` is aborted; the new routines start once all old routine calls of the object + have finished (bounded; a stuck call is reported and the new routines start anyway): a handover per object, so + routines need no names. Example it prevents: an old handler midway through `restart(W1)` and a new one seeing W1 + still pending and restarting it again. The bound is `closeTimeout` (owner, 2026-10-24); calls do not wait for the + handover, only the new routines do (slice 7b). A stuck old call past the handover is reported `drain_timeout` and + the whole old attempt is sealed then (audit A12: one seal token per attempt; its method calls had the same bound). +- **`on(source, fn)`:** any watchable (a document, a derive, a reference member, a registry value, `ref.state`). The + first call gets the current value (`[["r", value]]`, W2b), then one call per change; calls never overlap (S8); + changes made during a call arrive folded in the next (newest value, folded ops); a running call is not aborted by a + change (native sources too: a busy call is followed by one call with the newest value). A throwing call is + reported and called again with the current value after a backoff (1 s doubling to 60 s; + a change during the backoff calls at once; success resets it). The source ending (a released reference, its object's + end) ends the routine without a report; a retired document does not end it (the call has `retired: true`). +- **`onLive(fn)`:** called once when the object goes live. Returning ends it until the next reload or activation; + throwing is reported and restarts it after the same backoff (reset after a run of at least a minute); an abort is + no error. +- **Idempotent reconcilers:** a routine reconciles from the current value, writes no "in progress" flags and starts a + job for long work. Chord keeps no state for routines across reloads or restarts; what must survive is a document of + the object (e.g. a `lastCleanup` time, which also survives a crash). +- **Helpers:** `sleep(ms, ctx)` (rejects when `ctx` is aborted, leaves no timer or listener) and `untilAborted(ctx)`. + Routines and handlers wait only through calls that take `ctx`; otherwise Chord cannot stop them, and the handover + and the drain report them stuck. +- **Errors:** `onError` gets `routine_failed { object, routine, error }`, `routine` a `Position` (`{ kind: "on" | + "onLive", position }`; natives use the same type); never uncaught. +- **Keep-alive:** routines do not count in `host.activity` (an `onLive` runs forever). +- No bare timers or callbacks that touch state outside routines, jobs or calls. Gone (C4): `every` (an `onLive` with + `sleep`), `custom` and its `run(fn)` (an `onLive` with `try`/`finally`; callbacks that must run one at a time go + through a small queue or the source's async iterator), names, a due time kept across reloads. + +--- + +## 15. Jobs + +Owned work a call starts that outlives the call (a shell command, a session clone, a peer send, a login's work), in +Durable's sandwich: **intent** (a transaction), **run** (any effect), **finish** (a transaction on the current code). +Decided with the owner on 2026-10-24 (decision log "Routines and jobs open points", "Jobs keep accepts", "Job crash +handling"; C3 settled this way); slice 5 builds jobs in memory, slice 6 persistence, slice 7 their reload rules. + +```ts +const ShellState = defineState("chat.shell.state", { + schema: obj({ output: str() }), version: 1, initial: () => ({ output: "" }), +}); +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str(), author: Author }), + result: obj({ exit: int(), output: str() }), + state: ShellState, // optional: one document per run + concurrency: 1, // optional: at most one active run per object +}); + +host.serve(ChatStore, ({ conversation }, self) => { // local: no principals; views filter (below) + const entries = self.state(ChatEntries); + const shell = self.job(ShellJob, { + run: async ({ command }, ctx) => { + const out = new OutputBuffer({ maxLines: 2000, retain: "tail" }); + const progress = throttle(100, () => ctx.state.change((d) => { d.output = out.snapshot().text; }, ctx)); + try { + const r = await runProcess(command, ctx.abortSignal, (chunk) => { out.push(chunk); progress.schedule(); }); + return { exit: r.code, output: out.snapshot().text }; + } finally { + await progress.flush(); // the last output reaches outcome.state, also after an abort + } + }, + finish: async (outcome, tx) => { + const list = (await tx.draft(entries)).list; + list.push(outcome.kind === "done" + ? shellEntry(outcome.args, outcome.result) + : shellEntry(outcome.args, { interrupted: outcome.error.kind, output: outcome.state.output })); + }, // the run's state document is retired in this commit + }); + return { + shells: shell.running, + runShell: async ({ command, author, requestId }, ctx) => + ({ id: (await shell.start({ command, author }, ctx, { key: requestId })).id }), + abortShell: async ({ id }) => shell.abort(id), + }; +}); +``` + +- **Definition:** `defineJob(name, { version, args, result, state?, concurrency?, accepts?, persist? })`. `args` and + `result` are schemas; `state` is a document definition (a singleton, one document per run); `persist` arrives with + slice 6. `self.job(Job, { run, finish? })` is declared in an object's factory (not in a facet) and returns the job + handle: `start(args, ctx, { key? })` (a `Run { id, outcome(ctx) }` after the commit), `tx.start(job, args, { key? })` + inside `self.tx` (a `Run` handle at once, whose `outcome` rejects if the transaction is discarded), `outcome(id, ctx)` + (unsettled runs only), `abort(id, ctx)` (`"marked" | "terminal"`, as Durable's `abortTask`: stored, resolves once `run` settled; slice 6b) and `running`. +- **1. Intent:** one transaction on the owner's line. Arguments are checked against `args` (closed schema, like + method arguments). Dedupe first: a run with the same `key` that is not settled is returned as it is (the run whose + `finish` this transaction is never counts, so `finish` can start the same job again with the same key). Then the + `concurrency` check: at the limit `start` rejects with `unavailable/busy` (a queue is the owner's, built from a + document and a routine). Otherwise a run record `{ id: uuidv7, version, args, key }` and its state document + (`initial()`) are written in the commit, with whatever else the transaction writes (`tx.start`: a request entry and + the run, both or neither). `run` starts after the commit. +- **2. Run:** `run(args, ctx)` may do anything, outside transactions. `ctx` is a `Ctx` (the object's principal for a + remote type, the host's for a local one) plus `jobId` and `state`: `ctx.state.value` (synchronous) and + `ctx.state.change(fn, ctx)` (a transaction on the owner's line on the run's document, without the run's abort signal). + State changes are accepted until `run` settles, also after an abort (so output written while stopping still reaches `outcome.state`), and are + refused (`released`) afterwards, never recreating a retired document. Each change is a commit and, while someone + watches, a message: authors throttle frequent changes themselves (Chord does not coalesce them); awaiting or + catching each change is the author's (an unhandled rejection is the author's bug). External writes carry keys made + from `ctx.jobId`. A `run` that throws after its signal fired ends `aborted`, otherwise `threw` (reported as + `job_failed`). +- **3. Finish:** when `run` settles, Chord records the outcome on the run (`phase: "finishing"`), then calls the + current code's `finish(outcome, tx)` in a transaction (Durable's rule: only `await tx.*`). In the same commit Chord + deletes the run record and its key (Q9: dedupe covers unsettled runs only) and retires the run's state document. A + `finish` that throws changes nothing: it is retried with backoff (1 s doubling to 60 s), each failure reported + (`job_finish_failed`); after five consecutive failures the run is `blocked { finish_failed }`, its outcome, state + and key kept, retried on the next reload. `finish` is optional (then the commit only cleans up). + + ```ts + type Outcome = { readonly id: string; readonly args: Args; readonly state: State } & ( + | { readonly kind: "done"; readonly result: Result } + | { readonly kind: "failed"; readonly error: JobError } + ); + type JobError = + | { readonly kind: "threw"; readonly error: ChordError | PluginError } // run threw (reported) + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; // persisted runs (slice 6b) + // abort(id); a native restart on reload, in-memory runs (slice 7b); "restart": a persisted run abandoned after a crash (slice 6b), + // or a run cut off by its attempt's seal while the object continues (slice 6a) + ``` +- **Abort is a request:** `abort(id)` fires `run`'s `ctx.abortSignal`; `run` decides what it returns (a result that + already arrived stays a success); a run ignoring the signal keeps its slot. Abort of a finishing run does nothing; + abort of a blocked run drops it (no `finish`, its record, key and state document go, reported; owner, 2026-10-24). +- **Waiting:** `run.outcome(ctx)` resolves with the outcome once `finish` committed; the caller's abort ends only its + wait. Across a link nobody waits on a long call: UIs watch state. +- **`running`:** a JSON watchable, one entry per unsettled run, tagged by phase: + `{ id, version, key?, args, state, phase: "running" | "finishing" } | { id, version, phase: "blocked", reason }`. A + blocked run carries no `args` or `state` (it may be of another version than the code); views filter it as they + filter anything. Per-principal views derive what each principal may see (§6, the shared store): a reader without + output, a guest only their own runs. +- **Concurrency and keep-alive:** `concurrency` counts runs that are running or finishing; blocked runs do not count. + **Runs belong to the object** (owner, 2026-10-26): records, keys, the active count, `running`, run state documents + and pending actions are the object's, across attempts and native restarts; an attempt has the implementation and + its running invocations; a run's state changes and outcome are fenced by the run's own invocation; `finish` runs + on the current live attempt. **Jobs keep their object alive** (D4): running and finishing runs are a count on the object, not references (they + never appear in `host.references`); an object whose last reference goes stays while the count is above zero, and + ends (after its type's `linger`) when it reaches zero, dropping blocked runs with a report. `host.activity.jobs` + counts them. +- **Object end:** revocation (R11), `host.close` and a reload that removes the type end the object at once: runs get + the abort signal, the drain is bounded by `closeTimeout`; an in-memory run's outcome and state go with the object + (no `finish`). Jobs in `local` stores are never revoked (no principal): a store that runs work for a principal keeps + the grant as data and reconciles with a routine (§25). +- **Versions (`accepts`, slice 7b §8):** a job is versioned like everything else; `accepts` maps an older version's args + forwards and its result forwards (`{ 1: { args: (old) => current, result: (old) => current } }`; a producer's + direction, unlike a contract's). Its records keep every run's version: a new attempt's `self.job(Def, impl)` at + another version takes the records over (the same version with another fingerprint, over args, result and state, is + refused at declaration, as documents). A run started on an older version and still running finishes on its old + code across a pure swap (its invocation keeps the old implementation); `finish` runs on the new code with the + outcome's args and result mapped forwards (once per run, never per publication); `running` shows its args mapped; a + run not invoked yet runs on the new code with its args mapped. Its state document keeps its version while the old + code writes it and migrates through its own `migrate` once its run settled or moved to the new code (the outcome + `finish` gets). Without an adapter, for a run of a newer version than the code (a rollback), or for a job the new + code no longer declares, the run is `blocked { version }` (kept, not invoked) until a reload brings matching code; + blocked runs (`finish_failed` ones too) are retried on every reload of their object's plugin; a persisted run of + another version or fingerprint stays blocked with its record as stored until the reload that brings its version + resumes it. A native restart aborts running in-memory runs (`aborted { reason: "reload" }`) and `finish` runs on the + new code; persisted runs are stopped and resumed per their `resume` (6b R3, owner, 2026-10-29). A `finish` in flight + across a reload completes on the code it started with; the next one runs on the new code. A version change aborts + nothing and discards nothing. +- **Persistence (slice 6):** `persist: { resume: "rerun" | "abandon" }` keeps the run record and its state document + in the owner's store (owner: a `local` object with empty params, P3). How a run resumes after a crash is required, no + default: rerun (the current + `run`, args mapped forwards, state migrated; only work since the last state change repeats, so it must be + idempotent) or abandon (`finish` gets `failed { aborted, reason: "restart" }`); wording aligned with Durable + (replay-safe, abandon on restart). A recorded outcome resumes only `finish`, never `run`; an abort request is + persisted. A restart limit as Durable's (owner, 2026-10-27, slice 6b D6): at activation a run recorded `running` was + cut off by a crash: `restarts + 1`, stored, then it resumes (rerun or abandon); a clean close writes its stopped runs + back so the next activation counts nothing; a `ctx.state.change` (progress) resets the count; at five in a row the + run ends with `failed { kind: "restart_limit", restarts }`, `finish` runs, `job_interrupted` is reported (no + parking, as Durable since #11139). A persisted state document may defer storing a change + (`ctx.state.change(fn, ctx, { persistence: { kind: "deferred", withinMs? } })`, Durable's shape, default 1,000 ms; + stored before the outcome is recorded; slice 6b D8). +- **Chaining:** `finish` may `tx.start` the next job, atomically with the outcome (separate steps, branches). +- **Not jobs:** request/response work (a method), standing behaviour (a routine), input into running work (a flow, + §20: the owner's document plus methods), multi-step durable work with sleeps, waits or ownership trees on a host + with a harness (a Durable task). Jobs as objects (#9) are rejected: a third lifetime, two objects per run for two + principals, a `finish` across owners. + +--- + +## 16. The facet + +```ts +import { defineFacet } from "@app/core"; // base code's defineFacet, bound to the app's P + +export default defineFacet((host, self) => { // `self` (contributions) arrives with slice 8 + self.contribute(Http, { routes: { "GET /notes/export": exportNotes } }); + host.serve(Notes, (params, principal, self) => …); + host.serve(NoteStore, (params, self) => …); +}); +``` + +- **Facets are typed by the app's principal (owner, 2026-10-21):** base code exports `defineFacet` already bound to its + `P` (`export const defineFacet = bindFacet

()`, next to the principal union), and every facet (server, worker, + web, TUI) imports it from there; `host.load` accepts only facets typed for the host's `P`, so a factory's `principal` + narrows by variant. Nothing is checked at runtime: the principal a factory receives is the one the link stamped. +- A plugin's per-host setup, **declarations only** (owner, 2026-10-26): `host.serve`, `self.contribute` (slice 8), + `host.references`; no natives, no documents (C16 accepted), no routines, no jobs. Everything a plugin does at runtime + is done through objects: shared or persisted state and standing behaviour live in a `local` store object with empty + params (host lifetime, its own failure domain; its dependents park while it is failed and go on when it recovers), as p2 does (its 63 facets take only + `host`). A facet cannot fail after `load`. +- `host.serve(Type, factory, { linger? })`: `linger` (milliseconds, 0 by default; slice 3b) keeps a live object with + params alive that long after its last reference, so a new reference within that time reuses it with its documents + and dependencies; it never applies to objects that are not live. +- `host.serve` and contributions are valid only while the setup runs. The facet never `self.get`s a type it serves + itself. +- Between plugins only: object types and registries. No natives across plugins except through registry entries. + +--- + +## 17. Registries + +```ts +export const Http = defineRegistry("chord.http", { version: 2 }); // shared: identity only +host.serve(Http, { // the defining plugin's facet: the one provider (slice 8 D1) + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value; reports conflicts + accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, +}); +self.contribute(Http, { routes: { … }, wraps: [ … ] }); +const router = self.registry(Http); // a declared dependency; read router.value at use (await host.registry(Http, ctx) outside objects) +``` + +- Each contributor (a facet or an object) contributes **at most one value per registry**; the table is an immutable, + ordered list of `(contributor, value)`. Contributor order is sorted by plugin name (never load order): deterministic + on every host and across reloads, and deliberately meaningless; meaningful order comes from the owner's composition + (e.g. HTTP phases). A reload replaces a contribution as a + whole. +- **`compose` is a runtime composition, not a staging check:** contributions can be dynamic (known only once a native + is open, or changing at runtime; contributions may be readables), so no conflict check can be complete in staging. + `compose(table, { report })` is pure and runs whenever the table changes (after a commit, when a contributor goes + live or ends, when a dynamic contribution changes). It always delivers a value with its fallback built in (e.g. the + first by plugin name wins) and reports conflicts through `report` (owner notices and telemetry events); readers get + a plain value and never handle conflicts. If `compose` throws, the failure is reported and its + watches get a `failed` call, while `.value` keeps the last good composition (slice 8), **but only while every + contribution it was built from is still in the table** (dependency model decision 5, slice 7a): when one leaves (its + contributor ended, failed or restarted), the last good value is dropped and `.value` throws until `compose` + succeeds, so it never hands out a native whose contributor ended; only before any good composition, or after such a + drop, do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps + its old table). Helpers: `keyedEntries(table, entriesOf, report)` (a map, the first by plugin name wins, a duplicate + reported), `orderedEntries(table, itemsOf, orderOf?)`; both keep each item's plugin (named so beside the contracts' + `keyed`, slice 8 build). +- **Who contributes (R21):** only facets and host-lifetime objects (`local`, empty params); their factory's `self` has `contribute`, a + referenced object's `self` does not (a type error). Reading a registry's composed value stays open to every object. +- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an + object's from `live` until it ends, fails or restarts. +- **The owner decides** conflicts, order and selection (which contributions apply where). +- **One authoritative declaration (Q8):** only the facet of the plugin that defines a registry declares it to the host + (`compose`, version, `accepts`); other plugins' tokens only identify it by name and types (contribute and read, + never bring their own `compose`). A reload of the defining plugin replaces the declaration atomically (`serve` + replaces; a removed defining plugin's provider goes, its readers wait `not_served`) and the table + recomposes. Without a declaration (the defining plugin is not loaded) the registry has no composed value: readers + wait `failed { waiting }` (`unavailable/undeclared`), like a request for a type nobody serves; contributions wait in + the table. +- **Versions (Q8, the owner-adapter pattern):** a contribution carries the registry version its plugin was built + against; older ones are adapted with the defining facet's pure `accepts` when the table is built. A contribution + without a matching version or adapter (older without adapter, or newer) is left out of the table and reported (an + owner notice naming plugin, registry and versions); its plugin keeps working. Contributions are in-process values + (often functions) and are not schema-checked. +- **Use sites guard contributions:** a contribution is untrusted code (third-party plugins are a goal of p3). Every + use site catches synchronous throws and rejections when it calls an entry, reports naming the contributor (the + composed table keeps each entry's plugin name), and continues with the other entries. Durable already does this for + extensions (`extension_error`; a throwing tool becomes an error result); `compose` reports through `report`. +- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`; S2's `broken` and S3's backoff superseded + by slice 7a): the token is shared, the provider serves it (`host.serve(R, { compose, accepts })`); `self.registry(R)` + in a factory is a declared dependency: the attempt parks until the registry has a usable value at its version + (composed; mapped by `accepts[old].value` for an older reader), otherwise it shows `failed { waiting }` naming the + registry (`not_served`: `unavailable/undeclared`; `version`: a reader newer than the provider or without a mapping, + `contract_mismatch`; `compose`: a first `compose` or the mapping threw; `loading` before the publication), woken at + every change of whether the value is usable; `host.registry` waits the same way up to its `readyTimeout`; + contributions never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last + good composition (above); `retryOn` retries on value calls only; `host.root(fn, context, { principal? })` seals its + context at settle, abort and `host.close`. +- **Registries may carry natives.** An entry lives exactly as long as its contributor; consumers look up at each use + or follow the composed value and keep nothing from an entry beyond a use. +- **Contributions that change (owner, 2026-10-19; W4):** registries are host-local, so contributions are usually + in-process values (functions, runtimes, handles). A contribution is fixed for its contributor's life (dynamic parts + behind handles read at use; a change means a restart or reload), a `Watchable`, or a `NativeWatchable`; Chord follows + the latter two through the internal flush: on every change it replaces that entry (a new immutable table) and runs + `compose` again, in the owner's flush, never inside another owner's commit step. The composed value is itself a + `NativeWatchable`; use sites watch it or read its current value. For in-process values that change, + `self.writable(initial)` is a `NativeWatchable` with `set` (identity compared, never ops, never a state member): + + ```ts + const ext = self.writable(mcpExtension([])); // e.g. MCP: tools known only at runtime + self.contribute(HarnessExtensions, ext); + self.on(mcp.tools, ({ value: tools }) => ext.set(mcpExtension(tools))); + ``` + + Tools, sections, hooks, wraps and tasks reach the harness as one Durable `Extension` per plugin contributed to + p3's core registry; `compose` builds Durable's registry snapshot (Durable applies selection and wraps, and pins a + snapshot per task phase). Skills and other lists work the same way. +- **A contribution may be another registry's value** (it is followed: its change recomposes). A ring of registries + reading each other is caught when one composes (audit A2): a registry read while it composes throws `invalid/cycle`, + the contribution that read it is left out and reported `contribution_failed`, the rest of the registry works. +- **Removing a contribution:** there is no remove call (`contribute` is declarative, in the factory). An entry leaves + the table when its contributor ends, or after a reload whose new code no longer contributes. A live contributor that + has nothing to offer for now sets its readable to the registry's empty value (an extension without tools, empty + routes); the table keeps exactly one entry per live contributor. +- Registries add behaviour (tools, routes, renderers, commands, input gates); they never replace an owner's checks. + No lifecycle in entries: anything needing cleanup is a native of the contributing object. + +--- + +## 18. Start data and platform bindings + +There are no host values. Start data (session id, init, data directory) is a `local` object with empty params (host lifetime) +and constant state (`constant(value)`: a readable that never changes, R23), served by the base code's core facet from the start message: + +```ts +export const SessionInfo = local("app.session-info", { version: 1, params: {}, members: { init: InitSchema } }); + +// base code (apps//src): the core facet, passed to load() like any facet +export const coreWorkerFacet = (init: Init) => defineFacet((host) => { // base code's own bound defineFacet + host.serve(SessionInfo, () => ({ init: constant(init) })); // local, empty params: host lifetime +}); + +// any plugin +host.serve(NoteInbox, (_, self) => { + const info = self.get(SessionInfo, {}); + const watcher = self.open(async (context) => + watchDir(join(info.init.value.dataDir, info.init.value.id, "inbox"), { signal: context.abortSignal }), (w) => w.close()); + return {}; +}); // NoteInbox: local, empty params → host lifetime +``` + +Plugins read start data through a reference, after activation (in openers, derives, routines and methods); nothing +reads it synchronously in a factory. Platform bindings the host owns (a Durable Object's `state` and `env`) are core +objects whose `self.open` returns the binding and whose `close` does nothing. A long-lived native from the host is a +host-lifetime object in a core facet (the Durable harness, section 24). + +**Core vs. plugin is decided by where the code lives:** base code (`src/`, imported statically by the start code under +a fixed name such as `@app/core`, not switchable, a change restarts the host) or a plugin directory (found by listing, +loaded through loader D, named by its `package.json`, switchable, a change reloads it). Core is what the app always +needs and everything builds on (harness, catalog, access, session location); optional features are plugins (code +mode is a plugin). Both use exactly the same primitives. + +--- + +## 19. Presence + +```ts +const viewers = self.derive([host.references(Notes)], ([refs]) => refs.length); +``` + +`host.references(Type)` (slice 4b) is a JSON `Watchable` of the references to objects of a type on this host: +`{ params, principal?, incarnation }[]` (named `origin` until 2026-10-27). It is one rule (owner, 2026-10-24): the incarnation of the host that made +the reference as its caller sees it, so two tabs are two origins, and a guest's `ctx.get` shows the guest's tab. An +entry is added when a reference binds and removed when it ends, in that batch; a departure discovered while a batch +is sending lands in the next batch (so one commit stays one message per link). The host keeps a type's list from its +first `host.references(Type)` on (one small object per asked type; entries come and go with references). **A link drop is a departure** +(accurate and simple); smoothing (a phone switching networks) is the UI's job, e.g. a debounced viewer count. Counts +per person or origin are derived by the app from the principals and origins. There is no consumer-side linger +(a closed tab closes its link); provider-side `linger` keeps a just-closed object warm and keeps the worker alive +(section 3). Replaces the presence, viewer and login-abandon leases. "Leaving" is never derived from an unmount +(section 21). + +--- + +## 20. Input into running work + +### Flows kept by their owner + +```ts +start({ provider }, ctx) → { id } // uuidv7 +LoginFlow {id}: state | { kind: "running"; step?: { id, … }; notices } | { kind: "done" | "failed" | "cancelled"; notices } +answer({ id, step, value }, ctx) → ok | "step no longer active" +cancel({ id }, ctx) +``` + +The UI watches the state; a late or second answer to a step that is gone is refused; a promise-based callback (pi-ai's +`prompt`) stays inside the owner (its job keeps the resolver). Imperative consumers (tests, scripts, a CLI, an agent +tool) use a generic client helper over the same state, not a second protocol: + +```ts +async function* steps(flow, ctx) { + let last = 0; + for (;;) { + const s = await until(flow.state, (v) => v.kind !== "running" || (v.step && v.step.id > last), ctx); + if (s.kind !== "running") return s.kind; + last = s.step.id; + yield s.step; + } +} +for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); +``` + +A long-held server `step()` call is not used: a link drop fails it and the consumer cannot tell whether it missed a +step; state re-holds by itself and `last` keeps steps from being lost or repeated. + +### The Questions service + +Another pattern (`apps/p2/docs/questions.md`): a shared store with per-principal views (asker: `ask`; viewer: `open`, +`answer`), typed questions (`text`, `select`, `confirm`), policies over a live audience (`first`, `quorum`, `all`), +reattachment by `key`, deadlines, authorization per question, secrets redacted. Fits tool approval and a question the +agent asks everyone present. Its store must live in an object (it predates "natives only in objects"). + +> **Decision needed: where a flow lives.** Options: (a) a keyed document of its owner; (b) an object `LoginFlow {id}` +> living until done. (Jobs as objects were rejected, §15; a flow kept as its owner's document plus a job is the likely +> answer.) + +--- + +## 21. Reload and rollout + +### Per host + +`host.reload(facets, context)` (slice 7b; decision "Slice 7b's open points, as recommended" (1)) takes the host's +**complete new plugin set** after `load`: a plugin in it and not loaded is added; one loaded with another facet value is +changed (Chord compares no code); one passed as the very same facet object is unchanged and untouched (owner, +2026-10-29: base code passes its core facet value every time); one loaded and not in it is removed. It resolves with a +report once the commit happened and every object took its fate (swaps and retries had their first dependency check; +it does not wait for objects to go live); a plugin whose new code fails keeps its old code and is listed in the +report's `failed` (per plugin, below). +Reloads run one after another per host; before `load` resolved it throws, after `close` it rejects `released/closed`; +`context` bounds it (decision 4: no install watchdog): its abort before the commit rejects `aborted` and changes +nothing, after the commit it only ends the wait: it rejects `aborted` with `details.committed`, a `CommittedReload` +`{ added, changed, removed, failed }` naming what the commit applied (the plugins component adopts it, so a later +reload never passes a committed plugin's old facet; owner, 2026-10-30). + +```ts +type ReloadReport = { + readonly plugins: { readonly added: readonly string[]; readonly changed: readonly string[]; readonly removed: readonly string[] }; + readonly failed: readonly { readonly plugin: string; readonly error: unknown }[]; // new code failed: old code kept, an added one left out + readonly objects: readonly ({ readonly object: string } & ( + | { readonly kind: "swapped" } | { readonly kind: "restarted" } | { readonly kind: "ended" } + | { readonly kind: "parked"; readonly cause: FailureCause } | { readonly kind: "retried" }))[]; +}; +``` + +``` +1. load new bundles (the app's or loader's: a facet arrives evaluated); a plugin whose build or evaluation + fails keeps its loaded facet (`reload(facets, context, { failed })`) +2. staging every new setup into a staged table, and every affected object's new factory against a recording + `self` (declarations only, nothing activated, bound or opened); a plugin whose new code throws keeps its + old code, staging runs again without it, then the closure rule (below) +3. commit one batch: the host switches to the new code atomically, a publication (7a) +4. after each object takes its fate (below) +``` + +- **Per plugin, never all or nothing** (owner, 2026-10-30, decision "Per plugin at startup and reload"; supersedes 7b's + rollback per host): a plugin whose new code fails at a reload (its build or evaluation, a setup's or factory's throw + in staging, a declaration error, `checkAccepts`, a type or registry served twice, a remote type on a host without + principal or address) keeps its old code (its loaded facet, unchanged) and is reported in the reload's report + (`failed: [{ plugin, error }]`, the same list `load`'s report has; an added plugin that fails is left out); staging runs again without it, and every other plugin of the reload applies. **The + closure rule:** a changed plugin whose recorded requests would not bind because the plugin they need was kept old (or + is missing) is kept old too, so a consumer edited together with a broken provider keeps working on its old code. At + startup the same holds: `load` leaves a failing plugin out and resolves with `{ failed: [{ plugin, error }] }`; its + consumers wait `failed { waiting }`; `load` never closes the host for a plugin (base code decides whether its core + failing is fatal). A loader that kept a plugin on its loaded facet because its new code failed before staging (its + build, its evaluation) tells the host with `reload(facets, context, { failed })`: the host lists it and applies the + closure rule to it as to a plugin whose staging failed. What no rule detects: a consumer relying on new behaviour of a kept provider whose contract did not + change (as in a rollout across hosts). Runtime states (registry conflicts, failed opens, migrations) stay per object + after the commit (`onError` as always). +- **Staging** runs every new setup (each plugin into its own table, merged; an unchanged or kept plugin's loaded + entries are reused, its setup not run again) into a staged table beside the live one (types with their factory, + `linger` and provider, `bindProvider(contract, options.accepts)`; registry providers; facet contributions by + plugin), then, for + every live, activating, parked or failed object of a changed or removed plugin whose type is still served, its new + factory with the object's params (and principal) against a recording `self`: it records what the factory opens, + the documents, tables and jobs it declares with their versions, its contributions and requests, runs the checks an + attempt makes without effects (declared twice; persisted data outside a stable identity; contributions outside a + host-lifetime object; a document or job at an older version, or at its version with another schema), and hands out + inert handles. Staging is synchronous, so no request or arriving hold meets it half done (audit A1: model §1's + `installing` is cut). +- **The commit** (one batch): the served types become the staged ones (an unchanged plugin's entries stay as they + are); a removed plugin's registry provider goes (its readers wait `not_served`), a reloaded one's replaces the old + (`RegistryRecord.serve` replaces), the reloaded plugins' facet contributions are replaced by plugin (object + contributions follow their objects), every changed registry composes once; a removed type's presence list goes. + After the batch, each object takes its fate, host-lifetime objects of new types start (no barrier), and the + publication wakes the requests indexed under the names the reload serves (`UnboundIndex.wake(names)`, per name), + refused holds of those names re-run their start. +- **Fates** (decided in staging; objects of plugins not in the reload are untouched): + + | The object | Fate | + | --- | --- | + | its type is removed | **ended**: as an end (natives close `end`, callers of running calls rejected `released`, handlers drained, in-memory documents gone, persisted ones kept); its holders' requests are not ended: they wait unbound `not_served` (§7) | + | failed, parked, activating or waiting for its backoff | **retried** on the new code (`ref.retry()`'s rule, whatever broke it); an activating attempt is sealed and its natives close `reload` | + | live, neither its live attempt nor the new factory opens natives, and no table of the new factory is newer than the store's | **swapped** in place (below) | + | live, otherwise | **restarted**: new calls park (it shows `loading`), its contributions leave, running calls drain (bounded by `closeTimeout`; a call outliving the drain is cut off by the seal fence), the attempt is sealed (in-memory job runs end `aborted { reload }`, persisted ones are stopped and resumed, R3), natives close `reload`, the new attempt activates after that teardown (documents stay; eager table migrations run as at activation) | + | any of the above whose new code waits for a dependency | **parked** (decision 4): no overlap of old and new code; reported with the cause | + + A restarted object gets a new instance id (an old activation chain never matches it); a swapped one keeps it; a + retried one keeps it as any retry does. Cycles are checked again for the new attempts; `parked_long` is unchanged. +- **The pure swap** (decision "The seal fence": not a replacement while its calls run): a new attempt of the same + object (its documents, line and job records) activates at once; calls arriving meanwhile wait at + the gate (bounded by `readyTimeout`, `unavailable/park_timeout`). The new attempt's requests are made and bound + before the old attempt's are released (a request equal to an old one binds to the same target with no gap, M§3.9). + Running calls finish on the old attempt with its handles valid, except R15: an old call touching a document whose + version rose throws once (`released/version`), the document intact, the call not resent; the same for a table whose + version rose (old table handles' version fence: old code never issues SQL for a dropped column). Once the swap + ended (the new attempt live, parked or failed) and the old attempt's calls settled (bounded by `closeTimeout`, + `drain_timeout`), the old attempt is sealed and its references released; its job invocations finish on their old + code. Retiring attempts are bounded by `closeTimeout` each; a later swap never waits for them (several may retire + at once after quick reloads), while a restart, an end and the host's close do (an end rejects their callers at + once). The references move to the new instance's members by name (one frame from the last value; a member the new + variant lacks fails `invalid/member`; §7); exposed derives are matched by their member names, internal ones are + rebuilt; in-memory documents migrate through `migrate` when the new code declares a higher version; the + contributions are replaced in one batch; `self.writable` starts at its new initial value (decision 2); `retryOn` + sources are re-declared. **Routines hand over per object:** the old attempt's routines are aborted; the new ones + start once the old routine calls finished, bounded by `closeTimeout` (a stuck one is reported `drain_timeout` and + the whole old attempt is sealed then, audit A12: one seal token per attempt); calls never wait. +- **Natives are not kept across reloads by name:** natives have no line, transactions or schema, so old and new code + would interleave unchecked. In practice a restart costs little (the Durable harness is base code, terminal PTYs live + in their own process, model runtimes, sandboxes and the search index rebuild in milliseconds); reloading the plugin + that owns SSH connections or the relay socket hurts. +- **No factory trial, no migrate dry run, no consumer contract check during staging:** staging records declarations; + incompatible consumers wait at bind (`failed { waiting }`, `contract_mismatch`). The one exception is the closure + rule above: when a plugin of the reload failed, a changed plugin's recorded requests are checked as a bind would + check them, to keep it old. +- **Start barrier** (host start and first load; dependency model §6, decision (a), slice 7a): `load` runs the facets' + setups (the core facet with its start data among them) and **publishes** them in one batch (types served, + registries served and composed once, contributions in their tables); requests that waited for the publication are + placed after that batch (holds and `host.hold` wait for the publication only, never for the barrier: two hosts + whose objects need each other's types start without a deadlock). Then it activates the host-lifetime objects and + resolves when each has its **first outcome**: live, failed by its own cause, or parked with no blocker that is a + local object still without its first outcome (`local`: executing on this host, a keyed pin still loading here + included; a wait on another host or on a failed dependency ends the first pass), or ended (`host.close` settles a + waiting `load`). One first-outcome signal per object, separate from the readiness dependents wait for, counted once, + never reopened by retries. So `CodeMode → Harness` waits for the harness's outcome. **What it promises:** every + host-lifetime object here had its first try, and the contributions of those that went live are in their tables; + **not** every tool that will arrive (a parked or slow contributor, tools found in `onLive`, MCP): what a first turn + needs is p3's explicit admission check. Base code calls `harness.resume()` after `load` (the harness is an ordinary + host-lifetime object that opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing waits + for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported with + their error. Reloads need no barrier. + +### Rollout across hosts + +- **The rollout is the app's** (decision "Slice 7b's open points" (3)): it calls `host.reload` on each host in order, + server, then workers, then clients; Chord guarantees per host (staging, commit, a failing plugin kept on its old + code). The plugins component's `rollout` (slice 7c) runs such steps. An install never waits for activations (a reverse-order rollout would + deadlock). +- **Each host knows only its own install** (model §5): until its reload commits (while the app bundles, before + `host.reload`), a hold that does not fit its running code is answered `failed { waiting, version }` and the commit + wakes it (a blip of at most one install; staging itself is synchronous, audit A1). What a consumer on + another host sees follows §7: kept and freshly bound while its version is accepted; `failed + { waiting }` (`version`, `not_served`) otherwise, bound again by the install that serves it; a consumer whose own + code moves to a new version releases its old request and makes a new one. +- **The app's loader keeps its own duties:** the desired code set, a digest over the facet bundles' content hashes (an + unchanged server restart reinstalls nothing), serialized installs, a host that starts or reconnects installing the + current set before anything is routed to it, a rejected own install failing its own requests with the reason. + +### Loading plugins and the reload loop (slice 7c) + +- **No dev/prod split** (owner, 2026-10-30): a Chord app is self-modifying (the owner or an agent edits a plugin while + it runs), so finding, building, loading, watching and rolling out plugins is core machinery: one component, + `chord/next/plugins` (`src/next-node/plugins/`, Node-side; the kernel stays portable and takes facets however they + were made: static imports in tests and Durable Objects work the same). Reloads stay explicit: the watcher marks, + never applies. +- **A plugin** is a directory with a `package.json` (its `name` is the plugin's name, npm style) and one file per facet + kind the app names; plugins come from roots found by directory listing (no source parsing; the first root wins a + duplicate name). Dependencies between plugins resolve by contract name at runtime; names are prefixed + (`.`; tools `_`). A plugin carries the contracts it was written against (decision "Contracts come from where the + plugin's code came from"): plugins that change together import each other by relative path within a root + (copied into the importer's script); others take contracts as an npm dependency at a fixed version (built in slice 12, + which removed slice 10's imports by plugin name). +- **Loader D:** one esbuild script per facet (its own files, copied files of other plugins, its npm `dependencies` + inlined; host packages, `chord.external` packages and Node builtins external, passed in at evaluation); each facet kind is defined once, `kinds: { : { platform: "node" | "browser", hostPackages } }` (slice 11); evaluated with + `vm.Script` under a stable filename per (plugin, kind, digest); no module map: old generations are reclaimed by the + GC; V8's compilation cache turned off when the component is created. `pluginFile(path)` resolves a file of the + calling facet's own plugin. A plugin's dependencies are installed into its directory (`installDependencies`, as + coding-agent does: `--omit=dev --legacy-peer-deps --ignore-scripts`). esbuild is an optional peer dependency. +- **The reload loop:** a watcher marks plugins dirty from the files each facet used (a copied contract dirties every + user) and flags changes of base code (the app restarts its hosts); a build is per plugin (a failed one keeps its old code); `rollout` runs the app's + steps in order (server, workers, clients; decision "Slice 7b's open points" (3)). A reload control (a button, a CLI, + an agent tool) is the app's. One base bundle per surface (host packages and the app runtime) is an app's startup + optimization, the web's in slice 11. +- Project plugins, trust, snapshots, scopes and npm/git package installs: `apps/p2/docs/plugin-loading.md` (still the + reference for those; not in 7c). + +### The plugin model (2026-11-01; slice 12) + +Decisions "Contracts come from where the plugin's code came from", "Plugins import only the app's versioned API", +"The plugin model, points 3, 4, 5 and 8, imports and builds", "Single-file plugins", "Testing a plugin alone". The +rule underneath: a plugin carries what it was written against, never the running app's newest file. + +- **What a plugin may import:** its own files; the app's versioned API (`@acme/app/v3`, never Chord directly: Chord is + not a host package plugins may name; the API re-exports the Chord surface the app supports); its npm `dependencies` + (bundled at the installed version); other plugins' contract files: by relative path within one repository + (`../notes/contract.ts`, the same commit holds both sides), by package name across repositories (`@bob/notes`, + `@bob/notes/web`, pinned by an npm version or a git ref `github:bob/notes#`; a plugin inside a + multi-plugin repository through a dependency on the whole repository, by path into it or its `exports`). A package + lists the plugins it provides in `chord.plugins` (as pi's manifest). One facet imports one API version only. +- **Versions:** the app lists the API versions it provides; an unlisted one is a build error naming the available ones. + The process runs one Chord: an older API version is a layer over today's, re-exporting Chord's compatibility module + for its previous API while Chord keeps one; contracts bind through `accepts` whatever Chord runs. +- **Split by host kind:** the API has a shared module and per-kind modules (`/v3/web`, `/v3/tui`, `/v3/server`, + `/v3/worker`); a plugin's contract files mirror it (`contract.ts`, `contract.web.tsx`, `contract.tui.ts`, ...). A + contract file mentions native things only through `import type`. The build checks every import by the file it + resolves to: a per-kind file or module only for its kind, a package the kind does not provide refused. A token or + `local` contract used on one kind lives in that kind's file. Shared UI between plugins: an imported file (inlined, + frozen at the importer's build) or a registry entry (one live instance). +- **Base code** is the API modules, bootstrap (hosts, store, components such as `createHttp` and `createWorkers`, which + take settings and process values and return facets) and one start-data facet serving process values as core + objects. Every implementation, the harness and core registry providers included, is a plugin; the app's own root is + required (a failure at startup stops the app). +- **Single-file plugins:** `..ts(x)` at a root's top, grouped by name, no `package.json`, importing only the + API; a directory of the same name replaces it as the same plugin. +- **Installs** from git (a branch, tag or commit cloned into a root) or npm. **Builds** are one esbuild call per facet + from source (TS, TSX, JS, JSON, CSS, assets and WASM by URL); plugins bring no build hooks. +- **`testPlugin(dir, { kind, modules, stubs, start?, principal? }, ctx)`:** the plugin built as in production on an + in-memory host and store, its dependencies served by stub facets, contributions to registries collected for the + test, the contracts nobody serves listed. + +- **UI reload vs. leaving:** "leaving" is never derived from an unmount, only from explicit navigation or presence (a + dropped reference); a component unmounted by a reload has not left. + +> **Decision needed: project plugins.** Per-directory plugins with trust and snapshots (`plugin-loading.md`) need their +> own pass over this design. + +--- + +## 22. Wire + +Exposed documents, derives and readables are state members: a snapshot in `bound` when a hold binds, ops per commit +(a draft's recorded ops, no diff), ops from diffs for derives, ops of job state documents (slice 5), and failed +members in the entry of the message that carries them. **Commits are atomic for consumers:** locally, subscribers are +notified once after every document of the commit is adopted; remotely, one `state` message per commit per link and +origin carries an entry per touched hold, applied in one batch before listeners run (also when one Durable commit +touches several holds), so remote consumers never see a torn commit. Consumers apply ops with `applyImmutable`: no +validation, no clone of snapshots or ops. + +- **The protocol version** travels in `hello`; hosts of different versions refuse each other (a web page reloads for + a newer server). Error reasons and message kinds are part of it. Until p3 ships, changes keep `PROTOCOL_VERSION` 1 + (nothing is deployed); after that every message or reason change bumps it. +- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a pi-schema schema `MESSAGE`, a tagged union over + `kind`, from which the protocol types derive; every decoded message is checked against it (compiled once) unless + the link is trusted (section 3). Kinds: + + | Kind | Direction | Fields | + | --- | --- | --- | + | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | + | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | + | `revoke` | parent to child (route link) | the revocation value (4b) | + | `release` | to provider | `origin?`, `hold` | + | `retry` | to provider | `origin?`, `hold`, `hops` (the retry's hop budget, 1-16: 16 from the first host, one less for each host it passes on from, none sent at 0; audit A5); relays forward it unchanged | + | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | + | `abort` | to provider | `origin?`, `call` (named `cancel` until 2026-10-27) | + | `ack` | to provider | `origin?`; one per applied `bound`/`state` message | + | `closed` | relay to provider | `origin`: the downstream origin ended | + | `bound` | to consumer | `origin?`, `holds: [{ hold, variant, members: [{ member, value }], failed }]`; one batch | + | `state` | to consumer | `origin?`, `holds: [{ hold, members: [{ member, ops }], failed, retired }]`; one batch (`retired`: members retired since the last message, slice 6a) | + | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }` (`kind`: `transient`, `broken` or `waiting`, slice 7a), `retryIn?` (a transient cause's backoff; absent: broken, waiting); each new cause of a parked object is a message of its own | + | `loading` | to consumer | `origin?`, `hold`: a new attempt after a failure, or a relay's upstream dropped | + | `result` | to consumer | `origin?`, `call`, `value?` (part; absent: `null`), `segments?` | + | `error` | to consumer | `origin?`, `call`, `error` (a ChordError's JSON or a plugin's code, message, details) | + | `gone` | to consumer | `origin?`, `hold`, `error`: the held object ended (revocation 4b; a reload never ends a hold, it unbinds it with `failed`, slice 7b; relays never make one) | + | `ping`, `pong` | both | the heartbeat | + + A `bound` on a hold that is bound already replaces its replicas in one batch, another variant allowed (re-holds, + slice 7b's restarted objects and its swaps to another variant). +- **Codecs are pluggable per link** (both ends use the same): `binaryCodec` (default: a kind byte with a flag for a + following origin varint, varints, raw incarnations and fingerprints, length-prefixed strings and parts; about 23 B + for a streamed token) and `textCodec` (Delta-style JSON tuples without field names, a message with an origin wrapped + as `["@", origin, tuple]`; for debugging; `describeMessage` prints any message so). Measured: a 2,000-token reply + about 46 KiB instead of 196 KiB. No CBOR (revisit only if benchmarks show encoding as a bottleneck). +- **Parts are opaque** (slice 4a): `params`, `args`, `value`, snapshot values and ops are a `Part` (JSON text at the + origin, its UTF-8 bytes once a binary codec received it). A part is parsed only where it is owned, right when its + message is handled (the provider parses params and arguments, the consumer results, snapshots and ops), so a relay + forwards it unparsed and a malformed part is still a protocol error. A message naming an unknown id is ignored + before its parts are parsed. The text codec re-serializes parts at decode (one parse per part per hop, accepted for + a debugging codec). +- **Names travel as indexes** into the hold's bound version (each variant's state members and methods sorted; variants + sorted). A reload that no longer accepts that version unbinds its holds first (`failed { waiting }`, slice 7b). +- **Checks by layer:** the codec's structural decode, the `MESSAGE` envelope (skipped on trusted links), the bound + version (ids, indexes, order), then the owners (`checkParams`, `checkArgs`, Delta's decoder). +- **Unknown ids are ignored on both sides (Q1):** they are normal races (a `result` crossing an `abort`, a `call` + crossing `gone`, a state part crossing a `release`); the side that ended a hold or call settles its own pending work + (a consumer receiving `gone(h)` rejects its pending calls on `h`; per-direction order guarantees nothing else for `h` + follows). Protocol errors close the link and are reported (`protocol_error`): a message failing the schema or the + codec, a malformed part, a reused live id, more than N running calls of one origin, a broken handshake (a first + message that is no `hello` of the link's codec, a second `hello`, another protocol version). +- **Byte segments:** a payload (arguments, a result) whose method's schema has a `bytes()` leaf carries each + `Uint8Array` as a raw segment referenced by `{"$bytes": i}` and read back as a view of the received message; `$bytes` + is the one reserved key there (`invalid/args`). Never base64. State is JSON only. +- **Encoders:** each member of a hold has its own Delta encoder and decoder (path interning end to end between + provider and consumer), restarted with every snapshot. Two holds of the same principal (two tabs) encode the same + ops separately (O(change) each). A part is encoded once at its origin; relays never re-encode it. +- **Sending:** one `state` message per commit per link and origin while the window has a slot; otherwise the ops wait + in the members' pending queues and leave as one message when an `ack` frees a slot (section 3). +- **Relays** read and rewrite only envelopes (section 3, "Relays"). +- **Size limits per part (R2, Q2; slice 4b):** limits apply to each part (wire and decoded size), configurable per + link, checked from the part's length before it is parsed or inflated; the decoded limit is enforced in the one + decode function. An oversized part fails only its own hold or call (`too_large/part`); the other parts of the message + apply; a snapshot over the limit fails its hold with no automatic re-hold. The decoded limit travels with relayed + work: the relay checks the admitting link's wire limit itself and stamps its decoded limit on the upstream `hold` + (honoured on the parent link only, never taken from a client; calls inherit their hold's), and the host that + inflates enforces the smaller of its own and the stamped one while inflating, after the part's owner and id are + known (parts for unknown ids are never inflated, Q1). A relay checks only what arrives from downstream; answers + from upstream (the app's own workers) it forwards unchecked and unparsed, and the end that parses them checks them + against its own link's limits (owner, 2026-10-24: each end checks what it parses; a relay check needed a re-hold + with the same id, which old messages in flight could mistake for the new hold). So IPC carries no limit of its own. + A reference failed `too_large/part` is never held again automatically: it leaves its path, so a reconnect does not + hold it; only `ref.retry()` puts it back (a new hold). This lives only in the consumer's memory, as long as the + reference. The + WebSocket's `maxPayload` stays only the transport's hard cap (e.g. 64 MiB; code 1009 closes the connection). + Defaults for client WebSockets: 16 MiB on the wire and 64 MiB decoded per part; none on IPC. +- **Compression at the origin** (owner, 2026-10-19; slice 4b): a JSON part above a threshold (e.g. 16 KiB) is + compressed once by a host with a synchronous compressor when it encodes it (`createHost({ compression })`, Node's + zlib adapter in a Node-only subpath), whatever the link (IPC included), only if it shrinks; hold params are never + compressed; byte segments stay raw; the codec is in the part's envelope (the text codec tags every part, so no JSON + value is reserved); relays forward it compressed; the end that owns it inflates it. Option (b) (owner, 2026-10-24): + hosts without a compressor (browsers) send uncompressed and inflate asynchronously (`DecompressionStream`), holding + later messages of that link in order (bounded by the window and N; acks go out after apply). **Child links refuse + compressed parts** (protocol error): honest clients never send them, and a 65 KB part could inflate to 64 MiB on the + server; so asynchronous inflate happens only on a host receiving from its parent or a route. Because a host cannot + tell whether its parent sees it as a child (a tab) or over a route (a worker), call arguments sent up a parent link + are never compressed; results and state going up are. Compressed hold params are a protocol error on every link. + The asynchronous inflate runs one part at a time; the queue behind it is bounded by the sending peer's window and N, + so only a misbehaving parent or route peer could grow it (accepted: those are the app's own hosts). The adapter reports "too large" + and "corrupt" as a tagged result. Limits and the threshold count bytes. No per-link compression. +- **Trace carriers (R5; slice 9)** are an optional bounded field of `call` and `hold` (key count, string lengths); + relays read them from the envelope. +- **Adapted bindings get diffs** (slice 7b; decision "References are the only indirection" (5)): a consumer bound + through `accepts` attaches to the object instance's shared mapping for its version: per member one derive of the + instance's member through `(current) => old` (a changed member), of its whole state (a removed member), or the + member itself (unchanged), made on first use, dropped when no reference of that version is left, gone with the + instance; one map and one diff per commit per version whatever the number of old references, each hold encoding its + own ops, so an old tab never receives a large state whole on every commit. A new instance (a reload) has its own + mappings; the references move to them as to any new instance (one frame, never a new mapping applied to the old + instance's values). A version the instance does not accept gives no member: the reference waits unbound with its + last values. +- **A `failed` on a bound hold unbinds it** (slice 7b, model §5): the consumer applies it as a start-over plus the + failure (its calls waiting for a slot reject with it; the provider rejects its calls in flight once); the provider + detaches the hold's sources, drops its unsent pending ops and a queued `bound`, and keeps the ack slots of `bound` + and `state` messages already sent until their `ack` or the link's loss. A reload sends it when the object's type is + removed (`not_served`), its version is no longer accepted (`version`), or a swap parks on a failed dependency; a + restart or a parked swap still loading sends `loading`; both are followed by a fresh `bound` when the object is live + again (applied as a re-binding). A swap in place keeps the hold bound and sends it a fresh `bound` once the new + instance is live (audit A9, 2026-10-30: one rebind path; a tab's watchers may get a frame for an unchanged member). + +--- + +## 23. Optional components + +The app kit's Node components (slice 10, `slice-10.md`, decision "Slice 10's points"): `chord/next/http` and +`chord/next/workers` in `src/next-node/`; the web page host and the TUI are slice 11. Each is ordinary code on Chord's +public API; nothing in the kernel depends on them. + +### HTTP (`@earendil-works/chord/next/http`) + +The core is one Web-standard handler built from the registry `Http` (decision "The HTTP core is a fetch handler"): + +```ts +const http = createHttp(server, { // settings are arguments, fixed for the component's life + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own Node listener; none: the app mounts `fetch` + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, + closeTimeout: 30_000, + onError, // HttpError +}, ctx); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }, appCore], modules }, ctx); +// an app with its own server: hono.all("/*", (c) => http.fetch(c.req.raw, { localAddress, localPort, remoteAddress })) +// plain Node: nodeRequest(message, res) converts with the listener's checks; server.on("upgrade", http.upgrade) +await http.close(ctx); // before host.close: drains running requests +``` + +- **The facet** (plugin `chord.http`, passed as the same object at every reload) serves the registries `Http` + (`compose` builds the router) and `Authenticators`, and with `listen` the listener: a host-lifetime object whose + native is Chord's Node server, opened right after `load`'s publication, so a reload never closes it (decisions "The + HTTP server is a native object", "Settings are arguments, extensions are registries"). `fetch` resolves the + registries once; requests before the publication wait (503 past `readyTimeout`). +- **Contributions to `Http`:** `{ routes?, wraps?, sockets?, maxPayload?, hosts?, framers?, relays?, csp? }`, a value + or a writable (a change recomposes the router). A request uses the router as it was when it started. +- **The security baseline** (decision "The HTTP component's security baseline", on for every request and upgrade + before any plugin code): (1) **provenance**: local only when the connection's local address is loopback, `Host` is + a loopback name with the port and no `Forwarded`/`X-Forwarded-*` header; a forwarded request on loopback only + through a contributed relay whose `verify` names its origin (`{ kind: "relayed", relay, origin }`); otherwise only a + contributed host (https); duplicate or missing `Host`, userinfo in it, absolute-form targets: 403; an unparsable + target 400; `TRACE`, `TRACK`, `CONNECT` 501 (a route for them is refused at compose). A mounted handler without + `localAddress` and `localPort` is never local. (2) **Foreign requests** (Fetch Metadata) never reach the + authenticators: anonymous, except a top-level `GET` navigation to a `page: true` route. (3) A foreign `Origin` is + 403; an unsafe method from a browser needs the app's own `Origin`; a `crossSite: true` route opts out and always + runs anonymous. (4) **Default headers** on every answer, the component's own included: `no-store`, `nosniff`, + `no-referrer`, `X-Frame-Options: DENY` (without framers), a CSP (`default-src 'none'`, scripts `'self'` and a + per-response nonce `request.nonce`, `connect-src` self and its WebSocket, `frame-ancestors` the framers or + `'none'`, plus contributed `csp` sources); a response's own headers replace same-named ones. (5) `cookie(name, + value, { secret, request, sameSite?, maxAge? })` and `readCookie(request, name, secret)`: `HttpOnly`, `Path=/`, + `SameSite=Strict` or `Lax`, `Secure` on a remote origin, value and expiry signed. (6) `bytes` refuse foreign + requests, are attachments unless `inline` (then with a `sandbox` CSP), download names stripped of control and + bidirectional characters. +- **Who:** the `Authenticators` registry, in contributor order (an app needing precedence contributes one that + decides); the first principal wins, a throwing one is reported and skipped, none: anonymous. They run raced against + the request's signal and stop once it aborts. Who someone is stays app policy (D3); the registry replaces D3's + single function so plugins (a device pairing, a share) can add credentials. +- **A request's scope:** one `host.root` with its principal (an explicit `undefined`: anonymous; an anonymous scope + gets no remote object served here, `forbidden` → 403) spans matching, wraps, the route and the response until its + body is written or abandoned; its signal aborts on disconnect. `HttpRequest`: `method`, `path`, `params`, `query`, + `headers`, `origin`, `provenance`, `nonce`, `json(limit?)` (1 MiB; 400, 413), `body(limit?)`. +- **Patterns** (D8): exact segments, `:name`, a trailing `/*`; decoded segments with `/`, `\`, NUL, `.` or `..` are + 400; the most specific match wins segment by segment; a pattern from two contributions is a reported conflict, the + first by plugin name wins; `HEAD` falls back to `GET`; no route: 404. A route is a function or `{ handler, page?, + crossSite? }`. Wraps (D2) as below. + +```ts +type HttpResponse = { readonly headers?: Readonly> } & ( // D5 + | { kind: "json"; value: JsonValue; status?: number } | { kind: "html"; body: string; status?: number } + | { kind: "bytes"; mime: string; size?: number; read(offset, length, ctx): Promise; download?: string; inline?: true; ranges?: boolean } + | { kind: "redirect"; to: string; status?: 302 | 303 | 307 } | { kind: "error"; status: number; message?: string }); +``` + +`bytes` are pulled in 256 KiB reads as the consumer reads (ranges 206/416, `HEAD`, chunked without `size`, R6); no +read starts once the request's scope is sealed; a short or throwing read destroys the connection. 204, 205 and 304 +carry no body. `fileResponse(path, { download?, inline?, ranges?, mime?, root? })` stats at once (missing: 404; with +`root`, a real path outside it: 403), opens its handle at the first read, closes it when the scope seals. A throwing +route or wrap is a 500 without its message. + +```ts +type Wrap = { + name: string; + phase: "observe" | "headers" | "transform"; // the component's phases, outermost first + scope?: "own" | "all"; // default "own": only this contributor's routes + route?: string; // only this route; a missing route is a reported conflict + wrap: (req, next, ctx) => Promise; +}; +``` + +Wraps (D2) run within a phase in contributor order (no numeric priorities: plugins have no single registration +order; Fastify's lifecycle phases and encapsulation are the model). Wraps may add a security boundary on top of a +route's checks, never instead of them. + +- **Sockets** (D6): `sockets: { "/terminal/:id": (socket, request, ctx) => … }`, a `ws` WebSocket after admission + and the authenticators (no principal: 403 before plugin code), `maxPayload` per contribution (1 MiB), pinged after + 15 s of silence and terminated 15 s later. **Links:** `links.path` upgrades to `host.accept(webSocketLink(socket), { + principal, limits })`; a client that stops reading is cut off past 4 MiB waiting. **What is removed is closed at + once** (decisions Q6, "allowed hosts only through contributions"): a link or socket whose host, relay or socket + handler leaves the router is terminated and its scope sealed; an upgrade is checked again against the current router + after its authenticators answered. +- **`close(ctx)`:** new requests and upgrades refused (503), sockets and links closed (1001, terminated at + `closeTimeout`), running requests waited for up to `closeTimeout`, then sealed; before `host.close`. +- **Cost** (decision "a route's overhead accepted"): +20-21 µs per request over a plain Node handler (standard bench), + +12-13 µs warmed. `ws` is an optional peer dependency (D7). + +### Workers (`@earendil-works/chord/next/workers`) + +`createWorkers(host, Kind, { kind, entry, execArgv?, env?, authorize, principal, start, lock, idleMs?, plugins, +onOutput?, onError }, ctx)` routes `Kind` itself (`host.route`, keeping its `RouteHandle`): a forked worker per +address over IPC, the link made at the fork, a start message `{ address, start, scripts, lock }` with the app's start +data and the kind's complete current scripts (`plugins.scripts(kind)`); the 60 s start timer covers fork, lock and +connect only (Q4). `workers.state` (a JSON watchable per address key), `stop`, `retry`, `step(name)`, `close`; +`workerMain(fn, { hostPackages, redact, onError })` is the child's side (`fn({ address, start, parent, plugins, +onClose }, ctx)`, `plugins` a `WorkerPlugins` evaluating the shipped scripts, closers in reverse order). One live +worker per address (`processLock`: an abstract socket on Linux, a named pipe on Windows, a socket file elsewhere that +fails closed when stale; a start waits for a draining old holder without polling). + +- **Keep-alive** (D9, R3, Q5): the child reports when `host.activity`'s references, jobs and lingering objects and the + app's `busy` counts flip between all zero and not (local calls do not count; the first state with `ready`, which + also carries the load report); the server stops a worker idle for `idleMs` (5 min). +- **Never restarted by itself** (R2): a crash, a `fatal`, a link lost while its child lives (killed after a 5 s grace) + or a failed start leaves the address failed; its holders see `failed { waiting }` and stay on the path (Q1); + `retry(address)` or a rollout reaching workers clears it and starts the held ones (Q2). `stop` suspends the address + while it has holders (R1, Q3). +- **Errors in public and private** (decision "worker errors in public"): `state` and holders get a fixed message + (`worker failed to start`, `worker exited (SIGSEGV)`) or the child's redacted `fatal`; the full error and the stderr + tail go only to `onError` (`worker_failed`). Every app hook of a start (`start`, `principal`, `lock`, the fork) is + inside that boundary; a throwing `authorize` refuses the hold and is reported `authorize_failed`. `worker_exited` is + reported only for an exit nobody asked for. +- **Workers never build** (D11): the server's plugins component builds every plugin; workers evaluate shipped + complete sets (`createPluginRuntime`, no esbuild); `workers.step(name)` ships a rollout's set to every worker (D12: + `{ host, report } | { host, stopped: true } | { host, error }` per worker). + +### The app kit (`chord/next/app`; slice 13) + +Decisions "The app kit, point 1" to "point 5". `startApp(options)` on the server, `startWorker(options)` in a worker +entry. Opinionated and small: an app is its API, its built-in plugins and a start script. + +- **Shape:** the app's layout is fixed: `src/api/v/` (shared and per-kind modules) and the built-in plugins root + `plugins/`. The data directory is a `startApp` option (no environment variables), default a dot directory named after + the app in the home directory. One server; worker kinds are a list the app defines, each a host kind with its own + params (sharding is the app's) and how it starts (entry, start data, principal, idle time); zero kinds is a server + with clients only. +- **Principals and users:** the principal is tiny (a kind and an id) and the app's type; the kit needs only its server + and worker variants. User data lives in objects keyed by the user's id. Optional kit plugins: users (accounts with an + app-supplied profile schema), authenticators (device pairing, invite links, an ephemeral pick-a-name login). +- **Authentication:** no owner by loopback alone: the start script prints a link with a one-time token that sets a + signed owner cookie; deny by default (the HTTP component refuses requests without a principal unless the route or + socket is public); a session helper issues and revokes sessions; command-line clients send a token in a header (an + owner token file readable only by the app's OS user; a device pairing flow from other machines). +- **Storage:** one store for the server, a directory per worker under its kind and address with its store; plugins + never use data-directory paths. A files plugin per address (`put`, chunked transfers, a watchable index, caps) is + the default file storage; raw directories only where a real path is needed, owned by one object, under the worker's + directory. Export and import per address: the store's archive plus the worker's directory. +- **Plugins:** the app's root (required) and the user's root in the data directory; installs from git or npm; one + "apply changes" call builds and rolls out to the server, then every running worker, exposed by the app as it likes. + +### Web and TUI (slice 11) + +Decision "Slice 11's points" and "Slice 11, P3" to "P9". + +- `chord/next/web`: serving the surface, a page's client host with its connection loop (p2's `keepConnected`), loading + and applying the plugins' web facets. A base bundle per client kind built on the server from the app's entry + (content-hashed); at page load the kind's current plugin set as one combined file named by its hash, later changed + plugins one script each, each a script that registers itself (no eval); new scripts apply in place (`host.reload` in + the page); pages learn of them through an object the web component serves (the current set); a base code change + reloads the page. +- `chord/next/react`: `HostProvider`, `useHold`, `useWatch`, `useRegistry`, `useRefState`. +- `chord/next/tui`: a Node client connecting with a token, always running the downloaded code in a sandbox (a + permission-dropped worker thread, a bridge terminal, the trusted launcher outside); a component scope (`hold`, + `watch`, `registry`, released on dispose, one `changed` callback for the app's redraw), no pi-tui dependency. + +### Channels + +WebSocket byte channels with acknowledged windows, separate from object links, wait for a second user after the +terminals (slice 10 D6); they would be built on top of plugin sockets. + +### Chunked transfers + +Large byte transfers are chunked by the owning object: `begin`, `append({ upload, offset, bytes })`, `finish`, +`read({ id, offset, length })`, about 1 MiB per chunk. The awaited `append` is the backpressure; after a drop the +writer asks for the stored offset and resumes; unfinished uploads expire (the owner's routine). A helper wraps the +loop. Chord has no stream type (it would need its own rules for drops, cancellation, relays and restarts). + +--- + +## 24. Durable integration and portability + +### Durable + +- Durable keeps importing today's Chord's `context`, `delta`, `json`, `output` and `sqlite`; `next/` imports the same + modules, so `Context`, `Draft` and `JsonValue` are one type. Durable's conversation view, task graph and documents + are `Watchable`s through Durable's watches (Durable's `WatchHandle` now lives in Chord); Durable adopts and dispatches + them synchronously inside the storage's commit (S1, W1; with `StoreStorage` that is the store's commit step), so a derive over a Durable view travels with it. This needs Durable to publish a commit when it adopts it (owner, 2026-10-26; a pi change, upstream first; explainer `durable-observers/`): inside `adopt`, its trackers, telemetry events and internal observers (view mounts, task graph, scheduler, submissions) run, and the host `observeCommits` observers are dispatched in one microtask queued before the observers, so they run before every wait the commit resolves and every abort it queues, but after the storage's synchronous step, never inside Chord's batch; host listeners and watch listeners have no order between them (owner, 2026-10-26). Durable's public primitive for adapters is a synchronous observer, `observeConversation`, `observeTaskGraph`, `observeDoc` (its `advance(call, ctx)` gets a `WatchCall` and runs inside `adopt`: record only, no awaits, no Session calls, no throws); `watchXXX` is built on it for app code. Chord's `self.follow(attach)` (slice 6b D9) is a source over it. Otherwise a derive over two Durable views computes between their frames of one commit. With `StoreStorage`, Durable's `adopt` runs inside the host's batch; the store knows nothing of Durable (no special cases) and needs only its step and flush to stay synchronous. Durable's + state getters (`documentState`, `viewState`, `taskGraph()`) and the `replicatedState` family are removed (W14). Only + plain data crosses between Durable and v2 objects. +- **The harness is an object in the core facet** (base code): a `Harness` `local` object with empty params, so host lifetime (it runs for the worker's whole life, + whether or not anyone holds it) opens Durable paused with `self.open` during the barrier and gets model + access and execution environments from registry entries, looked up at each use: + + ```ts + const models = self.registry(HarnessModels); // declared dependencies (slice 8 S3): activation waits + const envs = self.registry(HarnessEnvironments); + const extensions = self.registry(HarnessExtensions); + const harness = self.open(async (context) => Harness.open(await StoreStorage.open(store, Harness.name, context), { + models: () => models.value, // read at use + env: (target) => envs.value(target.cwd), + settings: () => currentSettings(), + registry: registryReader(extensions, context), // Durable's RegistryReader over the composed value (slice 8) + }, context), (h, _reason, context) => h.close(context)); + ``` + + Durable's `models` and `settings` accept functions (pi `7ec1f648e`). Durable opens with scheduling paused; base code + calls `harness.resume()` after `await host.load(...)`; the worker connects to its parent first (W6), so host-lifetime + stores can reference server types during the barrier; requests arriving during `load` park like any call to a + loading object (refines R17). A missing tool gives its call an error result the model sees; a missing model makes the generation + report "model unavailable" and wait (not fail) until a model contribution appears; a task whose definition is missing + stays blocked (`missing_task`) until the plugin is installed or the task is aborted, as Durable does. +- **Durable's views from other objects** (slice 8 §5): the harness object contributes its `Native` to a + registry of p3's; another object declares that registry (`self.registry`), wakes on it (`self.retryOn`) and follows + a view through its value (`self.follow`). A Session close ends Durable's synchronous observers `failed` ("session + closed"; decision "Slice 8 S1"; app watches still end `ended`), so the follower fails `transient` and follows the + next Harness once its contribution arrives. `registryReader`'s listener runs inside Chord's batch: it only wakes + Durable's scheduler (a Chord call there throws `invalid/reentrant`). +- `HarnessModels` and `HarnessEnvironments` start with one contribution each, from the models and envs plugins' + objects that own the natives; several providers or environment kinds are their `compose`. After a models reload + the next request uses the new runtime; a running stream finishes on the old one. +- **Chord from Durable code (Q6):** tools, hooks and sections get Durable's `ToolExecutionApi` and a `Context`, never a + Chord `ctx`. p3's core opens the harness with a base context holding the host under a typed key + (`withContextValue(ChordHostKey, host, …)`); every phase context inherits it. A core helper opens a call scope from + any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, context)` (host principal; attribution as + data; references released when `fn` settles; slice 8 D3). No wrapping of + tools and no p3 `defineTool`. Clients are never reachable this way (they are not addressable): a tool that needs the + user goes through an owner's state, e.g. the Questions service (`apps/p2/docs/questions.md`): the model-facing + `ask_user` tool and any other tool call `Questions.ask`/`wait`, tabs render the open questions and `answer` them. + Documented with this example. +- **Code mode** (a plugin) keeps owning its `pi-codemode` sandbox; its tool entry looks up the current sandbox when + `execute()` starts, so a call pinned by a phase before a reload (e.g. waiting in an approval hook) starts on the new + sandbox; on `reload` and `close` its `close` detaches the old sandbox, returns at once and disposes of it when its + scripts finish or after `closeTimeout` (30 s at most, R16, Q7); after a worker stop Durable replays as today. +- **Telemetry:** `HarnessOptions.telemetry`; submissions and tasks store an optional carrier (section 29). + +### Portability + +Adopted in the kernel: host = identity plus incarnation; the loader is an adapter; values independent of the +transport; durable state only in persisted documents. A Durable Object deployment +(`apps/p2/docs/research/chord-on-durable-objects.md`) would map a logical host to a DO, compose plugins at build time +(Workers forbid `eval`), replace the relaying tree with edge routing and use alarms for routines. + +> **Decision needed: a Durable Object profile.** Problem: a DO can lose its memory while clients stay connected +> (hibernation); routines need wake-ups (one alarm per DO); links between DOs are RPC stubs. Hot reload is not needed +> there; plugin composition is. For later. + +--- + +## 25. Apps: ownership and porting + +- **Every thing has one owner, the smallest object it belongs to;** ownership nests; rules set at creation are written + in the creating commit (Durable `createConversation` with `init`). +- **Core owners are always present;** a plugin owner may be missing (optional dependencies are registries owned by the + dependent, section 7). Avoid a core god object: each core owner owns only what belongs to it. +- **Revocation of work in shared stores:** a store that runs work on behalf of a principal stores the grant (or device) + as data with the work and reconciles from the grant owner's live grants (a routine aborts jobs, withdraws questions, + drops queued input whose grant no longer exists); no revocation hook. The window between a revocation and the + routine's run is accepted (R10): a step started in it still runs; there is always some window. +- **Reading without starting a worker** is plain pub/sub in an ordinary plugin: a host-lifetime object of its worker facet + records the summary locally and publishes it to the plugin's server object (a method call from a routine on the source); clients read + the server object, which never starts a worker; it keeps the latest value per session (a document if it must + survive a server restart) and ignores reports from an older worker incarnation. **The caller's incarnation is + stamped by Chord (R20):** a call arriving over a link carries the calling host's incarnation from the link handshake + (`ctx.incarnation`, next to `ctx.principal`, not part of the principal or its key, so object identities do not + change with restarts); nobody sends it as data and it cannot be forged. On relayed calls it is the **origin** host's + incarnation (Q13, owner, 2026-10-21): the relay takes it from the link the hold came in on and sends it in + the upstream `hold` (a client's value is ignored, like its principal); calls inherit their hold's (section 4). Session list, status, children, unread and + artifact lists follow this. +- **Porting guideline:** what core enforces on data lives in a format core owns, written by the plugin that creates + the data; a plugin's private document is read only by that plugin. Example: subagents mark one-shot and + cwd-following child conversations in a core per-conversation policy document (`readOnly`, `followsSessionCwd`); + chat and relocation read it; `ReadOnlyConversations` and `SessionCwdFollowers` go away. +- **p3 is built next to p2,** fully isolated (`apps/p3`, own data directory), without migrating p2 data; plugins are + derived from the functionality map (`apps/p2/docs/p3-functions/`), not ported one to one. +- **Session lifecycle:** archive retires nothing (stops the worker, ends guest access, closes forwards); a session + delete removes the whole session (tagged documents, the worker's databases, files, the catalog entry). + +--- + +## 26. Decisions needed + +Open, decided with the slice named or a concrete case (numbers kept stable; everything else is decided and folded into +this document): + +| # | Topic | Section | When | +| --- | --- | --- | --- | +| 11 | where a flow lives | 20 | when login is ported | +| 14 | project plugins | 21 | own pass over `plugin-loading.md` | +| 16 | a Durable Object profile | 24 | when a DO target becomes concrete | +| 24 | a per-type drain override: a host option or fixed (no install watchdog: `reload` takes the app's context, decision "Slice 7b's open points" (4); the routine handover uses `closeTimeout`; a failing job `finish` backs off 1-60 s and blocks after five failures, decided) | 3 | if a real case needs it | +| 28 | Astra's circuit breaker (a hard transport-buffer cap plus a stall deadline); the heartbeat is built, the cap was never decided | 3 | if a real case needs it | +| 29 | round-3 cut C11 (`ctx.trace`); C3 and C4 are decided (§14, §15) | 8 | slice 9 | + +Apps (p3), from the functionality map, decided per domain when p3 is designed: + +| # | Topic | Problem and options | +| --- | --- | --- | +| 17 | session files | one owner `SessionFiles {session}` on the server for uploads, sent-file snapshots and artifact versions (caps per kind, index readable without a worker, copy on clone/fork, delete with the session); proposed, sub-points: one preview limit, fork's artifact list, copying attachments into remote environments | +| 18 | session delete | per-session objects plus `SessionTag` plus a session directory, or owner hooks through a registry; search index per session or global plus a hook | +| 20 | connection pools | SSH connections to one environment exist four times: one owner with a pool, or per-use connections | +| 21 | config files with a single writer | settings, models and auth are watched by every process and written by two: one owner on the server, others follow its state | +| 22 | file formats as versioned interfaces | `envs.json`, downloads, uploads have no version: version them like documents, or keep each internal to one owner | +| 23 | the terminal host process | a detached process with its own code: part of the revision (reloaded or restarted with it) or deliberately outside | + +--- + +## 27. Versions and adapters + +| Surface | Version | On change | +| --- | --- | --- | +| object type | `version` plus contract fingerprint | every change bumps; forward evolution and `checkAccepts` (Q10); otherwise `contract_mismatch` | +| registry | `version` | `accepts` adapter on contributions | +| document | `version` plus schema fingerprint | pure `migrate`, on access; a schema change without a bump fails the object `broken` | +| job | `version` | `accepts` maps older args and results forwards (`finish`, `running`); the state document's own `migrate`; `blocked` without an adapter or for a newer version | +| native | none | the object restarts on reload | +| Durable tasks and documents | `version` | Durable's `migrate` | +| link protocol | protocol version | refused; hosts update together | +| UI components, Chord and app APIs | none yet | exposed as they are; versioned with an SDK later | + +**Contract evolution in code (Q10; implemented in `packages/chord/src/next/contracts/`, see its README):** + +```ts +export const NotesV1 = remote("notes", { version: 1, params, variants: { + editor: { state: { notes: arr(NoteV1) }, methods: { add: method(NoteV1, Id) } }, +} }); +export const NotesV2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2)).args("add", NoteV2)); + +const accepted = checkAccepts(NotesV2, { + 1: { state: { editor: { notes: (current) => current.map(toV1) } }, methods: { add: { args: (old) => toV2(old) } } }, +}); +``` + +- A history starts with a base written in full with an explicit version (`remote()` / `local()`; `local` has flat + `members` and no variants). `evolve(prev, (e) => …)` always continues it with structured edits: `state`, + `addState`, `removeState`, `args`, `result`, `method(name, args, result)`, `addMethod`, `removeMethod` (local: no + variant argument), and for a local contract's keyed members `keyed`, `addKeyed`, `removeKeyed` (slice 7b; owner, + 2026-10-26: keyed members' versions are mapped per key). Every version is materialized and fingerprinted at declaration, so an old plugin's own copy of + v1 computes the same fingerprint; no build step, no generated files, no hand-copied fingerprints. +- **A new history base** (a params change or a change of the variant set) is a new `remote()`/`local()` with an + explicit version and no accepts for older versions: old plugins get `contract_mismatch` and must update. +- **Mappings live with the provider,** never in the contract: the pure `checkAccepts(contract, accepts)` (called by + the objects layer) requires, at type level and again at runtime, a mapping for every member that exists in the old + version and changed or was removed since: changed state `(currentMember) => oldMember`, removed state `(current + variant state) => oldMember` or `"refuse"`, `args: (old) => current`, `result: (current) => old`, removed methods + `{ call }` or `"refuse"`, changed keyed members `(current document) => old document` or `"refuse"` (mapped per key + for an older local consumer), removed keyed members `"refuse"`; added members need nothing. No-op edits need identity mappings. Untyped JavaScript + providers fail at declaration naming the member; unknown keys and wrong shapes are rejected. +- **The fingerprint** is a hash (`hash64`, portable, not a security boundary) of a schema-aware, data-only normal form + (`NORMAL_FORM_VERSION`, tied to the protocol): annotations dropped only on schema nodes, `required`/`enum` sorted, + union and intersection order kept (it is meaning: defaults), a whitelist of JSON Schema keywords, executable features + rejected; bytes are a `{ type: "bytes" }` data marker. Schemas are deep-copied into a private history (never frozen; immutable by convention). +- **Schema helpers:** `str num int bool nul lit oneOf union obj arr map opt bytes unknown` and `type Static`. +- Chord wraps state mappings as a derive and diffs per object and version (section 22); members an old consumer does not know do + not appear. +--- + +## 28. Building: slices + +Chord v2 is built in vertical slices; the plan, the feature map (every feature of this document assigned to the slice +that builds it) and the carry-over of deferred obligations are in `slices.md`. Slice specs (`slice-N.md`) stay as the +history of how each slice was specified; this document states the current rules. The first layered build (kernel steps +1-6) was deleted on 2026-10-22 and survives only in git history. Done in pi: `pi-telemetry` (propagation, links, +metrics, the store interface, the renderer), `chord/context` (`Context.telemetry`), pi-ai and Durable telemetry, +`chord/telemetry-sqlite` (section 29), Chord's shared `Watch`/`NativeWatch`/`PendingOps`, Delta's `appendText`. + +**Rules for every slice** (owner, 2026-10-22): the code follows the root `AGENTS.md` section "Chord v2 Code Style"; +one test file per source file, one `describe` per mechanism, the spec's test list case by case; cost guards through the +in-memory telemetry adapter (section 30); GC tests for every cache and binding; one or more runnable examples +(`packages/chord/test/next/examples/NN-topic.ts`); hot-path slices add a light benchmark script +(`test/next/bench/`, by hand) with results in `benchmarks.md`; `npm run check` and the slice's tests before reporting. +Caches are `WeakMap`s keyed by their owner; no module-level strong caches; no hot-path work no decision asked for. The +function doing counted work counts it through its context; call sites never count for others; plain totals without +site labels. + +**Testing the core:** every slice spec contains an explicit test list, one group per mechanism with its edge cases, +tested individually and exhaustively: activation and parked dependencies (the dependency model: unbound requests, waits without timers, wakes on every +change, retry passing on, a dependency failing while live, cycles, the start barrier's first outcomes), revocation over running jobs (R11), compatibility (R6), params +and arguments for every caller (R7), migration results and old calls on migrated documents (R8, R15), flow control +(the window per origin, folding, relays forwarding and returning acks), the incarnation fence (R20), the harness start +after the barrier (R17), native restart patterns (R16), and W1's publication list (a throwing derive, a chain A→B→C +where B throws, two owners alternating with a derive over both, reentrancy, a watch started after 150 frames, stop +during a running call, a retire with a derive over it, a reload during a routine call, GC counts of retained +revisions). + +--- + +## 29. Telemetry + +Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`; one adapter per host +(`createHost({ telemetry })`, NOOP by default) decides sampling, buffering, export and filtering. + +- **Types:** the adapter is a `TelemetryContext` (`startSpan` plus optional readonly `metrics` and `propagation` + capabilities); only hosts hold it, to start roots. `Context.telemetry` is always a `TelemetrySpan` (the NOOP span by + default); every span carries its adapter's `metrics` and `propagation`, so code with a context can record metrics + and inject carriers. `withTelemetry(span, context)`, `traceIn(options, fn, context)` (context last; pi change with S1); `value()` and keys stay. + `Ctx` extends `Context`, so `ctx.telemetry` is the current span; app values such as the principal stay out of + Durable's context. `ctx.trace(name, attributes, fn)` creates a child span and gives `fn` a child `ctx`. +- **Adapters trust typed inputs:** no validation of malformed caller values and no try/catch around recording; only + carriers from another process are checked (keys present, strings), else the root. Callers never protect themselves + against broken adapters. +- **Across links:** `call` and `hold` carry the caller's carrier as a bounded header field (section 22); the provider + extracts it and runs the method under it; relays add a `chord.relay` span. + + ``` + chord.call Notes.add [tab A] + └─ chord.relay [server] + └─ chord.method Notes.add [session:S1] principal=guest + · event chord.commit docs=2 ops=5 + ``` + +- **Shared work** (one activation for several getters, a worker start) is its own root span linked to its first + trigger; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background closes are + root spans linked to their trigger where one exists. +- **Tiers:** spans for calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine + runs and job runs; events for commits inside methods, retries and parking; metrics for state messages, WebSocket + bytes, commit durations, in-flight calls, active holds, open links, park durations and backpressure waits. IDs are + attributes of spans, never of metrics. +- **Nothing is measured or walked for telemetry:** only sizes the code already has. Errors carry a fixed category and + a message truncated to 500 characters; cancelling is not an error. +- **A typed telemetry schema** ships with Chord, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable + (`DURABLE_TELEMETRY_SCHEMA`) have; call sites that only have a context use the generic `addEvent`. +- **Storage and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare): one + row per settled span (events and links inline), metrics aggregated per series per 10 s, a 1 s batched flush with + retention (7 days), a bounded queue (dropped and counted). The shared read interface `TelemetryStore` (Jaeger + style): `findTraces` (name, attributes, errors, time window, limit, cursor) returns complete traces, `getTrace`, + `findMetrics`. Spans carry global ids and a trace id; links carry trace and span id. One renderer + (`pi-telemetry/render`) prints links and never follows them. A web trace viewer in the playground reads the store. +- **Durable** (done): submit and phase spans, stored carriers on submissions and tasks (`TaskOptions.trace`, creation + links), events after commit, report events; pi-ai emits `pi.ai.request`/`pi.ai.poll` from one observer in + `lazyStream`. Persisted Chord jobs store their start carrier. + +## 30. Cost model + +As little compute and memory as possible while staying correct; Delta was built for this (its own primitives are +practically free: `diffRevisions` on shared subtrees 0.018 ms, `applyImmutable` per small change 0.004 ms). Measured +before/after per hop: `apps/p2/docs/p3-reviews/v2-cost-model.md`. + +| Hop | Work | +| --- | --- | +| commit | Delta's `prepare` (copies and checks only placed values), adopt by pointer swap; storage writes the delta | +| reads in a transaction | the draft overlay, no copy; unchanged = no write | +| subscribers on the same host | the same value and ops by reference | +| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, cookbook) | +| send | one encode per touched hold member per commit at its origin (the member's Delta encoder); while the window is full, per op one path key and one map lookup (leaf index), and one encode per waiting member per freed slot | +| relay, per message | decode the envelope, swap the origin (one shallow copy), encode the envelope; parts copied once into the outgoing buffer, never parsed; per `bound`/`state` one FIFO entry; two bytes of origin per message upstream | +| relay, per hold | the address's params checked once (route admission, `authorize`) | +| receive | the compiled `MESSAGE` check on untrusted links; arguments validated once at the provider (compiled closed schema), params checked; one `ack` per `bound`/`state` message | +| heartbeat | one timestamp write per received message; one timer per host | +| apply | `applyImmutable`, structure shared | +| new hold | the value encoded once (unavoidable) | +| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted member, object instance and version (while a reference of the version holds it) | + +**Cost guards:** Chord reports full-value walks, deep copies, encodes, schema validations and materializations as +ordinary telemetry metrics, always on (no test mode; with the NOOP adapter they cost an empty call where such work +happens). Tests use the in-memory adapter and assert them per hop (e.g. one appended note to a 500 KB document with +10 viewers: no walks, no copies, one encode per hold part at its origin and none at relays, no validations); a `vitest bench` suite measures the hops. + +--- + +## 31. Rejected + +`replicatedState` and its source family as public API; a separate IndexedDB backend; generic JSON rows and +store-maintained index entries; last-write-per-row compaction; recording raw SQL statements or SQLite changesets; +plugins' own databases holding source data; rollback after losing the driver role; JSON-field operands in bulk +updates (until needed); a lazy-lifetime flag; a `lifetime` option; derive `previous`. Drain-based handover; version fences without transactions; object restart for state changes; Durable's Session as the +object store; volatile documents; per-group lines; generic call or request ids; optimistic transactions; an outbox for +transient delivery errors; revoking proxies on `.value`; resources replaced in place (`replace`, `deps`, stale +handles); registries created at runtime and entries resolved through mutable indirection (contributions that change +with their contributor, readable contributions and readers looking up the current value at use are fine, R25); generic +registry keys with a `conflict` flag; +`restart_required`; detached references counting toward presence; contracts taken from the running app; ceremony +(`native/` directories, `defineNative`, lints); weakening type checking; "magic" input-field matching for delete; a +factory trial; a migrate dry run; principal field patterns and re-stamping for revocation; `storeKey`; a consumer contract check during staging; reload votes; `defaultProjectTrust: "always"`; loader variants A and B; a topology layer above the host API; a stream type in +Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; host addresses as data naming an +input field; host-name strings in app code; host values (start data and platform bindings are core objects); +consumer-side linger; registry composition as a staging check; a JSONL document storage; `tx.get` for documents; validating state and documents on commits or reads (migration results are validated once, R8); copying values on same-host calls; op-rewriting projections and a +view algebra; keeping natives across reloads by name; Chord ending the process on a storage failure; an `optional` +flag on references; cross-relay reset coordination; count limits that refuse calls or holds; relays acking per hop, relay resets +(`reset { hold }`, `reset { holds }`), relay recovery and id renumbering at relays (replaced by flow control per +origin); a byte window, credit messages, snapshot admission and debt; `reset` from a consumer; a round-robin of +relayed calls at the relay; `state: false` on references (C8); `inVariant`/`expectVariant` (C15); per-call deadlines. diff --git a/packages/chord/docs/next/durable-observers/book.md b/packages/chord/docs/next/durable-observers/book.md new file mode 100644 index 000000000..71f6f4833 --- /dev/null +++ b/packages/chord/docs/next/durable-observers/book.md @@ -0,0 +1,432 @@ +--- +title: In the Step: when Durable's views change, and who hears it +cover: In the / Step +kicker: An explainer · Chord v2 slice 6 and pi-durable · 2026-10-26 +subtitle: Why Durable's views must change inside the storage's commit, why the first build broke hosts, and what the "publication at adoption" proposal changes +stats: + - 1 | Durable commit, seen as one change + - 4 | regressions the first build had, all in one gap + - 7 | things that change for hosts compared with HEAD + - 5 | decisions for the owner +note: | + Written for the owner from the build of `/home/badlogic/pi-observers` (uncommitted), the four reviews + (`/tmp/observers-review-{astra,fable,opus,sol}.md`) and the four checks of the proposal + (`/tmp/observers-proposal-{astra,fable,opus,sol}.md`), whose probes ran against HEAD, the first build and the + proposal. Code is Durable's `session/session.ts` and `harness/*.ts` at HEAD, simplified. Nothing here is decided + beyond what the decision log says ("Slice 6 first decisions", refined 2026-10-26). +colophon: In the Step · first edition · 2026-10-26 · from the observers build, its reviews and the proposal checks +--- + +# Before we begin {.preface} + +Mario's phone shows a p3 conversation. Next to the transcript a small status line reads "last: tool call, 1 task +running". It is a Chord derive over two Durable views: the conversation view (the transcript) and the task graph (the +running tasks). + +The model asks for a tool. Durable makes one commit, seq 41, that appends the tool-call entry and creates the tool's +task. The phone should go from "last: assistant, 0 running" to "last: tool call, 1 running" in one step. Today it can +show "last: tool call, 0 running" in between: a state that never existed. Slice 6 also puts Durable and Chord into one +store with one commit step, where every owner's memory must show a complete prefix of the log (S1, W1). Both need the +same thing: Durable's views must change inside the storage's commit, not after it. + +This book explains where Durable publishes a commit today, why moving half of it broke hosts, and what moving all of +it changes. + +## Seven things to know + +:::findings +1. **Durable adopts inside the storage, but publishes after it.** `Storage.commit(writes, adopt, context)` calls + `adopt` once the batch is durable; Durable's views, scheduler and host listeners all run later, after + `Storage.commit` returned. +2. **Two watches tear one commit.** A derive over two Durable views sees seq 41 in one view before the other. +3. **The first build moved only Durable's internal observers.** Host listeners and telemetry stayed after the + storage. Everything a host could see in between came out in the wrong order. +4. **All four regressions live in one gap:** the time between `adopt` and the end of `Storage.commit`. +5. **The proposal removes the gap:** the whole publication happens inside `adopt`. +6. **It changes seven things a host could observe**, most of them timing nobody documented, one a bug fix, one a real + problem for Chord. +7. **The real problem has a small fix:** host listeners one microtask after `adopt`, outside Chord's batch (Opus's + variant). +::: + +## Words used throughout + +**Durable** is pi's agent harness; a **Session** is its one line of atomic commits. **Storage** is Durable's storage +interface; `Storage.commit(writes, adopt, context)` stores a batch and calls **`adopt(seq)`** once it is durable. +Durable's **trackers** hold each loaded document's value in memory. **Internal observers** are Durable's own commit +listeners (`observeCommits`): the conversation **view** mounts, the **task graph**, the **scheduler** and the +**submissions**. **Host listeners** are what an app registers with `subscribeCommits`. A **watch** is a +`WatchHandle`: its listener runs a microtask after a frame arrives. **The gap** is the time between `adopt` and the +moment `Storage.commit` resolves. In Chord's store, the **commit step** is the store-wide serialized part of a commit +(backend transaction plus adoption), and the **batch** is Chord's synchronous adopt, mark and flush around it. +Letters in traces: **A** = `adopt`, **R** = `Storage.commit` resolves, **H** = host listeners, **C** = +`session.commit()` resolves. + +# Today {#today lede="Where a Durable commit is adopted and where it is published, at HEAD."} + +A Durable commit runs on the Session's line from start to end. Inside, `#runCommit` hands the batch to the storage +with an `adopt` callback, waits for it, and only then publishes: + +```ts +// session.ts, #runCommit, at HEAD (simplified) +const adopt = (seq: Seq): void => { + adoption = { documents: tx.adopt(seq) }; // trackers swap to the new values +}; +seq = await this.#storage.commit(writes, adopt, context); +// ... back in the Session, after R: +this.#publish(seq, writes, documents, context); // telemetry events, internal observers, host listeners +return result; // C +``` + +The SQLite storage calls `adopt` right after `COMMIT` returned. JSONL calls it and then still awaits its own cleanup +of sidecar files before it resolves: + +```ts +// jsonl/storage.ts +adopt(seq); // A +await this.reclaimSidecars(reclamations, context); // still inside Storage.commit +return seq; // R +``` + +:::sequence animate | One Durable commit at HEAD. The trackers change inside the storage; everything that tells anyone about it happens after the storage returned. +actor session "Session" "the line" accent +actor storage "Storage" "JSONL" muted +actor obs "internal observers" "views, graph, scheduler" +actor host "host" "subscribeCommits" +actor watch "watch listeners" "microtasks" blue +session -> storage: commit(writes, adopt) +storage -> storage: write, marker durable +storage => session: A: adopt(41), trackers swap +storage -> storage: reclaim sidecars +storage --> session: R +session -> obs: telemetry events, then observers (views advance, waits resolve) +session -> host: H: subscribeCommits(41) +obs -> watch: frames queued, delivered next microtask +session --> session: C +::: + +Two things follow. A host listener always runs before any watch listener of the same commit: the observers queue the +watch deliveries as microtasks, and the host listeners run synchronously right after them. And nothing is published +until the storage is completely done. + +## The tear + +Chord follows a Durable view through its watch. Two views mean two watches, and each delivers in its own microtask: + +```ts +const status = self.derive([conversationView, taskGraph], ([view, graph]) => ({ + last: view.entries.at(-1)?.kind, + running: graph.tasks.filter((t) => t.status === "running").length, +})); +``` + +```trace +commit 41 (entry + task) + microtask 1: view watch delivers 41 → derive: last "toolCall", running 0 ← never existed + microtask 2: graph watch delivers 41 → derive: last "toolCall", running 1 +``` + +The phone receives both. Chord's rule W1 says a commit changes its sources inside one step and the derives over them +recompute once, right after. For Durable's views that means: they must change inside `adopt`, and Chord must follow +them as sources marked there, not through watches. + +## Why after is not good enough for the store + +In Chord's store, Durable's storage (`StoreStorage`) runs `adopt` inside the store's commit step, and the step lets +the next owner's commit start as soon as `adopt` returned. Durable's `#publish` runs later, after its `await`. In +between, a Chord object's commit can adopt, flush and recompute a derive that reads a Durable view still at 40. The +owner ruled out the alternative, a store that waits for Durable's continuation: no Durable special cases in the store. + +# The First Build {#first lede="Moving half the publication into adopt, and the four things that broke."} + +The first build moved Durable's internal observers into `adopt` and left the rest of `#publish` (host listeners, +telemetry events) after R. To keep host code from waking too early, it added `afterPublish(callback)`: observers queue +what wakes a host (resolving a wait, aborting a run, `onWake`), and the queue runs after the host listeners. + +```ts +const adopt = (seq: Seq): void => { + const documents = tx.adopt(seq); + this.#observeStored(seq, writes, documents); // internal observers, inside A +}; +seq = await this.#storage.commit(writes, adopt, context); +this.#publish(...); // host listeners, after R +this.#settle(); // afterPublish callbacks +``` + +Durable's state (views, the scheduler's idle state, the metric counts) now changes at A. Hosts hear about it at H, +after R. Everything that runs in between sees a commit that is half published. Four reviewers found four ways that +shows, each reproduced with a probe. + +:::steps | The four regressions ++++ Watch listeners before host listeners +The observers call `Watch.advance()` inside A; the watch delivers a microtask later. The storage's own `await` and the +Session's `await` resume later still, so the watch listener runs first. + +```trace +HEAD: A … R → H → watch +first build: A (watch queued) → watch → R → H +``` + +Even with the memory storage. With JSONL the watch listener runs before `Storage.commit` resolved; if the storage then +rejects, the watch has shown a commit whose `commit()` failed. ++++ waitForIdle answers early +The scheduler's idle state is updated in A. A host that calls `waitForIdle()` in the gap gets the fast path ("already +idle") and resumes before H. + +```ts +harness.subscribeCommits((p) => seen.push(p.seq)); +// a task-graph watch listener runs in the gap: +await harness.waitForIdle(ctx); // resolves before seen has the terminal commit +``` ++++ A late subscriber misses the commit +When nothing listened at A, the publication is never built. A host that subscribes during the gap gets nothing for +that commit; at HEAD the listener list was read after R, so it did. ++++ The metric drifts after a rejection +A storage that rejects after A (JSONL's cleanup fails): the scheduler counted the new task in A and queued its `+1` +for the metric; the failure seals the scheduler, which subtracts `-1`; the queued `+1` is dropped with the failed +commit. + +```trace +HEAD: [] (nothing observed, net 0) +first build: [-1] (net -1) +``` +::: + +Fable found three more of the same kind: a host listener that closes the Harness makes resolved waits reject; +scheduler line jobs queued in A jump ahead of a host listener's own commit; abort listeners moved after H. The +patches each reviewer suggested were more `afterPublish`. The owner called it what it is: special-casing. + +# Publication at Adoption {#proposal lede="One rule: a commit is published when it is adopted, all of it, inside adopt."} + +The gap is the bug. The first build created it by splitting the publication. The proposal moves all of it: + +```ts +const adopt = (seq: Seq): void => { + const documents = tx.adopt(seq); // trackers, as today + const publication = this.#publication(seq, writes, documents); + this.#events(writes); // pi.durable.entry / .submission, contained + this.#notifyInternal(publication); // views, graph, scheduler, submissions + this.#notifyHosts(publication); // subscribeCommits +}; +seq = await this.#storage.commit(writes, adopt, context); +// after R: touch documents, sweep, fail() for an error an observer threw; nothing is published here +``` + +The order inside is HEAD's: telemetry events, internal observers, host listeners. `afterPublish` disappears. Waits +need nothing special: an observer resolves a promise in A, and the waiter's continuation runs later by nature, after +the host listeners already ran synchronously. One rule replaces the special cases: `adopt` runs no task code. The +only synchronous path into task code is `AbortController.abort()` of a newly marked run (abort listeners run inside +`abort()`), so that one is queued with `queueMicrotask`. `adopt` never throws: every listener, every report and every +telemetry call inside it is contained. + +:::sequence animate | The same commit under the proposal. Everything happens inside A; R and C only end it. +actor session "Session" "the line" accent +actor storage "Storage" "JSONL" muted +actor obs "internal observers" "views, graph, scheduler" +actor host "host" "subscribeCommits" +actor watch "watch listeners" "microtasks" blue +session -> storage: commit(writes, adopt) +storage -> storage: write, marker durable +storage => session: A: adopt(41) +session -> session: trackers swap, telemetry events +session => obs: observers: views advance, waits resolve +session => host: H: subscribeCommits(41) +obs -> watch: watches, waits, abort: next microtasks +storage -> storage: reclaim sidecars +storage --> session: R +session --> session: C +::: + +What this does to the regressions, verified by Opus and Fable with probes against HEAD: + +| regression | HEAD | first build | proposal | +| --- | --- | --- | --- | +| watch listeners vs H | H, watch | **watch, H** | H, watch | +| `waitForIdle` in the gap | after H | **before H** | after H | +| metric after a rejection following A | 0 | **−1** | 0 (+1, −1) | +| `close()` from H, then `waitForTask` | resolves | **rejects** | resolves | +| scheduler jobs vs H's own commit | H first | **jobs first** | H first | +| stuck state, lost reports | none | yes | none | +| allocations per commit | baseline | more | about HEAD | + +HEAD's whole non-e2e Durable suite (73 files, 1404 tests) passes on a build of the proposal. The four tests the first +build added that encode its split fail, as they should, and are replaced. + +# What Changes for Hosts {#changes lede="Seven things a host, a task or a storage could observe, compared with HEAD."} + +Nothing in the public API changes. These are the differences in behaviour the four checks found. + +## 1. With JSONL, everything runs before the storage is done + +```trace +HEAD: marker → A → cleanup … → R → observers → H → watches, waits → C +proposal: marker → A { observers, H } → watches, waits, abort → cleanup … → R → C +``` + +Host and task code woken by a commit now runs while JSONL still cleans up its sidecars. It cannot reach the storage +through the Session: the line is still held, so a commit or read it starts waits. The memory, SQLite and IndexedDB +storages do no work after A, so for them only the microtask placement differs, and the order is HEAD's. No spec +section promised "after `Storage.commit` resolved". Verdict: harmless; the spec gets one sentence. + +## 2. Abort listeners fire after host listeners + +```ts +runtime.signal.addEventListener("abort", () => log(hostSaw(id))); +// HEAD: false (abort before H) proposal: true (abort after H) +``` + +At HEAD the scheduler aborted a newly marked run inside `#publish`, before the host listeners. Under the proposal the +abort is a microtask after A, so after H. No contract fixes this order. Decision for the owner. + +## 3. A commit the storage rejects after A is published + +A host keeps a replica from `subscribeCommits`. JSONL wrote the marker (the commit is durable), called `adopt`, then +failed while cleaning up and rejected. + +```trace +HEAD: durable, but no observer and no host listener ran → the replica misses it + → replica ≠ storage; waits of that commit reject; the Session fails +proposal: published at A → the replica gets it → replica = storage + → waits of that commit resolve; the Session fails after R +``` + +Durable's spec says a replica fed by `subscribeCommits` "stays equal to Storage"; HEAD breaks that, the proposal keeps +it. Spec invariant 2 changes from "published only after its storage commit succeeds" to "published once it is +durable, even when `commit()` then rejects". Decision for the owner; all four checks recommend it. + +## 4. A subscriber that arrives late misses the commit + +```ts +// host code that runs in the gap, between A and R: +harness.subscribeCommits(listener); // HEAD: gets commit 41; proposal: does not, 41 was published at A +``` + +Consistent with the rule: a listener added after the publication misses it, as one added after `#publish` did at +HEAD. Documented, nothing to decide. + +## 5. An observer that throws fails the Session after the host listeners + +At HEAD, a throwing internal observer failed the Session at once, inside `#publish`: close listeners and the +`session_failed` report came before the host listeners. Under the proposal, `adopt` records the error, lets the other +observers and the host listeners run, and the Session fails after R. The commit stands, as before. Failing inside A +instead would make Durable's storage guard reject the caller of a durable commit with `SessionFailed`. Verdict: keep +it so. + +## 6. Telemetry that throws no longer breaks memory + +At HEAD, `#publish` called `context.telemetry.addEvent` for each entry and submission without protection. A throwing +adapter rejected a commit that was already durable and adopted, skipped every observer and host listener, and left the +Session healthy: the scheduler and the views missed the commit for good. Inside `adopt` nothing may throw, so the +proposal contains it and reports it; the commit resolves. This fixes a HEAD bug. The report needs a type (`host_error` +with callback `"telemetry"` is the suggestion). The same containment must cover Durable's report path itself +(`safeReport` records a metric and a span event outside its `try`), or `adopt` is not truly throw-free. + +## 7. Host listeners would run inside Chord's batch + +This is the one that matters. In Chord's store, `StoreStorage` runs Durable's `adopt` inside the store's commit step, +inside Chord's batch, where Chord refuses reentrant calls (`self.tx`, `get`, `watch`, `close` throw +`invalid/reentrant`). Under the proposal the host listeners run inside `adopt`, so inside that batch: + +```ts +// p3 code that mirrors Durable's commits into a Chord object +harness.subscribeCommits((publication) => + chord((ctx) => audit.record(publication.seq, ctx), context)); // throws invalid/reentrant +``` + +The listener keeps every rule Durable states (synchronous, no Session calls) and still fails. Its error is reported; +an `onReport` that itself uses Chord throws too, and a throwing `onReport` is dropped, so the work vanishes without a +trace. And every host listener's run time is added to the store-wide commit step that every other object's commit +waits for. + +:::flow wide | Under the proposal, a host listener runs inside Chord's batch; Chord calls from it are refused. +row "Store commit step (one at a time for the whole store)" accent +box backend "backend transaction" "`BEGIN … COMMIT`" muted +box adopt "Durable's adopt" "trackers, views, scheduler" accent +box hostin "host listeners" "inside the batch" dashed +row "Chord" +box flush "flush" "derives, wire, watches queued" blue +box refused "`self.tx` from a listener" "`invalid/reentrant`" dashed +arrow backend -> adopt "durable" +arrow adopt -> hostin "proposal" +arrow hostin -> refused "Chord call" +arrow adopt -> flush "after the step" +::: + +# The Microtask Variant {#variant lede="Host listeners one microtask after adopt: same results, outside Chord's batch."} + +Opus proposed a variant that keeps every result of the proposal and moves only the host listeners out of the step. +The publication is still built in A and the internal observers still run there. The host listeners are dispatched in +one microtask that is queued before the internal observers run: + +```ts +// inside adopt, after tx.adopt(seq) +this.#events(writes); // contained +if (this.#commitListeners.size > 0) { + const listeners = [...this.#commitListeners]; + queueMicrotask(() => { + for (const listener of listeners) + contained(() => listener(stored, context), (error) => this.report(error, LISTENER, context)); + }); +} +for (const listener of [...this.#internalListeners]) + contained(() => listener(observed, context), (error) => { failure ??= { error }; }); +``` + +Microtasks run first in, first out. The host dispatch is queued before anything the observers queue, so the host +listeners still run before every watch delivery, every resolved wait, every abort and `onWake`. And it is a +microtask, so it runs after the storage's synchronous step: after Chord's batch and flush, where Chord calls work. + +:::sequence animate | The variant inside Chord's store. The step stays synchronous and short; host listeners run right after it, first of everything the commit woke. +actor store "store step" "one at a time" accent +actor durable "Durable adopt" "views, scheduler" +actor chord "Chord batch" "mark, flush" blue +actor host "host listeners" +actor later "watches, waits, abort" muted +store -> store: backend COMMIT +store => durable: adopt(41): trackers, events +durable -> host: queue H (microtask 1) +durable -> durable: observers: views advance (sources marked) +durable -> later: queue watches, waits, abort (microtasks 2…) +store => chord: flush: derives over both views recompute once +store --> store: step ends, next commit may start +host -> host: H runs: Chord calls allowed +later -> later: then watches, waits, abort +::: + +Chord needs nothing Durable-specific: only that its commit step and its flush are synchronous, which its design +already states. The store never knows Durable's continuation, as the owner required. + +What the variant gives up compared with the proposal: host listeners are no longer called synchronously from inside +`adopt`, but one microtask later, still on the Session line and still before C. Opus ran HEAD's full non-e2e suite +against the variant (1404 of 1404) and all twelve probes: the same results as the proposal. One detail to settle while +building: the listener list is copied at A, so a listener removed between A and the microtask is still called once; +copying it inside the microtask avoids that, but then a listener added in between is called too. + +# Decisions {.epilogue lede="What the owner decides before the rebuild."} + +1. **The variant or the proposal.** The variant (recommended by Opus) runs host listeners one microtask after `adopt`, + outside Chord's batch. The proposal runs them inside `adopt` and needs the documented rule "no Chord calls from + `subscribeCommits` or `onReport`". +2. **Abort listeners after host listeners** (change 2): acceptable, or must the abort keep HEAD's place before them. +3. **Publish a commit the storage rejects after `adopt`** (change 3): recommended by all four checks; it fixes HEAD's + replica gap and changes spec invariant 2. +4. **The report type of a throwing telemetry call** (change 6): suggested `host_error` with callback `"telemetry"`. +5. **HEAD's order inside the publication** (events, internal observers, host listeners) and a fully throw-free + `adopt`, the report path included. These are fixes rather than choices; listed so they are not lost. + +After the decisions: the spec rewrites (invariant 2, section 4 and its pseudo-code, the Storage contract, section 10 +failures), new tests in place of the four that encode the split (order with the memory storage and with a storage that +works after `adopt`, `waitForIdle` in the gap, publication on a rejection with a balanced metric, close in the gap, +Chord-like reentrancy, telemetry throws), a rebuild by a fresh builder, and the four reviewers again. + +# Sources {.appendix} + +| what | where | +| --- | --- | +| HEAD code | `packages/durable/src/session/session.ts` (`#runCommit`, `#publish`), `harness/view.ts`, `harness/task-graph.ts`, `harness/scheduler.ts`, `harness/submissions.ts`, `storage/jsonl/storage.ts` | +| first build | `/home/badlogic/pi-observers` (uncommitted) | +| reviews of the first build | `/tmp/observers-review-{astra,fable,opus,sol}.md` | +| the proposal | `/tmp/observers-proposal.md` | +| checks of the proposal | `/tmp/observers-proposal-{astra,fable,opus,sol}.md`; probes in `/tmp/obsprobe-*` | +| decisions so far | `packages/chord/docs/next/decisions.md`, "Slice 6 first decisions" | +| Chord rules | `packages/chord/docs/next/design.md` §11 (W1, the batch), §24 (Durable integration) | diff --git a/packages/chord/docs/next/durable-observers/index.html b/packages/chord/docs/next/durable-observers/index.html new file mode 100644 index 000000000..fe6961cd5 --- /dev/null +++ b/packages/chord/docs/next/durable-observers/index.html @@ -0,0 +1,517 @@ + + +In the Step: when Durable's views change, and who hears it + + + + +

+
+
An explainer · Chord v2 slice 6 and pi-durable · 2026-10-26
+

In the
Step

+

Why Durable's views must change inside the storage's commit, why the first build broke hosts, and what the "publication at adoption" proposal changes

+
1Durable commit, seen as one change
4regressions the first build had, all in one gap
7things that change for hosts compared with HEAD
5decisions for the owner
+

Written for the owner from the build of /home/badlogic/pi-observers (uncommitted), the four reviews +(/tmp/observers-review-{astra,fable,opus,sol}.md) and the four checks of the proposal +(/tmp/observers-proposal-{astra,fable,opus,sol}.md), whose probes ran against HEAD, the first build and the +proposal. Code is Durable's session/session.ts and harness/*.ts at HEAD, simplified. Nothing here is decided +beyond what the decision log says ("Slice 6 first decisions", refined 2026-10-26).

+
+
+ + +
+

Before we begin

+

Mario's phone shows a p3 conversation. Next to the transcript a small status line reads "last: tool call, 1 task +running". It is a Chord derive over two Durable views: the conversation view (the transcript) and the task graph (the +running tasks).

+

The model asks for a tool. Durable makes one commit, seq 41, that appends the tool-call entry and creates the tool's +task. The phone should go from "last: assistant, 0 running" to "last: tool call, 1 running" in one step. Today it can +show "last: tool call, 0 running" in between: a state that never existed. Slice 6 also puts Durable and Chord into one +store with one commit step, where every owner's memory must show a complete prefix of the log (S1, W1). Both need the +same thing: Durable's views must change inside the storage's commit, not after it.

+

This book explains where Durable publishes a commit today, why moving half of it broke hosts, and what moving all of +it changes.

+

Seven things to know

+
    +
  1. Durable adopts inside the storage, but publishes after it. Storage.commit(writes, adopt, context) calls +adopt once the batch is durable; Durable's views, scheduler and host listeners all run later, after +Storage.commit returned.
    2. +
  2. Two watches tear one commit. A derive over two Durable views sees seq 41 in one view before the other.
    3. +
  3. The first build moved only Durable's internal observers. Host listeners and telemetry stayed after the +storage. Everything a host could see in between came out in the wrong order.
    4. +
  4. All four regressions live in one gap: the time between adopt and the end of Storage.commit.
    5. +
  5. The proposal removes the gap: the whole publication happens inside adopt.
    6. +
  6. It changes seven things a host could observe, most of them timing nobody documented, one a bug fix, one a real +problem for Chord.
    7. +
  7. The real problem has a small fix: host listeners one microtask after adopt, outside Chord's batch (Opus's +variant).
    8. +
+

Words used throughout

+

Durable is pi's agent harness; a Session is its one line of atomic commits. Storage is Durable's storage +interface; Storage.commit(writes, adopt, context) stores a batch and calls adopt(seq) once it is durable. +Durable's trackers hold each loaded document's value in memory. Internal observers are Durable's own commit +listeners (observeCommits): the conversation view mounts, the task graph, the scheduler and the +submissions. Host listeners are what an app registers with subscribeCommits. A watch is a +WatchHandle: its listener runs a microtask after a frame arrives. The gap is the time between adopt and the +moment Storage.commit resolves. In Chord's store, the commit step is the store-wide serialized part of a commit +(backend transaction plus adoption), and the batch is Chord's synchronous adopt, mark and flush around it. +Letters in traces: A = adopt, R = Storage.commit resolves, H = host listeners, C = +session.commit() resolves.

+
+ +
I

Today

Where a Durable commit is adopted and where it is published, at HEAD.

+

A Durable commit runs on the Session's line from start to end. Inside, #runCommit hands the batch to the storage +with an adopt callback, waits for it, and only then publishes:

+
// session.ts, #runCommit, at HEAD (simplified)
+const adopt = (seq: Seq): void => {
+  adoption = { documents: tx.adopt(seq) };          // trackers swap to the new values
+};
+seq = await this.#storage.commit(writes, adopt, context);
+// ... back in the Session, after R:
+this.#publish(seq, writes, documents, context);      // telemetry events, internal observers, host listeners
+return result;                                       // C
+

The SQLite storage calls adopt right after COMMIT returned. JSONL calls it and then still awaits its own cleanup +of sidecar files before it resolves:

+
// jsonl/storage.ts
+adopt(seq);                                           // A
+await this.reclaimSidecars(reclamations, context);   // still inside Storage.commit
+return seq;                                           // R
+
Sessionthe lineStorageJSONLinternal observersviews, graph, schedulerhostsubscribeCommitswatch listenersmicrotaskscommit(writes, adopt)write, marker durableA: adopt(41), trackers swapreclaim sidecarsRtelemetry events, then observers (views advance, waits resolve)H: subscribeCommits(41)frames queued, delivered next microtaskC
Figure 1. One Durable commit at HEAD. The trackers change inside the storage; everything that tells anyone about it happens after the storage returned.
+

Two things follow. A host listener always runs before any watch listener of the same commit: the observers queue the +watch deliveries as microtasks, and the host listeners run synchronously right after them. And nothing is published +until the storage is completely done.

+

The tear

+

Chord follows a Durable view through its watch. Two views mean two watches, and each delivers in its own microtask:

+
const status = self.derive([conversationView, taskGraph], ([view, graph]) => ({
+  last: view.entries.at(-1)?.kind,
+  running: graph.tasks.filter((t) => t.status === "running").length,
+}));
+
commit 41 (entry + task)
+  microtask 1: view watch delivers 41   → derive: last "toolCall", running 0    ← never existed
+  microtask 2: graph watch delivers 41  → derive: last "toolCall", running 1
+

The phone receives both. Chord's rule W1 says a commit changes its sources inside one step and the derives over them +recompute once, right after. For Durable's views that means: they must change inside adopt, and Chord must follow +them as sources marked there, not through watches.

+

Why after is not good enough for the store

+

In Chord's store, Durable's storage (StoreStorage) runs adopt inside the store's commit step, and the step lets +the next owner's commit start as soon as adopt returned. Durable's #publish runs later, after its await. In +between, a Chord object's commit can adopt, flush and recompute a derive that reads a Durable view still at 40. The +owner ruled out the alternative, a store that waits for Durable's continuation: no Durable special cases in the store.

+
+ +
II

The First Build

Moving half the publication into adopt, and the four things that broke.

+

The first build moved Durable's internal observers into adopt and left the rest of #publish (host listeners, +telemetry events) after R. To keep host code from waking too early, it added afterPublish(callback): observers queue +what wakes a host (resolving a wait, aborting a run, onWake), and the queue runs after the host listeners.

+
const adopt = (seq: Seq): void => {
+  const documents = tx.adopt(seq);
+  this.#observeStored(seq, writes, documents);       // internal observers, inside A
+};
+seq = await this.#storage.commit(writes, adopt, context);
+this.#publish(...);                                    // host listeners, after R
+this.#settle();                                        // afterPublish callbacks
+

Durable's state (views, the scheduler's idle state, the metric counts) now changes at A. Hosts hear about it at H, +after R. Everything that runs in between sees a commit that is half published. Four reviewers found four ways that +shows, each reproduced with a probe.

+
The four regressions

Watch listeners before host listeners

The observers call Watch.advance() inside A; the watch delivers a microtask later. The storage's own await and the +Session's await resume later still, so the watch listener runs first.

+
HEAD:        A … R → H → watch
+first build: A (watch queued) → watch → R → H
+

Even with the memory storage. With JSONL the watch listener runs before Storage.commit resolved; if the storage then +rejects, the watch has shown a commit whose commit() failed.

+

waitForIdle answers early

The scheduler's idle state is updated in A. A host that calls waitForIdle() in the gap gets the fast path ("already +idle") and resumes before H.

+
harness.subscribeCommits((p) => seen.push(p.seq));
+// a task-graph watch listener runs in the gap:
+await harness.waitForIdle(ctx);          // resolves before seen has the terminal commit
+

A late subscriber misses the commit

When nothing listened at A, the publication is never built. A host that subscribes during the gap gets nothing for +that commit; at HEAD the listener list was read after R, so it did.

+

The metric drifts after a rejection

A storage that rejects after A (JSONL's cleanup fails): the scheduler counted the new task in A and queued its +1 +for the metric; the failure seals the scheduler, which subtracts -1; the queued +1 is dropped with the failed +commit.

+
HEAD:        []        (nothing observed, net 0)
+first build: [-1]      (net -1)
+
+

Fable found three more of the same kind: a host listener that closes the Harness makes resolved waits reject; +scheduler line jobs queued in A jump ahead of a host listener's own commit; abort listeners moved after H. The +patches each reviewer suggested were more afterPublish. The owner called it what it is: special-casing.

+
+ +
III

Publication at Adoption

One rule: a commit is published when it is adopted, all of it, inside adopt.

+

The gap is the bug. The first build created it by splitting the publication. The proposal moves all of it:

+
const adopt = (seq: Seq): void => {
+  const documents = tx.adopt(seq);                              // trackers, as today
+  const publication = this.#publication(seq, writes, documents);
+  this.#events(writes);                                         // pi.durable.entry / .submission, contained
+  this.#notifyInternal(publication);                            // views, graph, scheduler, submissions
+  this.#notifyHosts(publication);                               // subscribeCommits
+};
+seq = await this.#storage.commit(writes, adopt, context);
+// after R: touch documents, sweep, fail() for an error an observer threw; nothing is published here
+

The order inside is HEAD's: telemetry events, internal observers, host listeners. afterPublish disappears. Waits +need nothing special: an observer resolves a promise in A, and the waiter's continuation runs later by nature, after +the host listeners already ran synchronously. One rule replaces the special cases: adopt runs no task code. The +only synchronous path into task code is AbortController.abort() of a newly marked run (abort listeners run inside +abort()), so that one is queued with queueMicrotask. adopt never throws: every listener, every report and every +telemetry call inside it is contained.

+
Sessionthe lineStorageJSONLinternal observersviews, graph, schedulerhostsubscribeCommitswatch listenersmicrotaskscommit(writes, adopt)write, marker durableA: adopt(41)trackers swap, telemetry eventsobservers: views advance, waits resolveH: subscribeCommits(41)watches, waits, abort: next microtasksreclaim sidecarsRC
Figure 2. The same commit under the proposal. Everything happens inside A; R and C only end it.
+

What this does to the regressions, verified by Opus and Fable with probes against HEAD:

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
regressionHEADfirst buildproposal
watch listeners vs HH, watchwatch, HH, watch
waitForIdle in the gapafter Hbefore Hafter H
metric after a rejection following A0−10 (+1, −1)
close() from H, then waitForTaskresolvesrejectsresolves
scheduler jobs vs H's own commitH firstjobs firstH first
stuck state, lost reportsnoneyesnone
allocations per commitbaselinemoreabout HEAD
+

HEAD's whole non-e2e Durable suite (73 files, 1404 tests) passes on a build of the proposal. The four tests the first +build added that encode its split fail, as they should, and are replaced.

+ + +
IV

What Changes for Hosts

Seven things a host, a task or a storage could observe, compared with HEAD.

+

Nothing in the public API changes. These are the differences in behaviour the four checks found.

+

1. With JSONL, everything runs before the storage is done

+
HEAD:     marker → A → cleanup … → R → observers → H → watches, waits → C
+proposal: marker → A { observers, H } → watches, waits, abort → cleanup … → R → C
+

Host and task code woken by a commit now runs while JSONL still cleans up its sidecars. It cannot reach the storage +through the Session: the line is still held, so a commit or read it starts waits. The memory, SQLite and IndexedDB +storages do no work after A, so for them only the microtask placement differs, and the order is HEAD's. No spec +section promised "after Storage.commit resolved". Verdict: harmless; the spec gets one sentence.

+

2. Abort listeners fire after host listeners

+
runtime.signal.addEventListener("abort", () => log(hostSaw(id)));
+// HEAD: false (abort before H)    proposal: true (abort after H)
+

At HEAD the scheduler aborted a newly marked run inside #publish, before the host listeners. Under the proposal the +abort is a microtask after A, so after H. No contract fixes this order. Decision for the owner.

+

3. A commit the storage rejects after A is published

+

A host keeps a replica from subscribeCommits. JSONL wrote the marker (the commit is durable), called adopt, then +failed while cleaning up and rejected.

+
HEAD:     durable, but no observer and no host listener ran → the replica misses it
+          → replica ≠ storage; waits of that commit reject; the Session fails
+proposal: published at A → the replica gets it → replica = storage
+          → waits of that commit resolve; the Session fails after R
+

Durable's spec says a replica fed by subscribeCommits "stays equal to Storage"; HEAD breaks that, the proposal keeps +it. Spec invariant 2 changes from "published only after its storage commit succeeds" to "published once it is +durable, even when commit() then rejects". Decision for the owner; all four checks recommend it.

+

4. A subscriber that arrives late misses the commit

+
// host code that runs in the gap, between A and R:
+harness.subscribeCommits(listener);   // HEAD: gets commit 41; proposal: does not, 41 was published at A
+

Consistent with the rule: a listener added after the publication misses it, as one added after #publish did at +HEAD. Documented, nothing to decide.

+

5. An observer that throws fails the Session after the host listeners

+

At HEAD, a throwing internal observer failed the Session at once, inside #publish: close listeners and the +session_failed report came before the host listeners. Under the proposal, adopt records the error, lets the other +observers and the host listeners run, and the Session fails after R. The commit stands, as before. Failing inside A +instead would make Durable's storage guard reject the caller of a durable commit with SessionFailed. Verdict: keep +it so.

+

6. Telemetry that throws no longer breaks memory

+

At HEAD, #publish called context.telemetry.addEvent for each entry and submission without protection. A throwing +adapter rejected a commit that was already durable and adopted, skipped every observer and host listener, and left the +Session healthy: the scheduler and the views missed the commit for good. Inside adopt nothing may throw, so the +proposal contains it and reports it; the commit resolves. This fixes a HEAD bug. The report needs a type (host_error +with callback "telemetry" is the suggestion). The same containment must cover Durable's report path itself +(safeReport records a metric and a span event outside its try), or adopt is not truly throw-free.

+

7. Host listeners would run inside Chord's batch

+

This is the one that matters. In Chord's store, StoreStorage runs Durable's adopt inside the store's commit step, +inside Chord's batch, where Chord refuses reentrant calls (self.tx, get, watch, close throw +invalid/reentrant). Under the proposal the host listeners run inside adopt, so inside that batch:

+
// p3 code that mirrors Durable's commits into a Chord object
+harness.subscribeCommits((publication) =>
+  chord((ctx) => audit.record(publication.seq, ctx), context));   // throws invalid/reentrant
+

The listener keeps every rule Durable states (synchronous, no Session calls) and still fails. Its error is reported; +an onReport that itself uses Chord throws too, and a throwing onReport is dropped, so the work vanishes without a +trace. And every host listener's run time is added to the store-wide commit step that every other object's commit +waits for.

+
STORE COMMIT STEP (ONE AT A TIME FOR THE WHOLE STORE)CHORDbackend transactionBEGIN … COMMITDurable's adopttrackers, views, schedulerhost listenersinside the batchflushderives, wire, watches queued`self.tx` from a listenerinvalid/reentrantdurableproposalChord callafter the step
Figure 3. Under the proposal, a host listener runs inside Chord's batch; Chord calls from it are refused.
+
+ +
V

The Microtask Variant

Host listeners one microtask after adopt: same results, outside Chord's batch.

+

Opus proposed a variant that keeps every result of the proposal and moves only the host listeners out of the step. +The publication is still built in A and the internal observers still run there. The host listeners are dispatched in +one microtask that is queued before the internal observers run:

+
// inside adopt, after tx.adopt(seq)
+this.#events(writes);                                            // contained
+if (this.#commitListeners.size > 0) {
+  const listeners = [...this.#commitListeners];
+  queueMicrotask(() => {
+    for (const listener of listeners)
+      contained(() => listener(stored, context), (error) => this.report(error, LISTENER, context));
+  });
+}
+for (const listener of [...this.#internalListeners])
+  contained(() => listener(observed, context), (error) => { failure ??= { error }; });
+

Microtasks run first in, first out. The host dispatch is queued before anything the observers queue, so the host +listeners still run before every watch delivery, every resolved wait, every abort and onWake. And it is a +microtask, so it runs after the storage's synchronous step: after Chord's batch and flush, where Chord calls work.

+
store stepone at a timeDurable adoptviews, schedulerChord batchmark, flushhost listenerswatches, waits, abortbackend COMMITadopt(41): trackers, eventsqueue H (microtask 1)observers: views advance (sources marked)queue watches, waits, abort (microtasks 2…)flush: derives over both views recompute oncestep ends, next commit may startH runs: Chord calls allowedthen watches, waits, abort
Figure 4. The variant inside Chord's store. The step stays synchronous and short; host listeners run right after it, first of everything the commit woke.
+

Chord needs nothing Durable-specific: only that its commit step and its flush are synchronous, which its design +already states. The store never knows Durable's continuation, as the owner required.

+

What the variant gives up compared with the proposal: host listeners are no longer called synchronously from inside +adopt, but one microtask later, still on the Session line and still before C. Opus ran HEAD's full non-e2e suite +against the variant (1404 of 1404) and all twelve probes: the same results as the proposal. One detail to settle while +building: the listener list is copied at A, so a listener removed between A and the microtask is still called once; +copying it inside the microtask avoids that, but then a listener added in between is called too.

+
+ +
∴

Decisions

What the owner decides before the rebuild.

+
    +
  1. The variant or the proposal. The variant (recommended by Opus) runs host listeners one microtask after adopt, +outside Chord's batch. The proposal runs them inside adopt and needs the documented rule "no Chord calls from +subscribeCommits or onReport".
    2. +
  2. Abort listeners after host listeners (change 2): acceptable, or must the abort keep HEAD's place before them.
    3. +
  3. Publish a commit the storage rejects after adopt (change 3): recommended by all four checks; it fixes HEAD's +replica gap and changes spec invariant 2.
    4. +
  4. The report type of a throwing telemetry call (change 6): suggested host_error with callback "telemetry".
    5. +
  5. HEAD's order inside the publication (events, internal observers, host listeners) and a fully throw-free +adopt, the report path included. These are fixes rather than choices; listed so they are not lost.
    6. +
+

After the decisions: the spec rewrites (invariant 2, section 4 and its pseudo-code, the Storage contract, section 10 +failures), new tests in place of the four that encode the split (order with the memory storage and with a storage that +works after adopt, waitForIdle in the gap, publication on a rejection with a balanced metric, close in the gap, +Chord-like reentrancy, telemetry throws), a rebuild by a fresh builder, and the four reviewers again.

+
+ +
A

Sources

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
whatwhere
HEAD codepackages/durable/src/session/session.ts (#runCommit, #publish), harness/view.ts, harness/task-graph.ts, harness/scheduler.ts, harness/submissions.ts, storage/jsonl/storage.ts
first build/home/badlogic/pi-observers (uncommitted)
reviews of the first build/tmp/observers-review-{astra,fable,opus,sol}.md
the proposal/tmp/observers-proposal.md
checks of the proposal/tmp/observers-proposal-{astra,fable,opus,sol}.md; probes in /tmp/obsprobe-*
decisions so farpackages/chord/docs/next/decisions.md, "Slice 6 first decisions"
Chord rulespackages/chord/docs/next/design.md §11 (W1, the batch), §24 (Durable integration)
+
+ +

In the Step · first edition · 2026-10-26 · from the observers build, its reviews and the proposal checks

+ + + diff --git a/packages/chord/docs/next/flow-control/book.md b/packages/chord/docs/next/flow-control/book.md new file mode 100644 index 000000000..0485fe6a0 --- /dev/null +++ b/packages/chord/docs/next/flow-control/book.md @@ -0,0 +1,333 @@ +--- +title: Flow control for Chord links +cover: Slow / Links +kicker: A design note · Chord v2 slice 4 · 2026-10-22 +subtitle: How a phone on a bad train connection keeps seeing the newest state of a streamed reply, and why the transport's own backpressure cannot do that +stats: + - 9.4 s | how old the newest state was with transport backpressure only + - 0.0 s | with an acknowledged window of two messages + - 345 ms | another client's p95 during a call flood, without a call window + - ~200 | lines for the converged design +note: | + Written from the measurements and the cross-checked reports of the reviewers Astra and Fable (2026-10-22): Node 24.4.1, + `ws` 8.22, separate server and client processes, a throttling proxy at 64 KiB/s. The numbers are single synthetic + runs: they show failure modes, not production capacity. Relays: first decided as per-hop acknowledgements (b) on + 2026-10-23, then replaced by flow control per origin on 2026-10-24 after the slice 4a benchmarks (chapter VI). +colophon: Flow control for Chord links · fourth edition · 2026-10-24 · based on the reviewers' experiments and design §3, §22 +--- + +# Before we begin {.preface} + +Mario is on a train. His phone shows a p3 session in which the agent is writing a long reply. Every token is a small +state change: the worker commits it, the server relays it, the phone applies it. The train enters a tunnel and the +phone's link drops to a trickle, 64 KB per second. The worker keeps producing tokens at full speed. + +Without flow control, every token still gets sent. The bytes pile up somewhere between the worker and the phone, and +when the train leaves the tunnel the phone has to work through a backlog of old tokens before it sees the current text. +It is not only slow: what it shows is minutes old, and nothing tells it so. + +This note explains what keeps the phone current, why the obvious answer (let the transport push back) fails, and the +design Astra and Fable converged on after measuring it. + +## Six things to know + +:::findings +1. **Transport backpressure arrives too late.** The kernel's and a proxy's buffers take megabytes before a WebSocket + reports anything buffered; the phone sees state 4 to 10 seconds old. +2. **Acknowledgements keep state fresh.** The receiving application acknowledges each message; at most two are in + flight; the newest state was 0.0 seconds old on the same 64 KB/s link. +3. **Count messages, not bytes.** Each message carries everything that piled up since the last acknowledgement, so + throughput adapts to the link by itself. +4. **Merge ops first, snapshot last.** 2,000 merged token appends cost 26 KB, sent one by one 46 KB; a snapshot of a + member with 500 KB of history would cost 508 KB. +5. **Calls need their own window.** A tab firing 100,000 calls pushed another client's p95 to 345 ms; with a window of + 64 respected by the caller it stayed at 0.66 ms and the flood waited in the buggy tab. +6. **Nothing is refused for being too many** (decision K16): extra state is merged, extra calls wait. +::: + +## Words used throughout + +A **link** connects two hosts. The **provider** owns an object and sends its state; the **consumer** holds a +reference to it. A **hold** is one reference over a link, with its own stream of **ops** (Delta's changes). A +**snapshot** is a hold's whole state. An **acknowledgement** (`ack`) is a tiny message the consumer sends after it +applied a state message. **In flight** means sent and not yet acknowledged. **Coalescing** means merging the ops that +wait to be sent. A **relay** is a host in the middle (the server between worker and tab). + +# Where the Bytes Hide {#hidden lede="Why the sender cannot see a slow phone."} + +A WebSocket in Node reports `bufferedAmount`: bytes the library still holds. It looks like the perfect signal: stop +sending when it grows. The experiment says otherwise. + +:::sequence animate | A flood of state messages to a slow phone through a proxy. The sender's buffer stays small; the kernel's and the proxy's do not. +actor worker "worker" "commits tokens" accent +actor ws "server WebSocket" "bufferedAmount" +actor kernel "kernel + proxy buffers" muted +actor phone "phone" "64 KB/s" blue +worker -> ws: token 1 … token 9,000 +ws -> kernel: written at once (bufferedAmount stays near 0) +kernel -> phone: drains at 64 KB/s +note kernel: megabytes waiting +worker -> ws: token 9,001 (the newest) +ws -> kernel: queued behind everything +kernel --> phone: arrives seconds later +::: + +| Flood, ~9,800 × 128 B per second, 64 KB/s link | Sender memory | Newest state the phone saw | +| --- | --- | --- | +| no flow control | 7.0 MB queued in the WebSocket, heap +18.5 MB | 9.4 s old | +| cap `bufferedAmount` at 64 KiB, merge pending state | +0.46 MB | **still 9.4 s old** | +| cap at 1 MiB (Fable's rerun) | small | 3.8 s, growing to 10.4 s | +| at most 1 to 4 acknowledged messages in flight, merge pending state | +0.4 MB | **0.0 s** | + +The transport's backpressure protects the sender's memory, but freshness depends on buffers the sender cannot see: +the operating system's socket buffer, autotuned to hundreds of kilobytes, and every proxy on the way. A normal +streamed reply (50 tokens per second, 23 bytes each) never fills them; a burst or a tunnel does. Only the receiving +application knows what it has applied. + +# Acknowledged Messages {#acks lede="A window of two messages in flight, and why it adapts to any link."} + +The rule: a provider has at most **K = 2** state messages in flight per link. The consumer sends `ack` (three bytes in +the binary codec) after it applied one. While two are in flight, new commits do not produce messages; their ops wait, +merged per hold, and go out in the next message once an `ack` arrives. + +:::steps | Two messages in flight on a slow link ++++ Fast link +Commits 1, 2, 3 each become a message; each is acknowledged before the next commit. Nothing waits: the window never +fills, so a normal stream behaves exactly as without flow control. ++++ The link slows +Messages 1 and 2 are in flight. Commits 3 to 400 arrive; their ops are folded into the pending queue of each member +they touch (chapter III). Memory: the net change, not the history; at most 10,000 ops per member. ++++ An ack arrives +`ack` for message 1. The provider sends message 3, carrying the folded ops of commits 3 to 400, encoded once. +The phone applies them as one batch and shows the newest text. ++++ Self-clocking +The slower the link, the more each message carries; the faster, the smaller and more frequent. Nobody measures +bandwidth: the acknowledgements pace the sender. +::: + +:::raw +
+
+ + + +
+ +
+
+ +::: + +The simulation above shows the same effect in miniature. Drag the link below the commit rate: the upper lane's queue +grows without bound and the phone falls further behind every second; the lower lane never has more than two messages +waiting, and the phone is at most a message behind. + +# Fold the Backlog {#merge lede="What a waiting hold keeps while the window is full, and why it never measures sizes."} + +While messages are in flight, each hold member keeps the ops of new commits in a **pending queue** and **folds** them: +a new op merges into the last pending op on the same path when that is safe, so a backlog keeps the net change, not +the history. When an acknowledgement frees a slot, the queue is encoded once into one message. + +| Last pending op | New op | Kept | +| --- | --- | --- | +| `a text "Hel"` | `a text "lo"` | one `a text "Hello"` | +| `s count 41` or an append | `s count 42` | the `s`: it overwrites what came before | +| anything | `r value` | only the `r`: a replacement makes everything before it moot | +| anything on a path in the index | a structural op (`p`, `m`, `d`, `t`, `s` of an object) | appended; the index is cleared | + +The merge partner is found through a **leaf index**: a map from path to the last pending append or primitive set on +that path. Leaf ops on different paths commute, so a new op may merge into an earlier one, not only into its +neighbour; a status field changing between two tokens no longer stops the reply from folding. Any structural op +clears the index, because after a splice `entries[7]` may name another entry. Past 10,000 pending ops the queue keeps +only "replace with the current value", which then costs nothing while the consumer stays slow. + +**Why no size rule.** The first idea was to send a snapshot once the merged ops outgrow the member. The reviewers +measured it on real p2 data and it failed three ways: "snapshot bytes plus sent op bytes" is no bound (an interned path +lets a value grow faster than its ops: 5,404 B of snapshot after 832 B of ops), it drifts 1,254 times too high under +churn, and an exact size needs a size on every node of the value. Folding alone came within about 10% of an exact +oracle, with no measuring at all. + +| 60 s stall, all workloads | Uncompressed | Compressed | +| --- | --- | --- | +| merged ops, no folding | 466 KB | 166 KB | +| adjacent folding | 399 KB | 147 KB | +| leaf index (the decision) | 372 KB | 145 KB | +| exact oracle (not buildable) | 335 KB | 136 KB | + +**One queue for every slow consumer.** The same queue serves Chord's shared watch handle: a listener still busy with +the last frame gets the commits made meanwhile as one call with the newest value and the folded ops, instead of +today's "after 100 frames, replace everything". With a listener taking one second, a conversation view's watch went +from 402 calls and 17.9 MB of ops to 312 calls and 257 KB. Durable's event stream keeps a queue of its own, because +events are not state: folding would drop them. + +:::raw +

Animation of the fold: How folding works.

+::: + +# Calls Get Their Own Window {#calls lede="Why acknowledged state does not protect against a call flood."} + +Calls are small and the provider acknowledges nothing until they finish, so a byte or message window on state says +nothing about how many handlers run. A tab with a bug: + +```ts +for (const note of notes) ref.add(note, ctx); // 100,000 calls, none awaited +``` + +| 100,000 calls, handlers awaiting 10 ms | Finished in | Provider memory | Another client's p95 round trip | +| --- | --- | --- | --- | +| no limit | 0.66 s | +56.5 MB, 43,824 running | 129 ms (345 ms with trivial handlers) | +| provider-side semaphore of 64 | 15.9 s | +42.5 MB, 96,736 queued on the provider | 0.38 ms | +| caller-side window of 64 | 17.1 s | **+4 MB, nothing queued on the provider** | 0.66 ms | +| handlers doing 10 ms of CPU work | | | blocked about 1 s, with or without a limit | + +The provider's `hello` announces a call window **N** (256 by default) per origin (chapter VI). A consumer keeps at +most N of its own calls unanswered per link; further calls wait in the link's call queue, in order. Nothing is refused (K16); a peer that sends +more than N is a protocol error. The flood waits in the tab that caused it. CPU-heavy handlers block the event loop +whatever the window: that stays the handler author's responsibility. + +# Relays {#relays lede="One link to a worker carries many tabs. Decided: flow control per origin."} + +The server relays every tab's holds to a session's worker over one link. Three ways to apply the acknowledged window +across that hop were considered: + +:::flow wide animate | (a) End-to-end acknowledgements: the server acknowledges the worker only after the tab did. +row "Worker" accent +box w "worker" "K = 2 per link" +row "Server" muted +box s "server" "acks upstream after downstream" +row "Tabs" blue +box fast "fast tab" "desktop" +box slow "slow tab" "phone in a tunnel" dashed +arrow w -> s "state" +arrow s -> fast "state" +arrow s -> slow "state, slow" +arrow slow -> s "ack (late)" dashed +arrow s -> w "ack only when every tab acked" accent +note "Simplest, but the slow phone holds the worker's one window, so the fast tab's updates arrive in coarser, later batches too." +::: + +:::flow wide animate | (b) Acknowledgements per hop: the server acknowledges at once and resets a slow tab. +row "Worker" accent +box w2 "worker" "K = 2 per link" +row "Server" muted +box s2 "server" "acks at once; window per tab" +row "Tabs" blue +box fast2 "fast tab" "desktop" +box slow2 "slow tab" "phone in a tunnel" dashed +arrow w2 -> s2 "state" +arrow s2 -> w2 "ack at once" accent +arrow s2 -> fast2 "state, current" +arrow s2 -> slow2 "window full: its parts dropped" dashed +arrow s2 -> w2 "reset: fresh snapshots" accent +note "The server cannot fold opaque parts, so it drops them and asks for snapshots. Built and measured in slice 4a: a 1 s stall cost three full snapshots (~1.5 MiB) and ~24 s to catch up under streaming." +::: + +:::flow wide animate | (c) Flow control per origin (decided 2026-10-24): the worker keeps a window per tab; the relay only forwards. +row "Worker" accent +box w3 "worker" "K = 2 per (link, origin)" +row "Server" muted +box s3 "server" "forwards; FIFO of senders" +row "Tabs" blue +box fast3 "fast tab" "origin 1" +box slow3 "slow tab" "origin 2, in a tunnel" dashed +arrow w3 -> s3 "state @1, state @2" +arrow s3 -> fast3 "state, current" +arrow s3 -> slow3 "state, slow" dashed +arrow slow3 -> s3 "ack (late)" dashed +arrow s3 -> w3 "ack @2 when the tab acked" accent +note "The relay numbers each tab (an origin) and forwards at once; each tab's ack goes back to the worker that sent the message. The worker folds the slow tab's commits in that origin's pending queues, exactly as for a direct tab; the fast tab is another origin with its own window." +::: + +How (c) works: + +- **Origins.** The relay gives each downstream link a number, never reused; every message upstream carries it + (`origin`), and the worker keys holds, calls, windows and acks by (link, origin, id). Ids are not renumbered. +- **What the relay keeps.** Per relayed hold its upstream link and its original `hold` message (to route calls and + hold again after a worker restart); per tab a FIFO of which worker sent each message in flight, so each in-order + `ack` goes back to the right worker. Nothing is parsed, folded, dropped or reset. +- **Calls.** N is per origin at the worker; the relay forwards calls at once, each tab bounded by its own window. +- **A tab leaves.** Its link closes; the relay sends `closed { origin }` to each worker it relayed to, which releases + that origin's holds and aborts its calls. +- **Cost.** A tab served by W workers may have 2W messages in flight (plus the relay's own 2). Two bytes of origin per + message on the worker link. + +# What Changes in the Design {.epilogue lede="The converged scheme, what it replaces, and the relay decision."} + +**Converged** (Astra and Fable, after cross-checking each other's experiments): at most K = 2 acknowledged state +messages in flight; pending ops folded per hold member through a leaf index, capped at 10,000; a call window N +announced in `hello`, extra calls queued by the caller; holds unlimited; size limits stay a separate check. + +**Replaced** from the design (§3, §22): the byte window and its credit messages, snapshot admission and debt (R19, Q3), +cut C12's call-credit accounting, any adapter backlog API. IPC and WebSocket behave the same, because acknowledgements +are messages. + +**Relays (owner):** per-hop acknowledgements (b) were decided on 2026-10-23 and built in slice 4a; the benchmarks +showed a slow tab recovering through resets far worse than a direct tab (24 s and three snapshots after a 1 s stall). +On 2026-10-24 they were replaced by **flow control per origin** (c): the worker keeps window, pending queues and calls +per (link, origin), the relay only forwards and returns acks. Measured after the rework: a slow tab through the relay +catches up exactly like a direct one. The circuit breaker stayed undecided; the heartbeat (15 s ping, 30 s close) +was built instead. + +# Measurements {.appendix} + +| Experiment | Setup | Result | +| --- | --- | --- | +| normal stream | 50 × 23 B/s through 64 KiB/s | no buffering, frames ≤ 2 ms old | +| flood, no control | ~9,800 × 128 B/s, 10 s | 7.0 MB in `bufferedAmount`, heap +18.5 MB, newest state 9.4 s old | +| flood, `bufferedAmount` ≤ 64 KiB + merging | same | heap +0.46 MB, newest state 9.4 s old | +| flood, 1 to 4 acked messages + merging | same | heap +0.4 MB, newest state 0.0 s old, 98 % fewer messages | +| client stops reading (loopback) | 20k × 90 B/s, 15 s | 23.4 MiB buffered, RSS +121 MB; merging above 1 MiB: flat | +| IPC to a busy child | 623k sends | `send()` false from ~64 KB, parent +1,035 MB; the callback acknowledges the write, not the handler | +| call flood | 100,000 calls, see chapter IV | caller-side window: +4 MB, other client p95 0.66 ms | +| slow tab through a relay, (b) resets | 64 KiB/s, 100 tokens/s into a 500 KB view, 1 s stall | ~24 s to catch up, three full snapshots (~1.5 MiB) | +| slow tab through a relay, (c) per origin | same, 1 s / 10 s / 60 s stall | caught up after 0.02 / 0.08 / 0.38 s, 485 B / 4.1 KiB / 23.7 KiB (same as direct) | diff --git a/packages/chord/docs/next/flow-control/fold/index.html b/packages/chord/docs/next/flow-control/fold/index.html new file mode 100644 index 000000000..3cb573bb5 --- /dev/null +++ b/packages/chord/docs/next/flow-control/fold/index.html @@ -0,0 +1,265 @@ + + + + + +How folding works + + + +
+

How folding works

+

What a hold does with commits while its link's window is full (two messages unacknowledged), and why folding needs no size measuring.

+ +

The rule

+

While the window is full, a hold keeps the ops of each new commit raw in a pending list. Before appending an op, it looks at the last pending op only:

+ + + + + +
last pendingnew opresult
a path "Hel"a path "lo"one a path "Hello" (text joined)
s path 41 or a path …s path 42one s path 42 (the set overwrites what came before)
anything on another pathanythingappended as it is
+

Applied in order, the folded list gives the same value as the unfolded one: only neighbours on the same path merge, so no op in between can see the difference. Cost: one path comparison per op. When an ack frees a slot, the pending list is encoded once into one message.

+
a = append texts = set valuep = insert into listsnapshot
+ +
+
+ + + + +
+
+ + + + +
+
+
link
window full: 2 unacked (tunnel)
+
commits
+
pending (raw)
+
sent
+
+
+ commits 0 + pending ops 0 + merged or dropped 0 + encodes 0 +
+
+
+ +

What the four cases show

+ + + + + + +
casepending after the stallwhy
streamed reply1 opevery token appends to the same path
counter, 100,000 sets1 opeach set overwrites the previous one
reply + status, interleavedgrowsthe neighbour is always on the other path: adjacent-only folding cannot merge
new list itemsgrowsinserts carry new data: nothing to fold, this is the real change
+

The last two cases are why a memory bound is still needed. It can be a plain count of pending ops after folding (say 1,000): past it the hold drops its list and marks “send the current value”, which then costs nothing more while the stall lasts.

+ +

Why not time since the last ack?

+

A time trigger (“no ack for T seconds: drop the list, send a snapshot later”) punishes the most common case: a phone in a tunnel while a reply streams into a long conversation. Folded, the 60 s stall costs one append of about 12 KB; a snapshot of the conversation costs its whole size (Astra's corpus: 4.6 KB of updates against 168 KB, recovery 0.2 s against 2.7 s). And before T, a fast producer on many paths still piles up rate × T ops: time bounds duration, not memory. With folding, the counter case (100,000 sets) is one op whatever the time, so time adds nothing there.

+
+ + + + diff --git a/packages/chord/docs/next/flow-control/index.html b/packages/chord/docs/next/flow-control/index.html new file mode 100644 index 000000000..a34b39a24 --- /dev/null +++ b/packages/chord/docs/next/flow-control/index.html @@ -0,0 +1,548 @@ + + +Flow control for Chord links + + + + +
+
+
A design note · Chord v2 slice 4 · 2026-10-22
+

Slow
Links

+

How a phone on a bad train connection keeps seeing the newest state of a streamed reply, and why the transport's own backpressure cannot do that

+
9.4 show old the newest state was with transport backpressure only
0.0 swith an acknowledged window of two messages
345 msanother client's p95 during a call flood, without a call window
~200lines for the converged design
+

Written from the measurements and the cross-checked reports of the reviewers Astra and Fable (2026-10-22): Node 24.4.1, +ws 8.22, separate server and client processes, a throttling proxy at 64 KiB/s. The numbers are single synthetic +runs: they show failure modes, not production capacity. Relays: first decided as per-hop acknowledgements (b) on +2026-10-23, then replaced by flow control per origin on 2026-10-24 after the slice 4a benchmarks (chapter VI).

+
+
+ + +
+

Before we begin

+

Mario is on a train. His phone shows a p3 session in which the agent is writing a long reply. Every token is a small +state change: the worker commits it, the server relays it, the phone applies it. The train enters a tunnel and the +phone's link drops to a trickle, 64 KB per second. The worker keeps producing tokens at full speed.

+

Without flow control, every token still gets sent. The bytes pile up somewhere between the worker and the phone, and +when the train leaves the tunnel the phone has to work through a backlog of old tokens before it sees the current text. +It is not only slow: what it shows is minutes old, and nothing tells it so.

+

This note explains what keeps the phone current, why the obvious answer (let the transport push back) fails, and the +design Astra and Fable converged on after measuring it.

+

Six things to know

+
    +
  1. Transport backpressure arrives too late. The kernel's and a proxy's buffers take megabytes before a WebSocket +reports anything buffered; the phone sees state 4 to 10 seconds old.
    2. +
  2. Acknowledgements keep state fresh. The receiving application acknowledges each message; at most two are in +flight; the newest state was 0.0 seconds old on the same 64 KB/s link.
    3. +
  3. Count messages, not bytes. Each message carries everything that piled up since the last acknowledgement, so +throughput adapts to the link by itself.
    4. +
  4. Merge ops first, snapshot last. 2,000 merged token appends cost 26 KB, sent one by one 46 KB; a snapshot of a +member with 500 KB of history would cost 508 KB.
    5. +
  5. Calls need their own window. A tab firing 100,000 calls pushed another client's p95 to 345 ms; with a window of +64 respected by the caller it stayed at 0.66 ms and the flood waited in the buggy tab.
    6. +
  6. Nothing is refused for being too many (decision K16): extra state is merged, extra calls wait.
    7. +
+

Words used throughout

+

A link connects two hosts. The provider owns an object and sends its state; the consumer holds a +reference to it. A hold is one reference over a link, with its own stream of ops (Delta's changes). A +snapshot is a hold's whole state. An acknowledgement (ack) is a tiny message the consumer sends after it +applied a state message. In flight means sent and not yet acknowledged. Coalescing means merging the ops that +wait to be sent. A relay is a host in the middle (the server between worker and tab).

+
+ +
I

Where the Bytes Hide

Why the sender cannot see a slow phone.

+

A WebSocket in Node reports bufferedAmount: bytes the library still holds. It looks like the perfect signal: stop +sending when it grows. The experiment says otherwise.

+
workercommits tokensserver WebSocketbufferedAmountkernel + proxy buffersphone64 KB/stoken 1 … token 9,000written at once (bufferedAmount stays near 0)drains at 64 KB/smegabytes waitingtoken 9,001 (the newest)queued behind everythingarrives seconds later
Figure 1. A flood of state messages to a slow phone through a proxy. The sender's buffer stays small; the kernel's and the proxy's do not.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flood, ~9,800 × 128 B per second, 64 KB/s linkSender memoryNewest state the phone saw
no flow control7.0 MB queued in the WebSocket, heap +18.5 MB9.4 s old
cap bufferedAmount at 64 KiB, merge pending state+0.46 MBstill 9.4 s old
cap at 1 MiB (Fable's rerun)small3.8 s, growing to 10.4 s
at most 1 to 4 acknowledged messages in flight, merge pending state+0.4 MB0.0 s
+

The transport's backpressure protects the sender's memory, but freshness depends on buffers the sender cannot see: +the operating system's socket buffer, autotuned to hundreds of kilobytes, and every proxy on the way. A normal +streamed reply (50 tokens per second, 23 bytes each) never fills them; a burst or a tunnel does. Only the receiving +application knows what it has applied.

+
+ +
II

Acknowledged Messages

A window of two messages in flight, and why it adapts to any link.

+

The rule: a provider has at most K = 2 state messages in flight per link. The consumer sends ack (three bytes in +the binary codec) after it applied one. While two are in flight, new commits do not produce messages; their ops wait, +merged per hold, and go out in the next message once an ack arrives.

+
Two messages in flight on a slow link

Fast link

Commits 1, 2, 3 each become a message; each is acknowledged before the next commit. Nothing waits: the window never +fills, so a normal stream behaves exactly as without flow control.

+

The link slows

Messages 1 and 2 are in flight. Commits 3 to 400 arrive; their ops are folded into the pending queue of each member +they touch (chapter III). Memory: the net change, not the history; at most 10,000 ops per member.

+

An ack arrives

ack for message 1. The provider sends message 3, carrying the folded ops of commits 3 to 400, encoded once. +The phone applies them as one batch and shows the newest text.

+

Self-clocking

The slower the link, the more each message carries; the faster, the smaller and more frequent. Nobody measures +bandwidth: the acknowledgements pace the sender.

+
+
+
+ + + +
+ +
+
+ +

The simulation above shows the same effect in miniature. Drag the link below the commit rate: the upper lane's queue +grows without bound and the phone falls further behind every second; the lower lane never has more than two messages +waiting, and the phone is at most a message behind.

+
+ +
III

Fold the Backlog

What a waiting hold keeps while the window is full, and why it never measures sizes.

+

While messages are in flight, each hold member keeps the ops of new commits in a pending queue and folds them: +a new op merges into the last pending op on the same path when that is safe, so a backlog keeps the net change, not +the history. When an acknowledgement frees a slot, the queue is encoded once into one message.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Last pending opNew opKept
a text "Hel"a text "lo"one a text "Hello"
s count 41 or an appends count 42the s: it overwrites what came before
anythingr valueonly the r: a replacement makes everything before it moot
anything on a path in the indexa structural op (p, m, d, t, s of an object)appended; the index is cleared
+

The merge partner is found through a leaf index: a map from path to the last pending append or primitive set on +that path. Leaf ops on different paths commute, so a new op may merge into an earlier one, not only into its +neighbour; a status field changing between two tokens no longer stops the reply from folding. Any structural op +clears the index, because after a splice entries[7] may name another entry. Past 10,000 pending ops the queue keeps +only "replace with the current value", which then costs nothing while the consumer stays slow.

+

Why no size rule. The first idea was to send a snapshot once the merged ops outgrow the member. The reviewers +measured it on real p2 data and it failed three ways: "snapshot bytes plus sent op bytes" is no bound (an interned path +lets a value grow faster than its ops: 5,404 B of snapshot after 832 B of ops), it drifts 1,254 times too high under +churn, and an exact size needs a size on every node of the value. Folding alone came within about 10% of an exact +oracle, with no measuring at all.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
60 s stall, all workloadsUncompressedCompressed
merged ops, no folding466 KB166 KB
adjacent folding399 KB147 KB
leaf index (the decision)372 KB145 KB
exact oracle (not buildable)335 KB136 KB
+

One queue for every slow consumer. The same queue serves Chord's shared watch handle: a listener still busy with +the last frame gets the commits made meanwhile as one call with the newest value and the folded ops, instead of +today's "after 100 frames, replace everything". With a listener taking one second, a conversation view's watch went +from 402 calls and 17.9 MB of ops to 312 calls and 257 KB. Durable's event stream keeps a queue of its own, because +events are not state: folding would drop them.

+

Animation of the fold: How folding works.

+
+ +
IV

Calls Get Their Own Window

Why acknowledged state does not protect against a call flood.

+

Calls are small and the provider acknowledges nothing until they finish, so a byte or message window on state says +nothing about how many handlers run. A tab with a bug:

+
for (const note of notes) ref.add(note, ctx);   // 100,000 calls, none awaited
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
100,000 calls, handlers awaiting 10 msFinished inProvider memoryAnother client's p95 round trip
no limit0.66 s+56.5 MB, 43,824 running129 ms (345 ms with trivial handlers)
provider-side semaphore of 6415.9 s+42.5 MB, 96,736 queued on the provider0.38 ms
caller-side window of 6417.1 s+4 MB, nothing queued on the provider0.66 ms
handlers doing 10 ms of CPU workblocked about 1 s, with or without a limit
+

The provider's hello announces a call window N (256 by default) per origin (chapter VI). A consumer keeps at +most N of its own calls unanswered per link; further calls wait in the link's call queue, in order. Nothing is refused (K16); a peer that sends +more than N is a protocol error. The flood waits in the tab that caused it. CPU-heavy handlers block the event loop +whatever the window: that stays the handler author's responsibility.

+
+ +
V

Relays

One link to a worker carries many tabs. Decided: flow control per origin.

+

The server relays every tab's holds to a session's worker over one link. Three ways to apply the acknowledged window +across that hop were considered:

+
WORKERSERVERTABSworkerK = 2 per linkserveracks upstream after downstreamfast tabdesktopslow tabphone in a tunnelstatestatestate, slowack (late)ack only when every tab ackedSimplest, but the slow phone holds the worker's one window, so the fast tab's updates arrive in coarser, later batchestoo.
Figure 2. (a) End-to-end acknowledgements: the server acknowledges the worker only after the tab did.
+
WORKERSERVERTABSworkerK = 2 per linkserveracks at once; window per tabfast tabdesktopslow tabphone in a tunnelstateack at oncestate, currentwindow full: its parts droppedreset: fresh snapshotsThe server cannot fold opaque parts, so it drops them and asks for snapshots. Built and measured in slice 4a: a 1 sstall cost three full snapshots (~1.5 MiB) and ~24 s to catch up under streaming.
Figure 3. (b) Acknowledgements per hop: the server acknowledges at once and resets a slow tab.
+
WORKERSERVERTABSworkerK = 2 per (link, origin)serverforwards; FIFO of sendersfast taborigin 1slow taborigin 2, in a tunnelstate @1, state @2state, currentstate, slowack (late)ack @2 when the tab ackedThe relay numbers each tab (an origin) and forwards at once; each tab's ack goes back to the worker that sent themessage. The worker folds the slow tab's commits in that origin's pending queues, exactly as for a direct tab; thefast tab is another origin with its own window.
Figure 4. (c) Flow control per origin (decided 2026-10-24): the worker keeps a window per tab; the relay only forwards.
+

How (c) works:

+
    +
  • Origins. The relay gives each downstream link a number, never reused; every message upstream carries it +(origin), and the worker keys holds, calls, windows and acks by (link, origin, id). Ids are not renumbered.
    • +
  • What the relay keeps. Per relayed hold its upstream link and its original hold message (to route calls and +hold again after a worker restart); per tab a FIFO of which worker sent each message in flight, so each in-order +ack goes back to the right worker. Nothing is parsed, folded, dropped or reset.
    • +
  • Calls. N is per origin at the worker; the relay forwards calls at once, each tab bounded by its own window.
    • +
  • A tab leaves. Its link closes; the relay sends closed { origin } to each worker it relayed to, which releases +that origin's holds and aborts its calls.
    • +
  • Cost. A tab served by W workers may have 2W messages in flight (plus the relay's own 2). Two bytes of origin per +message on the worker link.
    • +
+
+ +
∴

What Changes in the Design

The converged scheme, what it replaces, and the relay decision.

+

Converged (Astra and Fable, after cross-checking each other's experiments): at most K = 2 acknowledged state +messages in flight; pending ops folded per hold member through a leaf index, capped at 10,000; a call window N +announced in hello, extra calls queued by the caller; holds unlimited; size limits stay a separate check.

+

Replaced from the design (§3, §22): the byte window and its credit messages, snapshot admission and debt (R19, Q3), +cut C12's call-credit accounting, any adapter backlog API. IPC and WebSocket behave the same, because acknowledgements +are messages.

+

Relays (owner): per-hop acknowledgements (b) were decided on 2026-10-23 and built in slice 4a; the benchmarks +showed a slow tab recovering through resets far worse than a direct tab (24 s and three snapshots after a 1 s stall). +On 2026-10-24 they were replaced by flow control per origin (c): the worker keeps window, pending queues and calls +per (link, origin), the relay only forwards and returns acks. Measured after the rework: a slow tab through the relay +catches up exactly like a direct one. The circuit breaker stayed undecided; the heartbeat (15 s ping, 30 s close) +was built instead.

+
+ +
A

Measurements

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ExperimentSetupResult
normal stream50 × 23 B/s through 64 KiB/sno buffering, frames ≤ 2 ms old
flood, no control~9,800 × 128 B/s, 10 s7.0 MB in bufferedAmount, heap +18.5 MB, newest state 9.4 s old
flood, bufferedAmount ≤ 64 KiB + mergingsameheap +0.46 MB, newest state 9.4 s old
flood, 1 to 4 acked messages + mergingsameheap +0.4 MB, newest state 0.0 s old, 98 % fewer messages
client stops reading (loopback)20k × 90 B/s, 15 s23.4 MiB buffered, RSS +121 MB; merging above 1 MiB: flat
IPC to a busy child623k sendssend() false from ~64 KB, parent +1,035 MB; the callback acknowledges the write, not the handler
call flood100,000 calls, see chapter IVcaller-side window: +4 MB, other client p95 0.66 ms
slow tab through a relay, (b) resets64 KiB/s, 100 tokens/s into a 500 KB view, 1 s stall~24 s to catch up, three full snapshots (~1.5 MiB)
slow tab through a relay, (c) per originsame, 1 s / 10 s / 60 s stallcaught up after 0.02 / 0.08 / 0.38 s, 485 B / 4.1 KiB / 23.7 KiB (same as direct)
+
+ +

Flow control for Chord links · fourth edition · 2026-10-24 · based on the reviewers' experiments and design §3, §22

+
+ + diff --git a/packages/chord/docs/next/flow-control/window/index.html b/packages/chord/docs/next/flow-control/window/index.html new file mode 100644 index 000000000..52ac8c39c --- /dev/null +++ b/packages/chord/docs/next/flow-control/window/index.html @@ -0,0 +1,228 @@ + + + + + +A windowed conversation view + + + +
+

A windowed conversation view

+

Proposal for Durable: the view holds a window of the newest entries with a stable start; older entries are fetched in pages; a reconnect resumes after the last entry it saw.

+ + 
type ConversationView = {
+  conversation: ConversationRecord;
+  head?: EntryRecord;               // the current compaction summary, if any
+  entries: readonly EntryRecord[];  // the window: active entries with id >= from
+  earlier: boolean;                 // older active entries exist before it
+  docs: { "pi.live", "pi.agent", "pi.inbox", "pi.provider", "pi.usage" };
+};
+conversation.watch(context, { from?: EntryId, tail?: number });          // default: the last `tail`
+conversation.entries({ before, order: "descending" }, limit, cursor, ctx); // exists today
+ +

Watch it work

+

Shown with tail = 4 (the proposal: about 50). The window grows by appends until it holds twice tail; then the oldest half leaves in one splice.

+
+
+ + + + + +
+
+
store: every active entry
+
client: view window (blue), kept or fetched (purple)
+
+
+ window 0 entries + ops sent 0 + last snapshot - +
+
+
+
+ +

What each step costs

+ + + + + + + + +
stepwire
a new entryone append splice: p entries <len> 0 [entry]
window reaches 2 × tailone splice removing the oldest half: p entries 0 <tail> [], once every tail entries
streaming into the newest messageappends on docs["pi.live"].generation.message, unchanged
hold, re-hold, overflowa snapshot of at most 2 × tail entries plus the live documents, never the whole transcript
scroll upa page from entries(), outside the view: the client keeps it, the view never grows
reconnect with from: lastSeen + 1only the entries it missed plus the documents (issue #11327)
+

Why a stable start instead of “always the last 50”: a sliding window would remove one entry at the front for every entry appended at the back, two ops per entry, and every index in the window would shift. With a stable start, entries only append, and the front moves in one splice now and then.

+
+ + + + diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md new file mode 100644 index 000000000..747cc4e32 --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -0,0 +1,649 @@ +--- +title: Hosts and Plugins: a tutorial for Chord v2 +cover: Hosts / and Plugins +kicker: A guide · Chord v2 · 2026-10-31 +subtitle: Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs +stats: + - 3 | kinds of host in an app: server, worker, client + - 1 | file per host kind in a plugin + - 0 | restarts to apply a plugin edit + - 1 | rule: plugins talk only through object types and registries +note: | + A tutorial for people building apps on Chord v2, written by the redesign session from `design.md`, `slice-7c.md`, + `slice-10.md`, the decision log and the code in `packages/chord/src/next` and `src/next-node`. API names are the + current ones; where slice 10's fix round or the app kit (`chord/next/app`, being designed) will change what app code + writes, a note says so. +colophon: Hosts and Plugins · second edition · 2026-11-01 · Chord v2 slices 1-10 and 12 +--- + +# Before we begin {.preface} + +You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything. + +Chord gives you three things for that. **Hosts**: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. **Objects**: the state and methods your app is made of, living on one host and +used from any other. **Plugins**: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs. + +This book builds the app in that order. Each chapter adds one concept and shows complete code. + +## Seven things to know + +:::findings +1. **A host is one participant.** The server, each worker, each browser tab and each terminal UI is a host; hosts + connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask. +2. **An object has a type, and a type is a contract.** The contract names the object's state and methods; any host can + ask for an object by type and params, and Chord finds where it lives. +3. **A facet declares, an object acts.** A facet is a plugin's code for one host kind: it says which object types this + host serves. The objects hold the state and do the work. +4. **A plugin is a directory** (or a single file). A `package.json` with the plugin's name, and one file per host kind: + `server.ts`, `worker.ts`, `web.tsx`. It imports its host only through the app's versioned API (`@app/api/v1`). +5. **The server builds, every host evaluates.** The plugins component builds each facet into a small script; the + server, workers and tabs load the scripts for their kind. +6. **Plugins meet through types and registries.** One plugin uses another's objects through its contract, or + contributes to a registry another defines. Nothing else crosses between plugins. +7. **An edit is a rollout.** Change a file, build, roll out: the server, then every worker, swap the plugin's objects in + place; a broken edit keeps the old code running and reports why. +::: + +:::flow wide animate | How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server. +row "On disk" muted +box disk "plugins/notes/" "`server.ts`, `worker.ts`, `web.tsx`, `tui.ts`, `contract.ts`" muted +box core "src/ (base code)" "`@app/api/v1`: Chord's surface, host kinds, principals" muted +row "Server process" accent +box comp "plugins component" "discover, build every facet, watch, roll out" accent +box server "server host" "server facets; routes Project → workers" +row "Clients and workers" +box tab "client: tab or TUI" "web or tui facets, local UI objects; holds `Notes { p1 }`" blue +box w1 "worker p1" "worker facets; serves `Notes { p1 }`" +arrow disk -> comp "read, build" +arrow core -> comp "host packages" +arrow comp -> server "server scripts" +arrow comp -> tab "web or tui scripts" +arrow tab -> server "hold Notes { p1 }" +arrow server -> w1 "start (worker scripts), relay" +note "Each host loads only its own kind's facets. Plugins meet only through object types and registries." +::: + +## Words used throughout + +A **host** is one participant of the app, made with `createHost`. A **host kind** (`Server`, `Session`) describes +addressable hosts; an **address** (`Session({ id: "p1" })`) names one. An **object type** is a contract: `local` (used +on its own host only) or `remote` (used from any host). An **object** is one instance of a type, per params. A +**facet** is a plugin's declarations for one host kind. A **plugin** is a directory with a `package.json`. A +**reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **document** is data an object +stores, changed in **transactions**; a **table** holds many records with queries, in the host's **store**; a **derive** is a value computed from others; a **native** is an outside +resource an object opens and closes. A **registry** is an +extension point: one plugin defines it, others contribute. A **rollout** applies a build to the hosts in order. + +# One Host {#host lede="The smallest app: one process, one object type, one call."} + +Start without plugins. One host, one object that keeps notes: + +```ts +import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next"; + +const NoteList = obj({ items: arr(str()) }); // one schema, used by the contract and the document + +// the contract: what a Notes object offers +const Notes = local("notes.store", { + version: 1, + params: {}, // one object per host: no params + members: { + list: NoteList, // state anyone can read and watch + add: method(obj({ text: str() })), // a method: arguments checked by Chord + }, +}); + +// the document the object stores (its own version: storage can change without changing the contract) +const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) }); + +// the facet: which types this host serves, and how to make each object +const facet = defineFacet((host) => { + host.serve(Notes, (_params, self) => { + const doc = self.state(NotesDoc); + return { + list: doc, + add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx), + }; + }); +}); + +const server = createHost({ onError: console.error }); +await server.load([{ name: "@app/notes", facet }], ctx); +const notes = await server.get(Notes, {}, ctx); // a reference: live, with its state +await notes.add({ text: "buy milk" }, ctx); +console.log(notes.list.value); // { items: ["buy milk"] } +``` + +Four things happened. The **contract** `Notes` says what the object offers; the **document** says what it stores. +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). +The **facet** says this host serves +`Notes`, with a **factory** that declares the object's parts: a document, and a method writing it in a transaction. +`load` publishes the facet. `get` returns a **reference** once the object is live; its members are the contract's. + +The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes. + +# An Object's Parts {#parts lede="Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory."} + +A factory builds an object from a handful of parts, each declared with one call on `self`. The notes object grows to +show all of them. + +**Documents** hold the object's data: `self.state(NotesDoc)` gives a handle whose `.value` is the current data. Data +changes only in a **transaction**: `self.tx(fn, ctx)` runs `fn` with `tx`, and `await tx.draft(handle)` gives a +mutable copy of the document's data; whatever you change on it is committed at once when `fn` returns, or not at all +if it throws. + +```ts +add: async ({ text }, ctx) => self.tx(async (tx) => { + const draft = await tx.draft(doc); // the document's data, writable inside this transaction only + draft.items.push(text); // recorded as a change, committed when the function returns +}, ctx), +``` + +Documents live in memory unless they are **persisted**: `defineState(..., { persist: true, checkpointWhen })` keeps +one in the host's **store** (a SQLite file the app opens and gives to `createHost({ store })`), loaded when the +object activates. For many records with queries, an object declares a **table**: + +```ts +const NoteRows = defineTable("notes", { + version: 1, + columns: { id: str(), text: str(), at: int() }, + key: ["id"], + indexes: { by_time: ["at"] }, +}); + +host.serve(NoteStore, (_params, self) => { // a local type with empty params: it may persist + const rows = self.table(NoteRows); // created (or migrated) when the object activates + const recent = query(NoteRows).orderBy((n) => n.at, "desc"); + return { + add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx), + page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }), // 50 at a time, with a cursor + }; +}); +``` + +Persisted data lives only in `local` objects with empty params (one per host and type), so its address never depends +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go +through the same transactions as documents: a note row and a document can change in one commit. + +A **derive** is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store: + +```ts +const count = self.derive([doc], ([notes]) => notes.items.length); // 0, 1, 2, … as notes are added +return { list: doc, count, add }; // `count` can be a member like `list` +``` + +**Watching** is how anyone follows a value as it changes: a document handle, a derive, a reference's member. `watch` +gives the current value, then one call per change: + +```ts +const watch = await notes.count.watch(ctx); +watch.start(async (call) => { + if (call.kind === "value") console.log("notes:", call.value); // "failed" calls say a value cannot be computed now +}); +``` + +A **routine** is the object watching on its own behalf: `self.on(source, fn)` runs `fn` with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file"). + +A **native** is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. `self.open(open, close)` opens it while the object activates and closes it when the object ends: + +```ts +const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close()); +self.onLive(async (ctx) => { // a routine started once the object is live + for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx); +}); +``` + +Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place. + +# Several Hosts {#hosts lede="A server, a worker per project, browser tabs: how they find each other's objects."} + +Now give each project its own process. Declare the kinds of addressable host, and make notes a `remote` type that +lives on its project's worker: + +```ts +export const Server = hostKind("server"); +export const Project = hostKind("project", { id: str() }); + +export const Notes = remote("notes", { + version: 1, + params: { project: str() }, + hostAddress: ({ project }) => Project({ id: project }), // where an object of these params lives + variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } }, +}); +``` + +Two things are new. A `remote` type can be used from any host, and it has **variants**: different sets of members +for different callers, chosen by the factory (an `editor` variant with `add`, a `reader` variant for guests without +it). And `hostAddress` says where its objects live; it is the whole routing story. + +A tab asks for `Notes { project: "p1" }`. Its host computes the address, +`Project(p1)`, and sends the request to its parent, the server. The server has a **route** for `Project`: it starts (or +finds) the worker for `p1` and passes the request on. The worker serves `Notes`. The tab gets a reference whose state +follows the worker's document, one frame per change. + +:::flow wide animate | A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data. +row "Clients" blue +box tab1 "tab A" "holds Notes { project: p1 }" +box tab2 "tab B" "holds Notes { project: p1 }" +row "Server" accent +box server "server" "route Project → worker" +row "Workers" muted +box w1 "worker p1" "serves Notes { p1 }" +box w2 "worker p2" "serves Notes { p2 }" +arrow tab1 -> server "hold" +arrow tab2 -> server "hold" +arrow server -> w1 "relay" +::: + +Each host is created the same way, with what it is: + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); + +// worker.ts (started by the server for one project) +const worker = createHost

({ address: Project({ id }), principal: { kind: "worker", project: id }, onError }); +worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx); + +// a browser tab: no address, it is a client +const tab = createHost

({ onError }); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +``` + +`P` is your app's type of **principal**, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a `remote` object is made per principal, so a guest can get a read-only variant of the same notes. + +:::note | Coming with the app kit +Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components today +(`createWorkers`, `chord.http`), wired by the app. The opinionated app kit (`chord/next/app`, being designed) will do +this wiring for you: `startApp(...)` and `startWorker()`. +::: + +# Client Hosts {#clients lede="Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot."} + +A browser tab and a terminal UI are hosts like the server, with one difference: they have **no address**. Nobody can +send a request *to* a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state. + +```ts +// a browser tab (slice 11's page host will do this wiring for you) +const tab = createHost

({ onError }); // no address, no principal: a client +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx); + +const notes = tab.hold(Notes, { project: "p1" }, ctx); // a reference at once, live soon +(await notes.list.watch(ctx)).start(async (call) => { + if (call.kind === "value") render(call.value.items); // every change, as it commits on the worker +}); +await notes.add({ text: "from the tab" }, ctx); // a call: tab → server → worker +``` + +**Who the tab is** is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +`Notes` factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner. + +**What a tab serves** is only `local` types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache: + +```ts +// plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel +export default defineFacet((host, self) => { + host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) })); // lives in this tab only + self.contribute(Panels, { title: "Notes", Component: NotesPanel }); +}); +``` + +**When the connection drops** (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their `state` says `loading`, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from `ref.state`, nothing more. + +**A terminal UI** is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +`tui` facets, rendering with whatever terminal library the app chose. **What a client never does**: serve a `remote` +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker. + +# Plugins {#plugins lede="Move the code into a directory that can be added, edited and removed while the app runs."} + +So far the facets were part of the app's own code. A **plugin** is the same code in its own directory: + +```text +plugins/notes/ + package.json { "name": "@app/notes" } + contract.ts the Notes contract (other plugins import it) + worker.ts the facet for workers: serves Notes + server.ts the facet for the server: an export route + web.tsx the facet for browser tabs: a notes panel +``` + +The plugin's name is its `package.json` name. Each file named after a **host kind** is that kind's facet, its +default export a `defineFacet(...)`. A plugin has only the files it needs: a plugin that only adds a route has a +`server.ts` and nothing else. + +```ts +// plugins/notes/worker.ts +import { defineFacet } from "@app/api/v1"; +import { Notes } from "./contract.ts"; + +export default defineFacet((host) => { + host.serve(Notes, ({ project }, principal, self) => { + const doc = self.state(NotesDoc); + return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } }; + }); +}); +``` + +`@app/api/v1` is your app's **API**, version 1: the part of its base code plugins import, as a host package (the +plugin's build leaves it out; every host passes its own copy in). It holds what plugins share and never reloads: +Chord's builders re-exported (`local`, `remote`, `obj`, `str`, ...), host kinds, the principal type, a `defineFacet` +bound to it. A plugin never imports Chord itself; the version in the path says what it was written against. + +# Loading Plugins {#loading lede="Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it."} + +A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins' +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files, +but then every worker would compile every plugin again, and two workers started a minute apart could see two different +versions. So loading a plugin is two separate steps: + +1. **Build**: turn each plugin facet (`notes/worker.ts`, `notes/web.tsx`) into one self-contained **script**. This + happens once per edit, on the server, for every host kind. +2. **Evaluate**: run a script inside one process, which gives back the facet, ready for `host.load`. This happens in + every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for + the web facets. + +The two steps need different things, and that is why the API has two calls. + +## Host packages: what a script leaves out + +Look at what `notes/worker.ts` imports: + +```ts +import { defineFacet, obj, remote, Session, str } from "@app/api/v1"; // the app's API: its own part and Chord's +import { Notes } from "./contract.ts"; // the plugin's own file +import slugify from "slugify"; // an npm package in its dependencies +``` + +The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first must not. +If the script carried its own copy of the API, and with it of Chord, the worker would run two Chords: the plugin's +`ChordError` would not be the worker's `ChordError`, and the plugin's `defineFacet` would make a facet the worker's host +does not recognize. The plugin must see the very `Session` host kind and the very `Panels` registry the process +already has, not a copy. + +Packages the process already has, and that every plugin must share, are its **host packages**. A script leaves them +out and asks for them by name when it runs. Building therefore needs only their **names**: "do not copy `@app/api/v1` +into this script". + +## Building: `createPlugins`, on the server + +```ts +const plugins = createPlugins({ + roots: [{ path: "./plugins", required: true }], // where the plugin directories are + api: { name: "@app/api", versions: ["v1"] }, // the app's API and the versions it provides + kinds: { + server: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/server"] }, + worker: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/worker"] }, + web: { platform: "browser", hostPackages: ["@app/api/v1", "@app/api/v1/web", "react", "react/jsx-runtime"] }, + }, + onError, +}, ctx); +``` + +`createPlugins` is the builder, and it builds for every kind, because only the server has the sources. For each kind +it needs two facts. The **platform** says where the scripts run: a `node` script may import `node:fs`, a `browser` +script may import a stylesheet but not `node:fs`. The **host package names** say what to leave out; they differ per +kind, because a tab also shares React with its plugins and a worker does not, and the API's per-kind modules +(`/v1/web`) belong to their kind only. With `api` set, the build checks every import of the API: a version the app +does not list, another kind's module, two versions in one facet, or Chord imported directly is a build error naming +the file and line. Nothing here runs a plugin. + +## Evaluating: `plugins.load`, in the process that hosts + +Evaluating is where the names become things. When the script runs and asks for `@app/api/v1`, someone has to hand +it the actual module, the one this process imported when it started. Only the process itself has it: + +```ts +// server.ts +import * as api from "./api/v1/index.ts"; // the app's API, the module plugins import as "@app/api/v1" +import * as apiServer from "./api/v1/server.ts"; +import { serverFacet } from "./core.ts"; + +const report = await plugins.load(server, { + kind: "server", // evaluate the server kind's scripts + modules: { "@app/api/v1": api, "@app/api/v1/server": apiServer }, // this process's own instances + core: [{ name: "@app/core", facet: serverFacet }], // base-code facets, loaded beside the plugins +}, ctx); +``` + +`modules` is this process's answer to every host package name. The server passes its own; a worker passes its own in +`workerMain`; a page passes its own in `startPage`. Each process has different instances, which is exactly the point: +inside each process, plugins share that process's Chord and base code. + +`core` is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the +process's own code, often from values only this process has (`core.workerFacet(start)` closes over a worker's start +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together +with the plugins, so they are published in one step. + +## Why `plugins.load(host)`, not `host.load` + +A host has its own `load`, and `plugins.load` calls it. The host's `load` takes facets that already exist: functions +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing +is built or evaluated at runtime. + +The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them +with the process's modules, and then calls `host.load` with the result. It also remembers, per host, which facet each +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very +same objects. That memory is why the host is passed to it: `plugins.load(server, …)` on the server, +`plugins.load(worker, …)` in a worker (where `workerMain` gives it a `plugins` that only evaluates), the same call +in every process. + +:::flow wide | Building on the server needs names; evaluating in each process needs that process's modules. +row "Server: build (once per edit)" accent +box src "plugin sources" "`notes/worker.ts`, imports `@app/api/v1`" muted +box build "createPlugins" "copies own files and npm deps; leaves host packages out" accent +box script "worker script" "`require(\"@app/api/v1\")` left as a name" +row "Each worker: evaluate" blue +box mods "the worker's modules" "its own `@app/api/v1`, its own Chord" muted +box load "plugins.load(worker, …)" "runs the script, answers each name" blue +box facet "the facet" "uses the worker's `Session`, the worker's `defineFacet`" +arrow src -> build +arrow build -> script +arrow script -> load "shipped" +arrow mods -> load +arrow load -> facet +::: + +## What happens, step by step + +:::steps | `plugins.load(server, { kind: "server", … })` ++++ Discover +The component lists `./plugins` and reads each `package.json`. Its `name` is the plugin's name; the files named after +a kind are its facets. A directory without a `package.json`, or with two `server` files, is reported, not loaded. Files +at the root's top named `..ts` (`hello.server.ts`) are single-file plugins, grouped by name. ++++ Build +For each plugin with a `server` facet it builds one script, if no current one exists. A syntax error or an import the +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file +and line, never a crash. ++++ Evaluate +The script runs. Each host package name it asks for is answered from `modules`; anything else it may not have. The +script's default export must be a facet. ++++ Load +`host.load` runs the setups of `core` and of every plugin facet and publishes them together. A plugin whose build, +evaluation or setup failed is left out and listed in `report.failed` with the reason; the others load. Objects that +need it wait until a later edit loads it. +::: + +Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from `workerMain`, a page's from its base bundle). + +# Plugins Using Plugins {#together lede="Two ways one plugin uses another, and the one it may not."} + +A search plugin wants to search the notes. It **imports the contract** and asks for the object: + +```ts +// plugins/search/worker.ts +import { Notes } from "../notes/contract.ts"; // a plugin of the same root, by relative path: copied into search's script + +host.serve(Search, ({ project }, principal, self) => { + const notes = self.get(Notes, { project }); // a dependency: Search goes live once Notes is + const index = self.derive([notes.list], ([list]) => buildIndex(list.items)); + return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } }; +}); +``` + +Which file a plugin imports decides which version of the contract it claims, so the rule is: **a plugin carries the +contract it was written against.** Two cases: + +- **Plugins that change together** (the app's own, in its repository; a user's, in one root) import each other by + relative path, `../notes/contract.ts`: the same commit holds both sides. The file is copied into the importer's + script, so only contracts and other side-effect-free files belong there (a copied file may import only the API and + files of its own plugin). A relative path leaving the root (into base code, into another root) is a build error. +- **Plugins from another repository** take contracts as a dependency at a fixed version, by package name: + `@bob/notes` (its `contract.ts`, through the package's `exports`), `@bob/notes/web` (its `contract.web.tsx`), pinned + in `package.json` by an npm version or a git ref (`"@bob/notes": "github:bob/notes#"`). The build inlines + the installed version: when Bob's notes move to contract version 2, a consumer built against version 1 keeps + claiming version 1, and the provider serves it through `accepts` or the consumer waits "contract mismatch", visible, + never silently wrong. A plugin inside a repository holding several (which lists them in its own `package.json`, + `"chord": { "plugins": ["./notes", "./search"] }`) is a dependency on the whole repository, imported by path into it: + `@bob/plugins/notes/contract.ts`. + +A contract file may be for one kind: `contract.web.tsx` (React components in a type, a panel's registry token) is a +**web file**. Importing it, directly or through other files, from a facet of another kind is a build error naming +the chain (`server.ts → view.ts → ../notes/contract.web.tsx`). The shared `contract.ts` mentions native things only +through `import type`. + +`self.get` in a factory declares a dependency: Chord makes `Search` live only once `Notes` is, and if the notes +plugin is missing, `Search` waits with a clear reason ("waiting for notes: not served here") instead of failing. + +The other way is a **registry**: an extension point one plugin defines and others contribute to. The app's API +defines `Panels` in its web module, and every plugin with a UI contributes one: + +```tsx +// plugins/notes/web.tsx +import { Panels } from "@app/api/v1/web"; +export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel })); +``` + +What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the `Notes` +object's state. That rule is what lets the notes plugin change its storage without breaking search. + +# Changing a Plugin {#reload lede="Edit, build, roll out: how new code takes over while every tab stays connected."} + +You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out: + +```ts +const built = await plugins.build(plugins.state.value.dirty, ctx); +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +``` + +The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording `self` to see what each object would declare, and +then gives every object of the changed plugin its fate: + +| The object | What happens | +| --- | --- | +| holds no natives (open files, sockets), the new code neither | **swapped** in place: new methods, same documents; running calls finish on the old code | +| holds natives | **restarted**: calls wait, running ones finish, natives close and reopen with the new code | +| its type was removed | **ended**; its users wait until something serves it again | + +The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies. + +:::note | Versions +A change to a contract (a new field in `Notes`'s state) gets a new `version`. A tab built with the old contract keeps +working if the provider maps the old version (`accepts`); otherwise it waits with "version mismatch" until it reloads. +::: + +# Base Code and Plugins {#base lede="What goes into the app itself, what goes into plugins, and how an app ships plugins of its own."} + +Every app has two kinds of code, and the line between them decides what can change while the app runs. + +**Base code** is the app's own program: `src/`, started by `node server.ts`. It shrinks to three things: the **API** +(the versioned modules plugins import), the **bootstrap** (creating hosts, opening the store, components such as +`createHttp` and `createWorkers`) and one small **start-data facet** serving process values (start data, the opened +store) as core objects. Base code changes only with a restart: if it reloaded, the process would hold two `Panels` +registries, and old and new plugins would no longer meet. + +**Plugins** are everything else: notes, search, export, a terminal, the harness, the providers of the app's core +registries. They reload on an edit, a broken one is left out alone, and an agent can write one while the app runs. + +## The API, by version and by kind + +Plugins import their host only through the API, a version in the path: + +```ts +// src/api/v1/index.ts: "@app/api/v1", for every kind +export { defineRegistry, local, obj, remote, str } from "@earendil-works/chord/next"; // the Chord surface the app supports +export { defineFacet, Project, Server } from "../../core.ts"; // bound to the app's principal +// src/api/v1/web.ts: "@app/api/v1/web", for pages only +export const Panels = defineRegistry("app.panels", { version: 1 }); +``` + +The app lists the versions it provides, newest first (`api: { name: "@app/api", versions: ["v2", "v1"] }`), and each +kind's modules in `kinds..hostPackages`. When the app changes its API in a way that breaks plugins, it adds +`/v2` and keeps `/v1` as a thin layer over it for a while; once `/v1` goes, a plugin importing it fails to build with +"@app/api/v1 is not available: this app provides v2", the plugin alone, the message saying what to change. One facet +uses one version. Contracts the app promises others are re-exported in the API (`export { Notes } from +"../../plugins/notes/contract.ts"`), so a plugin built against `/v1` keeps v1's contract files. + +## Plugins the app ships + +An app comes with plugins of its own. They follow the same rules as anyone's, in a root inside the app, next to the +root where users and agents add theirs: + +```ts +const plugins = createPlugins({ + roots: [{ path: "./plugins", required: true }, join(dataDir, "plugins")], // the app's own, then the user's + api: { name: "@app/api", versions: ["v1"] }, + kinds, + onError, +}, ctx); +``` + +They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left out +alone when broken at a reload. The app's root is **required**: a plugin of it failing at startup makes +`plugins.load` reject with a `PluginError` naming it (after the load settled), so the app stops instead of running +without its core. A failure in the user's root is reported and the app runs on. If two roots have a plugin with the +same name, the first root wins, and the other is reported. + +## Small plugins, installed plugins, tested plugins + +- **A single file** at a root's top is a plugin: `plugins/hello.server.ts` and `plugins/hello.web.tsx` are the plugin + `hello` with two facets. It has no `package.json` and imports only the API; when it needs a second file or a + package, it becomes the directory `hello/`, which a reload treats as the same plugin. +- **Installing** from git or npm: `plugins.installPlugin({ git: url, ref }, root, ctx)` clones the ref (a branch, tag + or commit) into the root, `{ npm: name, version }` extracts the package; then its dependencies are installed. Removal + is deleting the directory. +- **Testing** a plugin alone: `testPlugin("./plugins/stats", { kind: "worker", api, modules, stubs }, ctx)` builds it + as production does, loads it on an in-memory host beside stub facets serving what it uses, collects what it + contributes to registries nobody serves, and lists the contracts it waits for (`docs/next/plugin-authoring.md`). + +# Reference {.appendix} + +| Call | What it does | +| --- | --- | +| `createHost

({ address?, principal?, store?, onError })` | a host: addressable with an address, else a client | +| `host.load([{ name, facet }], ctx)` / `host.reload(...)` | publish facets / apply a new plugin set | +| `host.get(Type, params, ctx)` / `host.hold(...)` | a live reference / a reference at once that follows its object | +| `local(name, { version, params, members })` / `remote(name, { version, params, hostAddress, variants })` | contracts | +| `defineFacet((host, self) => …)`, `host.serve(Type, factory)`, `self.contribute(R, value)` | a facet | +| factory `self`: `state`, `derive`, `get`, `registry`, `open`, `on`, `onLive`, `job`, `tx` | an object's parts | +| `createPlugins({ roots, api, kinds, onError }, ctx)` | the plugins component | +| `plugins.load(host, { kind, core, modules }, ctx)`, `build(names)`, `rollout(steps, built)` | load, build, roll out | +| `plugins.installPlugin(source, root, ctx)`, `testPlugin(dir, options, ctx)` | install a plugin, test one alone | +| `hostKind(name, params?)`, `Kind(params)` | host kinds and addresses | diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html new file mode 100644 index 000000000..9b30188ea --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -0,0 +1,696 @@ + + +Hosts and Plugins: a tutorial for Chord v2 + + + + +

+
+
A guide · Chord v2 · 2026-10-31
+

Hosts
and Plugins

+

Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs

+
3kinds of host in an app: server, worker, client
1file per host kind in a plugin
0restarts to apply a plugin edit
1rule: plugins talk only through object types and registries
+

A tutorial for people building apps on Chord v2, written by the redesign session from design.md, slice-7c.md, +slice-10.md, the decision log and the code in packages/chord/src/next and src/next-node. API names are the +current ones; where slice 10's fix round or the app kit (chord/next/app, being designed) will change what app code +writes, a note says so.

+
+
+ + +
+

Before we begin

+

You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything.

+

Chord gives you three things for that. Hosts: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. Objects: the state and methods your app is made of, living on one host and +used from any other. Plugins: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs.

+

This book builds the app in that order. Each chapter adds one concept and shows complete code.

+

Seven things to know

+
    +
  1. A host is one participant. The server, each worker, each browser tab and each terminal UI is a host; hosts +connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask.
    2. +
  2. An object has a type, and a type is a contract. The contract names the object's state and methods; any host can +ask for an object by type and params, and Chord finds where it lives.
    3. +
  3. A facet declares, an object acts. A facet is a plugin's code for one host kind: it says which object types this +host serves. The objects hold the state and do the work.
    4. +
  4. A plugin is a directory (or a single file). A package.json with the plugin's name, and one file per host kind: +server.ts, worker.ts, web.tsx. It imports its host only through the app's versioned API (@app/api/v1).
    5. +
  5. The server builds, every host evaluates. The plugins component builds each facet into a small script; the +server, workers and tabs load the scripts for their kind.
    6. +
  6. Plugins meet through types and registries. One plugin uses another's objects through its contract, or +contributes to a registry another defines. Nothing else crosses between plugins.
    7. +
  7. An edit is a rollout. Change a file, build, roll out: the server, then every worker, swap the plugin's objects in +place; a broken edit keeps the old code running and reports why.
    8. +
+
ON DISKSERVER PROCESSCLIENTS AND WORKERSplugins/notes/server.ts`, `worker.ts`,`web.tsx`, `tui.ts`, `contract.tssrc/ (base code)`@app/api/v1`: Chord's surface, hostkinds, principalsplugins componentdiscover, build every facet, watch,roll outserver hostserver facets; routes Project →workersclient: tab or TUIweb or tui facets, local UI objects;holds `Notes { p1 }`worker p1worker facets; serves `Notes { p1 }`read, buildhost packagesserver scriptsweb or tui scriptshold Notes { p1 }start (worker scripts), relayEach host loads only its own kind's facets. Plugins meet only through object types and registries.
Figure 1. How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server.
+

Words used throughout

+

A host is one participant of the app, made with createHost. A host kind (Server, Session) describes +addressable hosts; an address (Session({ id: "p1" })) names one. An object type is a contract: local (used +on its own host only) or remote (used from any host). An object is one instance of a type, per params. A +facet is a plugin's declarations for one host kind. A plugin is a directory with a package.json. A +reference is what you get when you ask for an object (host.get, self.get, ctx.get). A document is data an object +stores, changed in transactions; a table holds many records with queries, in the host's store; a derive is a value computed from others; a native is an outside +resource an object opens and closes. A registry is an +extension point: one plugin defines it, others contribute. A rollout applies a build to the hosts in order.

+
+ +
I

One Host

The smallest app: one process, one object type, one call.

+

Start without plugins. One host, one object that keeps notes:

+
import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next";
+
+const NoteList = obj({ items: arr(str()) });          // one schema, used by the contract and the document
+
+// the contract: what a Notes object offers
+const Notes = local("notes.store", {
+	version: 1,
+	params: {},                                        // one object per host: no params
+	members: {
+		list: NoteList,                               // state anyone can read and watch
+		add: method(obj({ text: str() })),            // a method: arguments checked by Chord
+	},
+});
+
+// the document the object stores (its own version: storage can change without changing the contract)
+const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) });
+
+// the facet: which types this host serves, and how to make each object
+const facet = defineFacet((host) => {
+	host.serve(Notes, (_params, self) => {
+		const doc = self.state(NotesDoc);
+		return {
+			list: doc,
+			add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx),
+		};
+	});
+});
+
+const server = createHost({ onError: console.error });
+await server.load([{ name: "@app/notes", facet }], ctx);
+const notes = await server.get(Notes, {}, ctx);        // a reference: live, with its state
+await notes.add({ text: "buy milk" }, ctx);
+console.log(notes.list.value);                          // { items: ["buy milk"] }
+

Four things happened. The contract Notes says what the object offers; the document says what it stores. +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). +The facet says this host serves +Notes, with a factory that declares the object's parts: a document, and a method writing it in a transaction. +load publishes the facet. get returns a reference once the object is live; its members are the contract's.

+

The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.

+
+ +
II

An Object's Parts

Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory.

+

A factory builds an object from a handful of parts, each declared with one call on self. The notes object grows to +show all of them.

+

Documents hold the object's data: self.state(NotesDoc) gives a handle whose .value is the current data. Data +changes only in a transaction: self.tx(fn, ctx) runs fn with tx, and await tx.draft(handle) gives a +mutable copy of the document's data; whatever you change on it is committed at once when fn returns, or not at all +if it throws.

+
add: async ({ text }, ctx) => self.tx(async (tx) => {
+	const draft = await tx.draft(doc);          // the document's data, writable inside this transaction only
+	draft.items.push(text);                      // recorded as a change, committed when the function returns
+}, ctx),
+

Documents live in memory unless they are persisted: defineState(..., { persist: true, checkpointWhen }) keeps +one in the host's store (a SQLite file the app opens and gives to createHost({ store })), loaded when the +object activates. For many records with queries, an object declares a table:

+
const NoteRows = defineTable("notes", {
+	version: 1,
+	columns: { id: str(), text: str(), at: int() },
+	key: ["id"],
+	indexes: { by_time: ["at"] },
+});
+
+host.serve(NoteStore, (_params, self) => {               // a local type with empty params: it may persist
+	const rows = self.table(NoteRows);                    // created (or migrated) when the object activates
+	const recent = query(NoteRows).orderBy((n) => n.at, "desc");
+	return {
+		add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx),
+		page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }),   // 50 at a time, with a cursor
+	};
+});
+

Persisted data lives only in local objects with empty params (one per host and type), so its address never depends +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go +through the same transactions as documents: a note row and a document can change in one commit.

+

A derive is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store:

+
const count = self.derive([doc], ([notes]) => notes.items.length);   // 0, 1, 2, … as notes are added
+return { list: doc, count, add };                                      // `count` can be a member like `list`
+

Watching is how anyone follows a value as it changes: a document handle, a derive, a reference's member. watch +gives the current value, then one call per change:

+
const watch = await notes.count.watch(ctx);
+watch.start(async (call) => {
+	if (call.kind === "value") console.log("notes:", call.value);   // "failed" calls say a value cannot be computed now
+});
+

A routine is the object watching on its own behalf: self.on(source, fn) runs fn with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file").

+

A native is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. self.open(open, close) opens it while the object activates and closes it when the object ends:

+
const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close());
+self.onLive(async (ctx) => {                                  // a routine started once the object is live
+	for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx);
+});
+

Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place.

+
+ +
III

Several Hosts

A server, a worker per project, browser tabs: how they find each other's objects.

+

Now give each project its own process. Declare the kinds of addressable host, and make notes a remote type that +lives on its project's worker:

+
export const Server = hostKind("server");
+export const Project = hostKind("project", { id: str() });
+
+export const Notes = remote("notes", {
+	version: 1,
+	params: { project: str() },
+	hostAddress: ({ project }) => Project({ id: project }),   // where an object of these params lives
+	variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } },
+});
+

Two things are new. A remote type can be used from any host, and it has variants: different sets of members +for different callers, chosen by the factory (an editor variant with add, a reader variant for guests without +it). And hostAddress says where its objects live; it is the whole routing story.

+

A tab asks for Notes { project: "p1" }. Its host computes the address, +Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or +finds) the worker for p1 and passes the request on. The worker serves Notes. The tab gets a reference whose state +follows the worker's document, one frame per change.

+
CLIENTSSERVERWORKERStab Aholds Notes { project: p1 }tab Bholds Notes { project: p1 }serverroute Project → workerworker p1serves Notes { p1 }worker p2serves Notes { p2 }holdholdrelay
Figure 2. A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data.
+

Each host is created the same way, with what it is:

+
// server.ts
+const server = createHost<P>({ address: Server(), principal: { kind: "server" }, onError });
+
+// worker.ts (started by the server for one project)
+const worker = createHost<P>({ address: Project({ id }), principal: { kind: "worker", project: id }, onError });
+worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx);
+
+// a browser tab: no address, it is a client
+const tab = createHost<P>({ onError });
+tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
+

P is your app's type of principal, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a remote object is made per principal, so a guest can get a read-only variant of the same notes.

+
Coming with the app kit

Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components today +(createWorkers, chord.http), wired by the app. The opinionated app kit (chord/next/app, being designed) will do +this wiring for you: startApp(...) and startWorker().

+
+
+ +
IV

Client Hosts

Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot.

+

A browser tab and a terminal UI are hosts like the server, with one difference: they have no address. Nobody can +send a request to a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state.

+
// a browser tab (slice 11's page host will do this wiring for you)
+const tab = createHost<P>({ onError });                               // no address, no principal: a client
+tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
+await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx);
+
+const notes = tab.hold(Notes, { project: "p1" }, ctx);               // a reference at once, live soon
+(await notes.list.watch(ctx)).start(async (call) => {
+	if (call.kind === "value") render(call.value.items);              // every change, as it commits on the worker
+});
+await notes.add({ text: "from the tab" }, ctx);                        // a call: tab → server → worker
+

Who the tab is is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +Notes factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner.

+

What a tab serves is only local types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache:

+
// plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel
+export default defineFacet((host, self) => {
+	host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) }));   // lives in this tab only
+	self.contribute(Panels, { title: "Notes", Component: NotesPanel });
+});
+

When the connection drops (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their state says loading, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from ref.state, nothing more.

+

A terminal UI is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +tui facets, rendering with whatever terminal library the app chose. What a client never does: serve a remote +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker.

+
+ +
V

Plugins

Move the code into a directory that can be added, edited and removed while the app runs.

+

So far the facets were part of the app's own code. A plugin is the same code in its own directory:

+
plugins/notes/
+  package.json        { "name": "@app/notes" }
+  contract.ts         the Notes contract (other plugins import it)
+  worker.ts           the facet for workers: serves Notes
+  server.ts           the facet for the server: an export route
+  web.tsx             the facet for browser tabs: a notes panel
+

The plugin's name is its package.json name. Each file named after a host kind is that kind's facet, its +default export a defineFacet(...). A plugin has only the files it needs: a plugin that only adds a route has a +server.ts and nothing else.

+
// plugins/notes/worker.ts
+import { defineFacet } from "@app/api/v1";
+import { Notes } from "./contract.ts";
+
+export default defineFacet((host) => {
+	host.serve(Notes, ({ project }, principal, self) => {
+		const doc = self.state(NotesDoc);
+		return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } };
+	});
+});
+

@app/api/v1 is your app's API, version 1: the part of its base code plugins import, as a host package (the +plugin's build leaves it out; every host passes its own copy in). It holds what plugins share and never reloads: +Chord's builders re-exported (local, remote, obj, str, ...), host kinds, the principal type, a defineFacet +bound to it. A plugin never imports Chord itself; the version in the path says what it was written against.

+
+ +
VI

Loading Plugins

Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it.

+

A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins' +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files, +but then every worker would compile every plugin again, and two workers started a minute apart could see two different +versions. So loading a plugin is two separate steps:

+
    +
  1. Build: turn each plugin facet (notes/worker.ts, notes/web.tsx) into one self-contained script. This +happens once per edit, on the server, for every host kind.
    2. +
  2. Evaluate: run a script inside one process, which gives back the facet, ready for host.load. This happens in +every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for +the web facets.
    3. +
+

The two steps need different things, and that is why the API has two calls.

+

Host packages: what a script leaves out

+

Look at what notes/worker.ts imports:

+
import { defineFacet, obj, remote, Session, str } from "@app/api/v1";   // the app's API: its own part and Chord's
+import { Notes } from "./contract.ts";                                  // the plugin's own file
+import slugify from "slugify";                                          // an npm package in its dependencies
+

The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first must not. +If the script carried its own copy of the API, and with it of Chord, the worker would run two Chords: the plugin's +ChordError would not be the worker's ChordError, and the plugin's defineFacet would make a facet the worker's host +does not recognize. The plugin must see the very Session host kind and the very Panels registry the process +already has, not a copy.

+

Packages the process already has, and that every plugin must share, are its host packages. A script leaves them +out and asks for them by name when it runs. Building therefore needs only their names: "do not copy @app/api/v1 +into this script".

+

Building: createPlugins, on the server

+
const plugins = createPlugins({
+	roots: [{ path: "./plugins", required: true }],      // where the plugin directories are
+	api: { name: "@app/api", versions: ["v1"] },          // the app's API and the versions it provides
+	kinds: {
+		server: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/server"] },
+		worker: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/worker"] },
+		web: { platform: "browser", hostPackages: ["@app/api/v1", "@app/api/v1/web", "react", "react/jsx-runtime"] },
+	},
+	onError,
+}, ctx);
+

createPlugins is the builder, and it builds for every kind, because only the server has the sources. For each kind +it needs two facts. The platform says where the scripts run: a node script may import node:fs, a browser +script may import a stylesheet but not node:fs. The host package names say what to leave out; they differ per +kind, because a tab also shares React with its plugins and a worker does not, and the API's per-kind modules +(/v1/web) belong to their kind only. With api set, the build checks every import of the API: a version the app +does not list, another kind's module, two versions in one facet, or Chord imported directly is a build error naming +the file and line. Nothing here runs a plugin.

+

Evaluating: plugins.load, in the process that hosts

+

Evaluating is where the names become things. When the script runs and asks for @app/api/v1, someone has to hand +it the actual module, the one this process imported when it started. Only the process itself has it:

+
// server.ts
+import * as api from "./api/v1/index.ts";                 // the app's API, the module plugins import as "@app/api/v1"
+import * as apiServer from "./api/v1/server.ts";
+import { serverFacet } from "./core.ts";
+
+const report = await plugins.load(server, {
+	kind: "server",                                          // evaluate the server kind's scripts
+	modules: { "@app/api/v1": api, "@app/api/v1/server": apiServer },   // this process's own instances
+	core: [{ name: "@app/core", facet: serverFacet }],       // base-code facets, loaded beside the plugins
+}, ctx);
+

modules is this process's answer to every host package name. The server passes its own; a worker passes its own in +workerMain; a page passes its own in startPage. Each process has different instances, which is exactly the point: +inside each process, plugins share that process's Chord and base code.

+

core is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the +process's own code, often from values only this process has (core.workerFacet(start) closes over a worker's start +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together +with the plugins, so they are published in one step.

+

Why plugins.load(host), not host.load

+

A host has its own load, and plugins.load calls it. The host's load takes facets that already exist: functions +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing +is built or evaluated at runtime.

+

The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them +with the process's modules, and then calls host.load with the result. It also remembers, per host, which facet each +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very +same objects. That memory is why the host is passed to it: plugins.load(server, …) on the server, +plugins.load(worker, …) in a worker (where workerMain gives it a plugins that only evaluates), the same call +in every process.

+
SERVER: BUILD (ONCE PER EDIT)EACH WORKER: EVALUATEplugin sourcesnotes/worker.ts`, imports`@app/api/v1createPluginscopies own files and npm deps; leaveshost packages outworker script`require("@app/api/v1")` left as anamethe worker's modulesits own `@app/api/v1`, its own Chordplugins.load(worker, …)runs the script, answers each namethe facetuses the worker's `Session`, theworker's `defineFacet`shipped
Figure 3. Building on the server needs names; evaluating in each process needs that process's modules.
+

What happens, step by step

+
plugins.load(server, { kind: "server", … })

Discover

The component lists ./plugins and reads each package.json. Its name is the plugin's name; the files named after +a kind are its facets. A directory without a package.json, or with two server files, is reported, not loaded. Files +at the root's top named <name>.<kind>.ts (hello.server.ts) are single-file plugins, grouped by name.

+

Build

For each plugin with a server facet it builds one script, if no current one exists. A syntax error or an import the +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file +and line, never a crash.

+

Evaluate

The script runs. Each host package name it asks for is answered from modules; anything else it may not have. The +script's default export must be a facet.

+

Load

host.load runs the setups of core and of every plugin facet and publishes them together. A plugin whose build, +evaluation or setup failed is left out and listed in report.failed with the reason; the others load. Objects that +need it wait until a later edit loads it.

+
+

Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from workerMain, a page's from its base bundle).

+
+ +
VII

Plugins Using Plugins

Two ways one plugin uses another, and the one it may not.

+

A search plugin wants to search the notes. It imports the contract and asks for the object:

+
// plugins/search/worker.ts
+import { Notes } from "../notes/contract.ts";   // a plugin of the same root, by relative path: copied into search's script
+
+host.serve(Search, ({ project }, principal, self) => {
+	const notes = self.get(Notes, { project });   // a dependency: Search goes live once Notes is
+	const index = self.derive([notes.list], ([list]) => buildIndex(list.items));
+	return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } };
+});
+

Which file a plugin imports decides which version of the contract it claims, so the rule is: a plugin carries the +contract it was written against. Two cases:

+
    +
  • Plugins that change together (the app's own, in its repository; a user's, in one root) import each other by +relative path, ../notes/contract.ts: the same commit holds both sides. The file is copied into the importer's +script, so only contracts and other side-effect-free files belong there (a copied file may import only the API and +files of its own plugin). A relative path leaving the root (into base code, into another root) is a build error.
    • +
  • Plugins from another repository take contracts as a dependency at a fixed version, by package name: +@bob/notes (its contract.ts, through the package's exports), @bob/notes/web (its contract.web.tsx), pinned +in package.json by an npm version or a git ref ("@bob/notes": "github:bob/notes#<commit>"). The build inlines +the installed version: when Bob's notes move to contract version 2, a consumer built against version 1 keeps +claiming version 1, and the provider serves it through accepts or the consumer waits "contract mismatch", visible, +never silently wrong. A plugin inside a repository holding several (which lists them in its own package.json, +"chord": { "plugins": ["./notes", "./search"] }) is a dependency on the whole repository, imported by path into it: +@bob/plugins/notes/contract.ts.
    • +
+

A contract file may be for one kind: contract.web.tsx (React components in a type, a panel's registry token) is a +web file. Importing it, directly or through other files, from a facet of another kind is a build error naming +the chain (server.ts → view.ts → ../notes/contract.web.tsx). The shared contract.ts mentions native things only +through import type.

+

self.get in a factory declares a dependency: Chord makes Search live only once Notes is, and if the notes +plugin is missing, Search waits with a clear reason ("waiting for notes: not served here") instead of failing.

+

The other way is a registry: an extension point one plugin defines and others contribute to. The app's API +defines Panels in its web module, and every plugin with a UI contributes one:

+
// plugins/notes/web.tsx
+import { Panels } from "@app/api/v1/web";
+export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel }));
+

What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the Notes +object's state. That rule is what lets the notes plugin change its storage without breaking search.

+
+ +
VIII

Changing a Plugin

Edit, build, roll out: how new code takes over while every tab stays connected.

+

You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out:

+
const built = await plugins.build(plugins.state.value.dirty, ctx);
+const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx);
+

The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording self to see what each object would declare, and +then gives every object of the changed plugin its fate:

+ + + + + + + + + + + + + + + + + + + +
The objectWhat happens
holds no natives (open files, sockets), the new code neitherswapped in place: new methods, same documents; running calls finish on the old code
holds nativesrestarted: calls wait, running ones finish, natives close and reopen with the new code
its type was removedended; its users wait until something serves it again
+

The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies.

+
Versions

A change to a contract (a new field in Notes's state) gets a new version. A tab built with the old contract keeps +working if the provider maps the old version (accepts); otherwise it waits with "version mismatch" until it reloads.

+
+
+ +
IX

Base Code and Plugins

What goes into the app itself, what goes into plugins, and how an app ships plugins of its own.

+

Every app has two kinds of code, and the line between them decides what can change while the app runs.

+

Base code is the app's own program: src/, started by node server.ts. It shrinks to three things: the API +(the versioned modules plugins import), the bootstrap (creating hosts, opening the store, components such as +createHttp and createWorkers) and one small start-data facet serving process values (start data, the opened +store) as core objects. Base code changes only with a restart: if it reloaded, the process would hold two Panels +registries, and old and new plugins would no longer meet.

+

Plugins are everything else: notes, search, export, a terminal, the harness, the providers of the app's core +registries. They reload on an edit, a broken one is left out alone, and an agent can write one while the app runs.

+

The API, by version and by kind

+

Plugins import their host only through the API, a version in the path:

+
// src/api/v1/index.ts: "@app/api/v1", for every kind
+export { defineRegistry, local, obj, remote, str } from "@earendil-works/chord/next";   // the Chord surface the app supports
+export { defineFacet, Project, Server } from "../../core.ts";                           // bound to the app's principal
+// src/api/v1/web.ts: "@app/api/v1/web", for pages only
+export const Panels = defineRegistry<Panel, readonly Panel[]>("app.panels", { version: 1 });
+

The app lists the versions it provides, newest first (api: { name: "@app/api", versions: ["v2", "v1"] }), and each +kind's modules in kinds.<kind>.hostPackages. When the app changes its API in a way that breaks plugins, it adds +/v2 and keeps /v1 as a thin layer over it for a while; once /v1 goes, a plugin importing it fails to build with +"@app/api/v1 is not available: this app provides v2", the plugin alone, the message saying what to change. One facet +uses one version. Contracts the app promises others are re-exported in the API (export { Notes } from "../../plugins/notes/contract.ts"), so a plugin built against /v1 keeps v1's contract files.

+

Plugins the app ships

+

An app comes with plugins of its own. They follow the same rules as anyone's, in a root inside the app, next to the +root where users and agents add theirs:

+
const plugins = createPlugins({
+	roots: [{ path: "./plugins", required: true }, join(dataDir, "plugins")],   // the app's own, then the user's
+	api: { name: "@app/api", versions: ["v1"] },
+	kinds,
+	onError,
+}, ctx);
+

They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left out +alone when broken at a reload. The app's root is required: a plugin of it failing at startup makes +plugins.load reject with a PluginError naming it (after the load settled), so the app stops instead of running +without its core. A failure in the user's root is reported and the app runs on. If two roots have a plugin with the +same name, the first root wins, and the other is reported.

+

Small plugins, installed plugins, tested plugins

+
    +
  • A single file at a root's top is a plugin: plugins/hello.server.ts and plugins/hello.web.tsx are the plugin +hello with two facets. It has no package.json and imports only the API; when it needs a second file or a +package, it becomes the directory hello/, which a reload treats as the same plugin.
    • +
  • Installing from git or npm: plugins.installPlugin({ git: url, ref }, root, ctx) clones the ref (a branch, tag +or commit) into the root, { npm: name, version } extracts the package; then its dependencies are installed. Removal +is deleting the directory.
    • +
  • Testing a plugin alone: testPlugin("./plugins/stats", { kind: "worker", api, modules, stubs }, ctx) builds it +as production does, loads it on an in-memory host beside stub facets serving what it uses, collects what it +contributes to registries nobody serves, and lists the contracts it waits for (docs/next/plugin-authoring.md).
    • +
+
+ +
A

Reference

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallWhat it does
createHost<P>({ address?, principal?, store?, onError })a host: addressable with an address, else a client
host.load([{ name, facet }], ctx) / host.reload(...)publish facets / apply a new plugin set
host.get(Type, params, ctx) / host.hold(...)a live reference / a reference at once that follows its object
local(name, { version, params, members }) / remote(name, { version, params, hostAddress, variants })contracts
defineFacet((host, self) => …), host.serve(Type, factory), self.contribute(R, value)a facet
factory self: state, derive, get, registry, open, on, onLive, job, txan object's parts
createPlugins({ roots, api, kinds, onError }, ctx)the plugins component
plugins.load(host, { kind, core, modules }, ctx), build(names), rollout(steps, built)load, build, roll out
plugins.installPlugin(source, root, ctx), testPlugin(dir, options, ctx)install a plugin, test one alone
hostKind(name, params?), Kind(params)host kinds and addresses
+
+ +

Hosts and Plugins · second edition · 2026-11-01 · Chord v2 slices 1-10 and 12

+
+ + diff --git a/packages/chord/docs/next/http-workers/book.md b/packages/chord/docs/next/http-workers/book.md new file mode 100644 index 000000000..f340004ef --- /dev/null +++ b/packages/chord/docs/next/http-workers/book.md @@ -0,0 +1,378 @@ +--- +title: Serving and Spawning: HTTP and workers in Chord v2 +cover: Serving / and Spawning +kicker: An explainer · Chord v2 slice 10 · 2026-10-31 +subtitle: How a request reaches a plugin's route, who it acts as, and how a worker process per session starts, stops and fails +stats: + - 1 | registry plugins contribute routes, wraps, sockets and hosts to + - 0 | automatic worker restarts + - 5 min | idle before a worker stops (default) + - 6 | security rules on by default +note: | + Written by the redesign session from `slice-10.md`, the decision entries "Slice 10's points", "Origins only from + relays", "The HTTP component's security baseline" and their neighbours in `decisions.md`, design §3 and §23, and the + build after its four fix rounds (security baseline, `fetch` core, error boundaries). +colophon: Serving and Spawning · first edition · 2026-10-31 · Chord v2 slice 10 +--- + +# Before we begin {.preface} + +Mario runs a small chat app on his laptop. Each conversation is a **session**, and each session runs in its own +process, a **worker**, so a session that crashes or eats memory takes nothing else down. A browser tab talks to the +server; the server talks to the workers. Plugins add features: one adds a chat panel, one adds file downloads, one +adds a terminal. Mario edits plugins while the app runs, and sometimes an agent edits them for him. + +Three things have to work for him without thought. A plugin adds an HTTP route, and it is served safely: no other +website open in his browser can read his data or act as him through it. A session's worker exists exactly while +someone uses it. And when a worker crashes, nothing restarts it in a loop: he sees why it failed and decides. + +Slice 10 builds two components for this: `chord/next/http` (the server's HTTP and WebSocket entry) and +`chord/next/workers` (a worker process per session). Both are Node code on top of Chord's kernel; the kernel itself +stays portable. + +## Eight things to know + +:::findings +1. **Plugins contribute routes, Chord serves them.** The HTTP component defines one registry, `Http`; plugins + contribute routes, wraps (middleware), WebSocket handlers and allowed hosts to it, and every reload swaps them. +2. **A route returns a description.** JSON, HTML, bytes, a redirect or an error; the component writes it, with ranges, + `HEAD`, download headers and backpressure done once for every plugin. +3. **Security is built in.** Where a request came from, whether another website sent it, and which headers every + response carries are decided by the component before any plugin code runs. The app only says who someone is. +4. **A worker starts on the first use of its session** and stops after five idle minutes. The server forks it; the + worker connects back and loads its plugins. +5. **A worker never restarts by itself.** A crash or a failed start leaves the session failed with its error until a + person presses Retry or rolls out new code. +6. **Workers never build plugins.** The server builds every plugin once; a worker evaluates the scripts it is sent. +7. **A rollout reaches every host.** The server reloads first, then every running worker, each reporting its result. +8. **Everything but the bootstrap is changeable by plugins.** Routes, hosts, start data, idle times: contributions or + functions read at each use. +::: + +## Words used throughout + +A **route** is a function a plugin contributes for a method and path pattern. A **wrap** runs around routes. A +**socket** is a WebSocket a plugin handles with its own protocol; a **link** is Chord's own WebSocket at `/chord`, +over which tabs hold objects. **Admission** is what the component checks before any plugin code. **Provenance** is +where a request came from (local, an allowed host, a relay). A request is **foreign** when a browser sent it for +another website. A **worker** is a process serving one **address** (`Session({ id })`). A worker's **start message** +carries its start data and its plugins' **scripts**. A **rollout** applies a build to the server and then the workers. + +# The App We Build {#app lede="A server, workers, two plugins: the files a reader needs to follow the rest."} + +The whole app is four files of its own plus its plugins. The server creates its host, the HTTP component and the +workers component, then loads its plugins: + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +const plugins = createPlugins({ + roots: ["./plugins"], + kinds: { + server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + }, + onError, +}, ctx); +const http = createHttp(server, { listen: { host: "127.0.0.1", port: 8788 }, onError }, ctx); // settings are arguments +const workers = createWorkers(server, Session, { + kind: "worker", entry: "./worker.ts", plugins, principal: (a) => ({ kind: "worker", session: a.params.id }), + start: (address, ctx) => ({ id: address.params.id, dir: sessionDir(address) }), + lock: (address) => `chat:${dataDir}:${address.key}`, authorize: mayReach, onError, +}, ctx); +await plugins.load(server, { kind: "server", core: [core, { name: "chord.http", facet: http.facet }], modules }, ctx); +// `core` contributes the app's authenticator: self.contribute(Authenticators, authenticate) +``` + +```ts +// worker.ts: one per session +await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(start.dir, "session.sqlite"))), ctx); + onClose((ctx) => store.close(ctx)); + const worker = createHost

({ address, principal: { kind: "worker", session: start.id }, store, onError }); + onClose((ctx) => worker.close(ctx)); + worker.connect(parent, { principal: { kind: "server" }, trusted: true }, ctx); + await plugins.load(worker, { kind: "worker", core: [workerCore(start)] }, ctx); + return { busy: { tasks: 0 } }; +}, { hostPackages: modules, redact: (error) => ({ message: "worker failed" }), onError }); +``` + +A plugin is a directory with a `package.json` and one file per kind it needs. The chat plugin keeps its messages in a +worker (each session has its own) and contributes one route on the server for exporting them: + +```ts +// plugins/chat/server.ts +export default defineFacet((host, self) => { + self.contribute(Http, { + routes: { + "GET /s/:session/export": async (request, ctx) => { + if (!(await mayView(ctx.principal, request.params.session, ctx))) return { kind: "error", status: 403 }; + const chat = await ctx.get(Chat, { session: request.params.session }); // reaches the session's worker + return { kind: "json", value: await chat.export(ctx) }; + }, + }, + }); +}); +``` + +`ctx.get(Chat, { session })` is the one line that connects the two halves: `Chat`'s host address is the session's +worker, the server's route for `Session` finds (or starts) that worker, and the call goes over its link. + +# A Request, Start to End {#request lede="What happens between a TCP connection and the route's response, in order."} + +Every request takes the same path. The order matters: nothing a plugin wrote runs before the component decided that +the request may run at all, and as whom. + +:::flow wide animate | One request through the HTTP component. Everything left of `authenticate` is the component's own; plugins start at the router. +row "Admission" accent +box prov "Provenance" "local, allowed host or relay; else 403" +box foreign "Foreign?" "another website's browser request" +box origin "Origin" "foreign Origin 403; unsafe methods need ours" +row "Who" blue +box auth "authenticate" "the app: cookies, devices" +box anon "anonymous" "foreign or no credential" muted +row "Plugins" +box router "Router" "pattern → route, as of now" +box wraps "Wraps" "observe, headers, transform" +box route "Route" "returns a description" +row "Answer" muted +box write "Write" "headers, ranges, chunks, backpressure" +arrow prov -> foreign +arrow foreign -> origin +arrow origin -> auth +arrow origin -> anon "foreign" +arrow auth -> router +arrow router -> wraps +arrow wraps -> route +arrow route -> write +::: + +**Provenance** answers "where from". A request is local only if three things hold: the socket arrived on a loopback +address, the `Host` header names loopback with the port, and no proxy header (`Forwarded`, `X-Forwarded-*`) is present. +Anything else must name a host the app allowed (a tunnel's public name) and is then treated as remote https. Anything +else is refused before a single plugin line runs. The security audit found that checking the `Host` header alone let +anyone reaching the port claim to be local: the socket address is the part a remote client cannot fake. + +**Foreign** answers "did another website make the browser send this". Browsers say so in `Sec-Fetch-Site`. A foreign +request never reaches `authenticate`: it runs anonymous, whatever cookies it carries. That one rule stops a page on +another site from reading Mario's data through an `` or ` + diff --git a/packages/chord/docs/next/layer-1.md b/packages/chord/docs/next/layer-1.md new file mode 100644 index 000000000..0129181df --- /dev/null +++ b/packages/chord/docs/next/layer-1.md @@ -0,0 +1,311 @@ +# L1: data and contracts (spec) + +Status: **approved (owner, 2026-10-22)**, rewritten 2026-10-22 as the single source for a from-scratch implementation +(the first implementation was discarded for code quality). Every rule here is decided; the decision log +(`packages/chord/docs/next/decisions.md`) has the reasons. Code style: the section "Chord v2 Code Style" of the root +`AGENTS.md`. + +## What L1 is + +Everything about **data** that the upper layers need and that does not depend on hosts, objects or links: how an +object type is declared and evolves (contracts), what a value is, how params become an identity, how arguments are +checked, what a principal key is, what an error is. Pure functions, types and weak caches; no I/O. + +Source: `packages/chord/src/next/contracts/` (declarations) and `packages/chord/src/next/data/` (per-call checks, +principals, errors), split into files by concept. Everything else under `src/next/` is the frozen old kernel: never +imported, never edited. Reused: `src/context/` (`Context`, `BACKGROUND_CONTEXT`, `context.telemetry`) and +`src/json.ts` (`JsonValue`), both unchanged. Schemas were TypeBox 1.3 when L1 was built; since the schema switch (2026-11-01) they are pi-schema's plain JSON Schema (design §3 "Values"). The TypeBox details below are historical. + +## 1. Values + +```ts +type Value = null | boolean | number | string | Uint8Array | Value[] | { [key: string]: Value }; +``` + +- JSON plus byte leaves. No `undefined` inside values, no non-finite numbers, no foreign prototypes, no accessors. +- Values from links are already strict JSON plus bytes; values from same-host code are plain data **by convention**, + immutable once handed over. L1 never walks, copies or freezes a value to enforce that. State is never checked. +- `canonicalJson(value: JsonValue, context)`: sorted object keys, `-0` written as `0`. Used only for principal and + revocation keys, which are JSON (no bytes), so it lives with principals. + +## 2. Contracts: declarations + +A contract is the declared type of an object: name, kind, version, params, members. Two kinds: + +```ts +const Notes = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note) }, methods: { add: method(obj({ text: str() }), str()), clear: method() } }, + reader: { state: { notes: arr(PublicNote) } }, + }, +}); +const Info = local("app.session-info", { + version: 1, + params: {}, + members: { init: obj({ title: str() }), clear: method() }, // a schema is state, method(...) is a method +}); +``` + +- `remote` has variants (per-principal views, at least one); `local` has one flat `members` map and no variants. +- `method()`, `method(args)`, `method(args, result)`, `method(undefined, result)`. An omitted side is `null` on the + wire: a no-argument method is called with `null`. +- **Variants (D11):** a member name has the same kind (state or method) in every variant that has it; a method has the + same signature (args and result) in every variant; state schemas may differ per variant (redaction). Enforced in the + types of `remote(...)` and at declaration (for untyped callers). +- **Closed argument objects (Q12):** every object schema inside a method's arguments sets + `additionalProperties: false`, nested ones included, raw `Type.Object` included; record schemas (`map`) are allowed. + Checked once per declared or edited method, at declaration. `obj()` always produces closed objects and cannot be + opened. +- Declaration checks (the types cannot express them for JavaScript callers): nonempty name, version a positive safe + integer, no unknown declaration keys, params policy (section 4), D11, closed arguments, the schema vocabulary + (below). Each fails with an `Error` naming the path (`notes.editor.methods.add.args`). +- Declarations are immutable by convention: a contract holds Chord-owned snapshots of its schemas (copied once at + declaration, so later changes to the app's schema objects cannot desync validation from the fingerprint), never the + app's schema objects. Nothing in L1 is frozen (owner, 2026-10-22). Contracts are branded: hand-built or spread copies are refused by `evolve`, `checkAccepts` + and `bindProvider` (a private `WeakMap` from contract to its data). +- **Placement** is not part of L1 contracts: the `placement` field arrives with host kinds (L7) and is never part of + the fingerprint. + +### Schema helpers + +`str({ min, max, pattern })`, `num({ min, max })`, `int({ min, max })`, `bool()`, `nul()`, `lit(v)`, `oneOf(...literals)`, +`union([...])`, `obj(props)`, `arr(items, { min, max })`, `map(value)` (string-keyed record), `opt(schema)`, +`bytes({ min, max })`, `unknown()`, and `type Static`. They return ordinary TypeBox schemas, so `Static` and TypeBox's +compiler work unchanged. `min`/`max` mean string length, numeric bounds or array length. + +`bytes()` is a `Uint8Array` leaf with optional length bounds, checked by `instanceof` and length, never enumerated. +Its enumerable form is the data marker `{ type: "bytes", minLength?, maxLength? }`; the TypeBox check is attached +non-enumerably and rebuilt from the marker whenever a schema is snapshotted, so no callback is ever copied from app +data. + +**Vocabulary (rejected at declaration otherwise):** data primitives, arrays and tuples, objects and records, unions and +intersections, `$defs`/`$ref` (use `$ref` for recursion), constraints (`pattern`, `format`, bounds, `multipleOf`, +`uniqueItems`, property counts), `default`, `readOnly`/`writeOnly`, annotations, bytes. Rejected: conditional and +dependent schemas, `not`, `contains`, `propertyNames`, `unevaluated*`, `contentSchema`, `definitions`, unknown +keywords, TypeBox's executable features (`Unsafe` other than bytes, refinements, transforms), accessors, symbol keys, +cyclic objects, non-JSON data. + +### Fingerprint + +Every version gets a fingerprint at declaration: `hash64` (a portable 64-bit hash, two FNV-style lanes, 16 hex +characters; not a security boundary) of `JSON.stringify({ normalFormVersion, value })` over a **normal form**: + +- object keys sorted; `required` and `enum` sorted; union, intersection and tuple order **kept** (it is meaning); +- annotations dropped on schema nodes only (`title`, `description`, `examples`, `$comment`, `deprecated`); defaults, + formats, patterns, bounds and `additionalProperties` kept; +- params as ordered `[fieldName, schema]` pairs (declaration order is identity order); members as sorted dictionaries; +- `{ kind: "remote", params, variants }` or `{ kind: "local", params, members }`; placement excluded. + +`NORMAL_FORM_VERSION = 2` is exported; changing the normal form changes it and is a link-protocol change. +One schema-aware walk produces both the snapshot and the normal form. + +### Evolution (Q10) + +A history starts with a base written in full (`remote()`/`local()` with an explicit version). `evolve(prev, (e) => +e...)` appends the next version (`prev.version + 1`) with structured edits: + +| Edit | Meaning | +| --- | --- | +| `state(variant, name, schema)` | change one state member (local: `state(name, schema)`) | +| `addState(variant, name, schema)` | add a state member | +| `removeState(variant, name)` | remove a state member | +| `args(name, schema)`, `result(name, schema)` | change one side of a method in every variant that has it | +| `method(name, args, result)` | replace both sides | +| `addMethod(variant, name, args, result)` | add a method (local: no variant); to another variant only with the same signature | +| `removeMethod(name)` | remove a method from every variant | + +- Params and the variant set never change by evolution. A change to either is a **new history base**: a new full + declaration with an explicit version and no accepted older versions. +- A member's kind never changes, also not after a removal and re-add. Checks run after every edit. +- `evolve` must return the builder it was given (no foreign builders). Every version is materialized and + fingerprinted at declaration, so an old plugin's copy of v1 has the same fingerprint as the history's v1. +- The types track the whole history: `Contract` carries the version, the current description and every step with its + edits, so `checkAccepts` can require exactly the right mappings at compile time. + +### Accepted versions and mappings + +A provider may accept older versions of its contract with mappings it owns: + +```ts +const NotesV2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2)).args("add", NoteV2)); +const accepts = { + 1: { + state: { editor: { notes: (current) => current.map(toV1) } }, // current member -> old member + methods: { add: { args: (old) => toV2(old) } }, // old args -> current args + }, +}; +checkAccepts(NotesV2, accepts); // { versions: [{ version: 1, fingerprint }] } +``` + +`checkAccepts(contract, accepts)` is pure: it validates the table without calling any mapping and returns the accepted +versions with their fingerprints. Mappings are **direct** (old to current, never chained). For accepted version `v`, +considering every edit after `v`, a mapping is required exactly for: + +- a changed state member: `(currentMember) => oldMember`; +- a removed state member: `(currentVariantState) => oldMember` (the whole current state of that variant), or + `"refuse"`; +- a method whose args changed: `{ args: (old) => current }`; whose result changed: `{ result: (current) => old }`; + both: both functions; +- a removed method (from any variant that had it in `v`): `{ call: async (oldArgs, context) => oldResult }` or + `"refuse"`; `call` and `"refuse"` are also allowed for changed methods. + +Added members need nothing. No-op edits and changes made and reverted still require (identity) mappings: the check is +by edit, not by schema comparison. Unknown versions, the current version, missing mappings, unnecessary keys and wrong +shapes fail at declaration with the path (`notes accepts[1].methods.add`). An absent `state` or `methods` section means +empty and is allowed only when nothing is required there. The same rules hold at the type level (`AcceptsTable`). + +`Implementation` derives the provider's shape: per variant, state getters `() => value`; methods +`(args, context) => Promise`. + +## 3. Principals + +- The app's principal type `P` is plain JSON with a `kind` field: no schema, no version, never persisted by Chord + (objects persist their own projection, e.g. chat's `Author`). +- `principalKey(principal, context, customKey?)`: default `canonicalJson(principal)`; an app may pass its own key + function, which must be the same on every host. Cached per principal object in `WeakMap`s; a custom key function's + cache is keyed by the function too, so a dropped function is never retained by a live principal. +- `revocationKeys(principal, revokedBy, context)`: the canonical JSON of each JSON value `revokedBy(principal)` + returns (`host.revoke` is L7). + +## 4. Params and identity + +```ts +const { params, key } = checkParams(Notes, { session: "S1" }, context); // throws invalid/params +const identity = identityKey(Notes, key, principalKey(principal)); // "notes|S1|" +``` + +- Declaration policy: params are a flat record of required fields, each `str()`, `int()`, `bool()` or a union of + string literals (`oneOf("chat", "agent")`), constraints allowed; no defaults, optional, nested, bytes, numbers other + than integers. Checked in the types and at declaration. +- `checkParams(contract, raw, context)`: one compiled closed-object check of `raw` (constraints enforced), the compiled + validator cached per contract in a `WeakMap`. Then the key is the field values in declared order, each escaped + (`\` and `|`), joined with `|`. It returns the caller's own object: no pre-check walk, no copy, no freeze. +- `identityKey(contract, key, principalKey)`: `name|key|principalKey`, each part escaped; `local` takes no principal + key (by type). The key is an in-process map key; links carry the params themselves. + +## 5. Arguments + +```ts +const provider = bindProvider(NotesV2, accepts); // once per registration and reload +const checked = checkArgs(provider, "add", raw, boundVersion, ctx); // invalid/args with path +// { kind: "method"; args (current shape); result?: (current) => old } | { kind: "call"; args (old shape); call } +checkArgs(provider, "clear", null, 2, ctx); // no-argument methods take null +``` + +- `bindProvider(contract, accepts)` runs `checkAccepts` once and keeps the app's accepts table as it is (immutable by + convention; no copy, no freeze). It builds a private dispatch table, held in a `WeakMap` keyed by the returned + provider: accepted version -> method -> `{ schema, validator, mapping }`. Validators compile on first use per + (version, method); a weak cache keyed by schema object lets equal schema objects share a compiled validator. +- `checkArgs(provider, method, raw, boundVersion, context)`: three map lookups, one compiled check against the + provider's schema for that version, then at most one direct `args` mapping old -> current. It returns a tagged + value (owner, 2026-10-22): `{ kind: "method"; args; result? }` (args in the current shape; `result` maps the current + result back for an old-version caller) or `{ kind: "call"; args; call }` (a replaced method: the checked old args + and the provider's `call`, which L4 invokes). Current-version calls are never mapped. Unknown method: + `invalid/member`. Version not accepted, or `"refuse"`: `contract_mismatch`. Mapping output is trusted. +- Every caller is checked, `local` included (Q15). Unknown fields fail (closed schemas); nothing is stripped, copied, + frozen or defaulted. Results and state are never checked. +- Memory: a binding lives as long as its registration and the calls in flight. No module-level strong cache holds a + contract, binding, schema or validator. +- TypeBox keeps its last build in module-level accumulators: after every `Compile`, reset them (`ResetFunctions`, + `ResetExternal`, in `finally`), so no build state is retained. + +### A failed check + +A failed compiled check uses TypeBox's public `Errors(schema, value)` and reports the first error. The error is +`invalid/args` or `invalid/params`, message `args/text: ` (fixed text for the two common cases: +`args/extra: unknown field`, `args/text: missing`), `details: { path }` with TypeBox's +instance path (`/text`, `/` for the root), including missing (`required`) and unknown (`additionalProperties`) +fields at their own path. TypeBox does not escape `/` and `~` inside keys, so a key containing them makes the path +ambiguous (owner, 2026-10-22: accepted; it is a diagnostic). + +## 6. Errors + +```ts +type ChordErrorCode = "invalid" | "contract_mismatch" | "forbidden" | "gone" | "not_ready" | "released" + | "too_large" | "unavailable" | "aborted"; +``` + +| Code | Reasons | +| --- | --- | +| `invalid` | required: `args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle` | +| `contract_mismatch` | optional: `updating` | +| `unavailable` | optional: `locked`, `park_timeout` | +| `too_large` | optional: `part` | +| others | none | + +- One `ChordError` class (an `Error`), with `code`, `reason`, `message`, `details?: JsonValue`. The constructor, + instances and JSON form are tagged unions: a wrong pairing is a type error, and narrowing on `code` narrows + `reason`. `toJSON()` emits `{ code, reason?, message, details? }`; stack and cause never cross the wire. +- `PROTOCOL_VERSION = 1` is exported. Reasons and pairings are link protocol: adding or moving one bumps it. A test + pins the table together with the version and fails with "bump PROTOCOL_VERSION" when they diverge. L6 compares the + version in the handshake before accepting anything. +- `chordErrorFromJson(raw)`: `raw` is fresh JSON from the link decoder (a trust boundary). Checks the envelope only: + known code, a reason allowed for that code (required for `invalid`), string message, no other keys. `details` is + kept by reference: no walk, no copy, no counters. +- Helpers: `isChordError(error, code?)`, `invalid(reason, message, details?)`, `asChordError(error, label)` (wraps a + foreign error as `unavailable`). +- `FailureCause = { kind: "transient"; error } | { kind: "broken"; error }`: the failing stage decides, never the + error code. `broken(cause)` marks an open's failure as broken (bad configuration); `openFailure(cause, label)` turns + an open's thrown value into the cause. Used by L4. +- Plugin errors have their own shape (L4); they never use Chord reasons. + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a `get` | one compiled closed-object check, one joined key; no walk, no copy | +| a call | three map lookups, one compiled check, at most one direct mapping; no copy | +| first `get` of a contract | one compilation | +| first call per accepted version and method | one compilation (or a cache hit by schema object) | +| a failed check | one counted diagnostic walk (`Errors` builds the whole list on this rare path) | +| a first default principal key, a revocation key | one canonical walk | +| declaration | snapshot, closure check per method, fingerprint per version | +| binding | `checkAccepts` once, dispatch table built, nothing compiled | +| a state change | nothing in L1 | + +**Counters:** `chord.data.walks`, `chord.data.validations`, `chord.data.compilations` are telemetry counters +(`context.telemetry.metrics?.counter(name).add(1)`), always on. The function doing the work counts it; call sites never +count for others. Tests assert them per hop with the in-memory telemetry adapter. + +## Footguns + +- A provider must not mutate arguments or params it received (caller-owned, immutable by convention). +- `principalKey` must be the same function on every host; a different one splits identities silently. +- Params keys must escape the separator, so two different params never join to one key. +- Mappings are direct: when version 3 arrives, every still-accepted version's mappings are updated (compose step + functions: `(old) => v2to3(v1to2(old))`). + +## Tests (one group per mechanism, every case) + +- Contracts: every declaration check with its error path; D11 (kind and signature across variants, at runtime and + in the types); closed arguments (nested, raw `Type.Object`, records allowed, edits checked, state not checked); + schema vocabulary (each rejected feature); bytes (marker, bounds, check rebuilt after snapshot, no callback copied); + snapshots are independent of later changes to the app's schema objects; branding (spread copies refused). +- Fingerprints: stable across key order and annotations; sensitive to defaults, bounds, union order, params order, + `additionalProperties`; the same v1 from a separate declaration; local vs remote differ; a fixed known hash for one + fixed contract (protects the normal form). +- Evolution: each edit; kind changes refused (also remove then re-add); params and variant set fixed; new bases; + long chains (versions beyond 9, 99); history types. +- Accepts: every required-mapping rule above, at runtime and at the type level (`@ts-expect-error` for missing, + unnecessary and wrongly typed mappings); `"refuse"`; `call`; direct mapping over several versions; no mapping is + invoked by `checkAccepts`. +- Params: missing, extra and wrong fields with their paths; literal membership; constraints; the same key whatever + the property order; separator escaping (no two params join to one key); declaration refusals. +- Arguments: unknown fields (nested too); no copy (identity of the passed object); `local` and `remote`; bytes by + `instanceof` and length; old version checked with the old schema then mapped; current never mapped; `null` for + no-argument methods; unknown method; unaccepted version; `"refuse"`; lazy compilation (counted once per version and + method). +- Principals: default key, custom key, caching (no second walk), revocation keys. +- Errors: tagged union in types (`@ts-expect-error` for wrong pairings); JSON round trip; envelope checks (each + refusal); details kept by reference; the pinned protocol table; `FailureCause`, `broken`, `openFailure`. +- Cost guards: the counters per hop in the table above. +- GC: dropped contracts, bindings, dispatch tables, mappings, schemas and validators are collected after the last call + settles; params validators per contract; custom principal-key functions dropped while their principals live; nothing + retained by TypeBox's build accumulators. + +## Size + +Target: well below 1,500 source lines for `contracts/` and `data/` together, tests excluded. The type-level history +and mapping types are the heaviest part; keep the rest lean. diff --git a/packages/chord/docs/next/layer-2.md b/packages/chord/docs/next/layer-2.md new file mode 100644 index 000000000..e5dc5dee2 --- /dev/null +++ b/packages/chord/docs/next/layer-2.md @@ -0,0 +1,216 @@ +# L2: watchables and derives (spec, for the owner's approval) + +Status: **superseded (owner, 2026-10-22)**: the L2 code built from this spec was discarded (too much generic +machinery for callers that did not exist yet: nodes, sinks, deliveries, publications) and the build switched to +vertical slices (`slices.md`). Still valid: the section "Decided while specifying" below, and Chord's shared watch +handle (`src/watch.ts`). Everything else here is background only; do not implement from it. + +Original status: approved; open points decided (below). The single source for implementing L2; every rule comes from the decision log +(`packages/chord/docs/next/decisions.md`: "Watchable and one watch API", W1, W2, W3, W4, W12, W14, Q14, R23, R24, +"Changing contributions, `self.writable`", "Cost counters") unless marked **Decision needed**. Code style: the section +"Chord v2 Code Style" of the root `AGENTS.md`. + +## What L2 is + +Everything observable on one host, and how a change travels: a value changes inside a **publication** (one atomic step +that may change many sources), the derives over it are recomputed right after, and every consumer sees each +publication once, completely, in order. No objects, no storage, no links: L3 (store) publishes document commits +through it, L4 (objects) builds members from it, L6 (links) encodes what it flushes, L9 (registries) composes from it. + +Source: `packages/chord/src/next/watch/` (one directory; files by concept). Reused unchanged: Chord's shared +`src/watch.ts` (the watch-handle types **and implementation**, moved there from Durable's `CommittedWatch` by a pi change +first, owner 2026-10-22; Durable uses the same implementation), `src/delta/` (`Op`, `diffRevisions`), `src/context/`, +`src/json.ts`, and L1's `next/data` errors (`invalid/reentrant`). Nothing from `next-frozen/`. + +## 1. The two kinds + +```ts +interface Watchable { // documents, derives, reference members, constant(v), Durable views + readonly value: T; + watch(context: Context): Promise>; // listener (value, ops, context) +} +interface NativeWatchable { // writable(initial), composed registry values + readonly value: T; + watch(context: Context): Promise>; // listener (value, context) +} +``` + +- A `Watchable` carries JSON and changes with ops (Delta's `Op[]`): the ops turn the previous value into the next. + Values are immutable by convention: a new value shares unchanged parts with the old one; nobody mutates a value. +- A `NativeWatchable` carries any in-process value (functions, runtimes); a change is a new value by identity + (`!==`). Mutating a native value in place notifies nobody. It has no ops and can never be a contract state member or + go on the wire (Q14, W4). +- Both carry a runtime kind tag (`"json"` / `"native"`), so L4 can refuse a native one as state with one comparison + (no value walk). + +## 2. Publications (W1, W12) + +```ts +const publication = createPublication(); // one per host +publication.publish(() => { // one atomic step + notes.set(nextNotes, ops); // adopt: pointer swap + mark + index.set(nextIndex, indexOps); +}); // flush runs here, synchronously, before publish returns +``` + +- `createPublication()` returns the host's publication. A **source** (`jsonSource(initial, publication)`, + `writable(initial, publication)`) belongs to one publication. +- `publish(adopt)` runs `adopt` synchronously. Inside it, `source.set(value, ops)` (JSON) or `source.set(value)` + (native) swaps the value and **marks**: dependent derives become dirty, and the ops are appended by reference to each + watch's queue and each sink's batch. O(1) per dependent. Then, still synchronously and without any `await`, the + **flush**: + 1. recompute every dirty derive that has a consumer, in dependency order (each derive once per flush, after all its + sources), diffing its old and new value with `diffRevisions` into ops, and marking its own dependents; + 2. deliver each touched sink its batch for this publication (section 6); + 3. call the publication's flush listeners once (L6 uses this to send one message per link); + 4. schedule the queued watch frames (their listeners run later, asynchronously, in order). +- Every change goes through `publish`: L3's commit step (after the backend commit; S1), in-memory commits (no store + seq, the publication's own counter, W12), `writable.set` (which publishes itself), registry recomposition and + inbound wire batches. Each publication has a number (a counter per publication) for ordering and lazy checks. +- **Reentrancy:** during `adopt` and the flush, `publish`, `watch`, `attach`, a sink's `detach` and `writable.set` + throw `invalid/reentrant` (W1: no new transaction, watch or release inside the step). Stopping a watch handle is + allowed anywhere: it only detaches, and delivery tolerates removal during iteration. +- **Contexts** (context last, required, as in Durable): `createPublication(context)` keeps the host's context (it + counts lazy-read computes, since a getter has none); `publish(adopt, context)` and `writable.set(value, context)` take the context + that counts the flush's work and becomes the frames' context. `publication.number` is public (L3's in-memory commit + number, W12); `publication.onFlush(listener)` returns its removal. +- **Source ends:** a JSON source has `set(value, ops)`, `retire()`, `end()` and `end({ error })`, all only inside + `publish` (events in publication order), delivered in that publication's flush after its batch: watches retire, end + or fail; sinks get a retirement as `update(null, [["r", null]])` then `end()`, an end as `end()`, a failure as + `fail(error)`. A failed source's `value` throws its error, so derives over it fail with the same error; a retirement + is a change to `null` for derives over it; an end changes nothing for them. `watch` or `attach` on an ended source + returns a handle or sink ended the same way. A writable has only `set`. +- A source set outside `publish` throws (a programming error, `Error`, not a ChordError). + +## 3. Derives (W1, W4, W14, R24) + +```ts +const count = derive([notes], ([n]) => n.list.length, publication); +const visible = derive([notes, settings], ([n, s]) => ({ list: n.list.filter(...), theme: s.theme }), publication); +``` + +- `derive(sources, compute, publication)`: sources are `Watchable`s and `NativeWatchable`s (mixed) created by L2 on + the same publication (sources, derives, writables, constants; a foreign watchable such as a Durable view becomes an + L2 source first, L3d); `compute` is a synchronous pure function of their current values; the result is JSON and the + derive is a `Watchable`. +- **Attached only while consumed:** a derive with at least one consumer (a watch, a sink, or a consumed derive over + it) is attached to its sources and recomputed in the flush. A derive without consumers holds no subscription; reading + its `value` recomputes it if any source's publication number moved since its last compute (lazy). So an unused + derive never keeps work or memory alive, and nothing leaks when consumers go away. +- Ops: when a consumed derive recomputes, `diffRevisions(old, new)` gives its ops (identity short-circuit: unchanged + shared subtrees cost one `===`). A native source's change marks the derive dirty without ops. A recompute that + returns the identical value (`===`) publishes nothing. +- **Dependency order:** a derive's rank is one more than its highest-ranked source; the flush recomputes by rank. + Sources exist before the derive, so cycles cannot be built (cross-object cycles are rejected at activation, Q5, L4). +- **A throwing derive** fails only itself: its `value` throws the error, its watches end `{ reason: "failed", error }`, + its sinks get a failure, and derives over it fail with the same error. It stays failed (broken: no automatic + retry; L4 rebuilds it on reload). The publication and every other consumer continue. +- No `previous` (W14). The footgun and its remedy are documentation: a derive that builds new objects for unchanged + data costs O(value) per flush per consumer; `mapShared` (section 7) keeps unchanged elements identical. + +## 4. Constant and writable (R23, W4) + +- `constant(value)`: a `Watchable` that never changes; its watches never get a frame and end only by `stop` or + cancellation. No publication needed. Used for start data. +- `writable(initial, publication)`: a `NativeWatchable` with `set(value)`; `set` publishes by itself (one publication + per call); an identical value (`===`) publishes nothing. Used for registry contributions known only at runtime. + +## 5. Watch handles (W2, W3; Chord's shared implementation) + +L2 does not implement watch handles: it uses the one implementation in Chord's `src/watch.ts` (Durable's semantics, +moved there by the pi change). Sources create a handle per `watch(context)` and feed it: `advance` per publication +(during the flush, by reference), `retire`, `end`, `end({ error })`, cancellation from the context's signal. + +- `watch(context)` returns the handle at once (the Promise resolves immediately for local sources; L5 reference + members may wait). It holds the value current at acquisition and buffers every later publication. +- `start(listener)` installs the one listener; frames are delivered in order, each awaited, never inline; the listener + gets a context without the abort signal. +- **Overflow:** after 100 pending frames the backlog collapses: JSON to one frame `[["r", newest]]`, native to the + newest value. +- **Retirement** is an explicit source event (`source.retire()`, used by L3 for retired documents): the last frame is + `[["r", null]]` with value `null`, then the watch ends `retired`. Never inferred from a `null` value (W2a). +- **Ends:** `stopped`, `cancelled`, `retired`, `ended` (the source ended), `failed { error }` (the source failed, e.g. + a throwing derive), `listener_error { error }`; `stop` drops pending frames and never aborts or awaits a running call. + Termination clears every reference into the source (W3). +- `NativeWatchHandle`: the same lifecycle with listener `(value, context)`. +- **W2b** (routines delivering the acquisition value first) is L8's. + +## 6. Sinks (internal, for L6 and L9) + +A sink is a synchronous consumer inside the flush, used where a watch's asynchronous delivery is too late: the wire +encoder (L6: one part per hold, one message per link per publication) and registry composition (L9). + +```ts +const sink = attach(watchable, { update(value, ops) {...}, fail(error) {...}, end() {...} }); // returns detach() +``` + +- `update` is called once per publication that touched the source, during the flush, with that publication's ops + (several sets of one source within one publication arrive as one batch, in order). Native sources: `update(value)`. +- A sink counts as a consumer (derives over the source stay attached). `detach()` removes it; detaching inside the + flush throws `invalid/reentrant`. +- Not exported to apps; L4-L9 import it from `next/watch`. + +## 7. `mapShared` (W14, design §12) + +`mapShared(f)` returns a function over arrays that calls `f` only for elements it has not seen (by identity) and +returns the previous result for unchanged ones, so a derive over a list of 2,000 notes with one change builds one new +object and `diffRevisions` emits one op. The cache is a `WeakMap` from source element to result. + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a publication | per set source: pointer swap, O(1) per dependent (mark, append ops by reference) | +| flush | each dirty consumed derive: one compute plus one `diffRevisions`; nothing for unconsumed derives | +| a watch frame | the value and ops by reference; no copy, no diff | +| a sink update | the ops by reference | +| an unconsumed derive read | one compute if a source moved, else nothing | +| constant | nothing | + +**Counters** (telemetry, the function doing the work counts it): `chord.watch.computes` (derive computes) and +`chord.watch.diffs` (`diffRevisions` calls). No walks or copies anywhere in L2. + +## Footguns + +- A derive that rebuilds unchanged data costs O(value) per flush per consumer (use `mapShared`; R24: changing one + item of a flat array still copies the array's slots). +- Mutating a value in place breaks every consumer silently (immutable by convention). +- Async work in a derive is impossible by design; it belongs in a routine (L8) writing an in-memory document. + +## Tests (one group per mechanism, every case) + +- Publication: several sources in one publication reach a watch as consecutive frames and a sink as one batch; + publication numbers increase; setting outside `publish` throws. +- Derives: dependency order (diamond: A→B, A→C, B+C→D computes D once, after B and C); identical result publishes + nothing; native source marks without ops; lazy read recomputes only after a source moved; attach on first consumer, + detach after the last (no subscription left; GC); a chain A→B→C where B throws (C fails, A's other consumers + continue); a failed derive stays failed. +- Two sources published alternately with a derive over both: the derive sees every combination exactly once per + publication. +- Reentrancy: `publish`, `watch`, `attach`, `writable.set`, `detach` inside a derive and inside a sink throw + `invalid/reentrant`; `stop` inside a sink or listener is allowed and delivery continues correctly. +- Watch handles (the handle itself is tested in Chord's shared tests): through L2 sources, a watch started after 150 + publications; retire; end and fail from the source; stop during a running call; cancellation by the context. +- Constant and writable: constant never delivers; writable identical set publishes nothing. +- `mapShared`: unchanged elements return the identical result; a changed element is recomputed. +- Sinks: one batch per publication; failure and end; detach. +- Cost guards: computes and diffs per hop (an unconsumed derive computes nothing on publish). +- GC: stopped handles hold no source; unconsumed derives hold no subscription and are collected with their sources; + `mapShared` caches drop with their elements; retained revisions after overflow are bounded (W1's GC count). + +## Size + +Target: about 500 source lines, tests excluded. + +## Decided while specifying (owner, 2026-10-22) + +- **Watchable state is JSON only.** No bytes in documents, derives, reference members or any watchable value; bytes + appear only in method arguments and results (passed whole, carried as raw byte segments by L6) and in chunked + transfers. Delta stays JSON-only. Supersedes W4's "JSON plus bytes" for watchables. +- **`mapShared` lives in `next/watch/`** for now; it moves into Delta when v2 goes upstream. +- **C10 rejected:** `constant` (JSON `Watchable`) and `writable` (`NativeWatchable`) stay two, because Q14 needs the + kind distinction. +- **One publication per host** (`createPublication()`, created by the host and passed to every source, derive and + writable on it; apps never see it), never a module-level singleton. +- **No CBOR:** the wire stays JSON plus raw byte segments (R1, Q4); revisit only if L6 benchmarks show encoding as a + bottleneck. diff --git a/packages/chord/docs/next/plugin-authoring.md b/packages/chord/docs/next/plugin-authoring.md new file mode 100644 index 000000000..c21f477a3 --- /dev/null +++ b/packages/chord/docs/next/plugin-authoring.md @@ -0,0 +1,164 @@ +# Writing a Chord plugin + +For agents and people writing plugins for an app built on Chord v2. Self-contained: everything a plugin author needs +is here. In the examples the app is "Acme", its API package `@acme/app`; replace it with your app's (its docs name the +package and the versions it provides). + +## What a plugin is + +A plugin is a feature of a running app: it serves objects (state plus methods) and contributes to the app's +registries (routes, panels, tools). The app loads it on its hosts (the server, its workers, browser pages, terminal +clients), reloads it when it is edited, and leaves it out alone when it is broken. Plugins talk to each other only +through object contracts and registries. + +## Layout + +A **directory plugin**, in one of the app's plugin roots: + +```text +plugins/stats/ + package.json { "name": "@bob/stats", "dependencies": { ... } } required; the name is the plugin's + server.ts the server facet one file per host kind the app has, each optional: + worker.ts the worker facet .ts, .tsx, .js or .mjs; default export: a facet + web.tsx the web (page) facet + tui.ts the terminal client facet + contract.ts contracts every kind may import (data only) + contract.web.tsx contracts and tokens for web facets only (`contract..ts(x)` per kind) + *.ts, *.css, assets/ plain files of the plugin +``` + +- `package.json`: `name` (an npm package name, scoped or not) is the plugin's name; `dependencies` are bundled into + its scripts; `"chord": { "external": ["better-sqlite3"] }` keeps named packages external in Node facets (native + addons). +- A **package of several plugins** (a repository holding them) lists their folders in its own `package.json`: + `"chord": { "plugins": ["./notes", "plugins/*", "!plugins/draft"] }` (paths, globs and `!` exclusions; `.` is the + package itself; a listed folder inside another listed one is refused). Without the field, the package is one plugin + when its top has facet files, else **every** subfolder with a `package.json` is a plugin: a repository with nested + packages that are no plugins (a helper library, examples, fixtures) must list its plugins in `chord.plugins`. +- A plugin with no facet file holds contracts others import. +- A **single-file plugin** is a file at a root's top named `..ts` (or `.tsx`, `.js`, `.mjs`): + `hello.server.ts` and `hello.web.tsx` are the plugin `hello` with two facets. No `package.json`, so no + dependencies; it imports only the app's API and defines its contracts inline. When it needs more, make it the + directory `hello/` with a `package.json` named `hello`: the app treats it as the same plugin. + +## A facet + +```ts +// plugins/stats/worker.ts +import { defineFacet, int, local, str } from "@acme/app/v3"; +import { Indexers } from "@acme/app/v3/worker"; +import { Notes } from "../notes/contract.ts"; + +const Stats = local("bob.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +export default defineFacet((host, self) => { + self.contribute(Indexers, { name: "word count" }); // a registry entry while the plugin is loaded + host.serve(Stats, ({ project }, self) => { // a factory, run per object + const notes = self.get(Notes, { project }); // a dependency: Stats waits until Notes is live + return { words: self.derive([notes.text], ([text]) => text.split(/\s+/).filter(Boolean).length) }; + }); +}); +``` + +A facet only declares (`host.serve`, `self.contribute`, `host.references`), synchronously. Everything at runtime +happens in objects: state in documents and tables, standing behaviour in a `local` object with empty params (it lives +as long as the host). Names are prefixed by the plugin (`bob.stats`). + +## What a plugin may import + +| Import | Resolves to | +| --- | --- | +| its own files (`./label.ts`, `./icon.svg?url` in web facets) | inlined into its script | +| the app's API, a version in the path: `@acme/app/v3`, its kind's module `@acme/app/v3/worker` | the app's own instance (a host package), never copied | +| another plugin of the same root (one repository): `../notes/contract.ts` | inlined (a copy, frozen at this build) | +| a package in `dependencies`: `@bob/notes`, `@bob/notes/web` | inlined from `node_modules` at the installed version, through the package's `exports` | +| a plugin inside a dependency on a whole repository of several: `@bob/plugins/notes/contract.ts` | inlined, as any package file | +| a `chord.external` package, a Node builtin (Node facets) | loaded when the script runs | +| `pluginFile` from `@earendil-works/chord/next/plugins` | the absolute path of a file of this plugin | + +Refused at build, each with the file, line and rule: + +- `@acme/app/v1` when the app does not list v1 ("@acme/app/v1 is not available: this app provides v3, v2"); + another kind's module (`@acme/app/v3/web` from a worker facet); two API versions in one facet. +- Chord itself (`@earendil-works/chord/next`): the app's API re-exports what plugins may use. +- A relative path leaving the plugin's root (base code, another root); a file of a directory that is no plugin. +- A file copied from another plugin importing anything but the API and its own plugin's files. +- A per-kind file (`contract.web.tsx`, `contract.Web.ts` alike) or another kind's facet file (`web.tsx`) reached from + a facet of another kind, directly or through other files (the error names the chain). Files inside npm packages are + checked only when the package is plugin code (its `package.json`, or one above it in the package, has a `chord` + field); ordinary libraries name files as they like (`pdf.worker.mjs`). +- A package file whose real path is outside the plugin's own `node_modules` (a `file:` dependency linked elsewhere, a + package importing `../../src`). +- A copied file importing a Node builtin or a package. +- A package not in `dependencies`; `import.meta`, `__dirname`, `__filename`, a literal `import("x")`, top-level + `await` (read files through `pluginFile`). +- In a single-file plugin, anything but host packages. + +Contract files mention native things (React components, functions) only through `import type`; at runtime they import +only the API and the plugin's own data-only files. + +## Contracts and versions + +A plugin carries the contract it was written against. Plugins changing together (one root, one repository) import +each other's contract files by relative path. A plugin from another repository takes the contracts as a dependency +pinned to a version: an npm version, or a git ref (`"@bob/notes": "github:bob/notes#"`, `"git+https://...#v2"`). +The app's own contracts for third parties come with its API (`@acme/app/v3` exports `Notes`). When a contract +changes shape, bump its `version`; the provider serves older consumers through `accepts`, else they wait "contract +mismatch" (visible, never wrong data). + +## How the app builds it + +The server builds each facet with one esbuild call from source: TypeScript and TSX (JSX with the automatic React +runtime), JS, JSON, CSS (web facets: one stylesheet per facet), assets and WASM by URL (`import url from "./x.svg?url"`). +No type check, no build hooks, no `dist` needed; generated files are committed. The script is evaluated in each +process of its kind with that process's API modules. A failed build or evaluation leaves the plugin out (at startup) +or keeps its old code (at a reload), reported with the error. + +## Installing + +- `plugins.installPlugin({ git: "https://github.com/bob/stats.git", ref: "v1.2.0" }, root, ctx)`: a shallow clone of + the branch, tag or commit into `/stats`; its plugins are the folders its `chord.plugins` lists, else by + convention. `{ npm: "@bob/stats", version: "1.2.0" }`: the package extracted into `/@bob/stats`. Then their + dependencies are installed (`npm install --omit=dev --ignore-scripts`; no lifecycle scripts run). Refused: a plugin + name the app has already, an existing directory, a ref or name starting with `-`. +- Removing is deleting the directory. To update, install another ref (a new directory) or delete and install again. + +## Testing a plugin alone + +```ts +import { testPlugin } from "@earendil-works/chord/next/plugins"; +import * as apiV3 from "@acme/app/v3"; +import * as apiV3Worker from "@acme/app/v3/worker"; + +const Notes = apiV3.local("acme.notes", { version: 1, params: { project: apiV3.str() }, members: { text: apiV3.str() } }); +const Stats = apiV3.local("bob.stats", { version: 1, params: { project: apiV3.str() }, members: { words: apiV3.int() } }); +const notesStub = apiV3.defineFacet((host) => host.serve(Notes, () => ({ text: apiV3.constant("a b c") }))); + +const t = await testPlugin("./plugins/stats", { + kind: "worker", + kinds: ["server", "worker", "web"], // the app's kinds (by default: those its files name) + api: { name: "@acme/app", versions: ["v3"] }, + modules: { "@acme/app/v3": apiV3, "@acme/app/v3/worker": apiV3Worker }, // the real API modules + stubs: [{ name: "@acme/notes", facet: notesStub }], // facets serving what it uses + principal: { kind: "worker", project: "p1" }, // the host's principal (optional) +}, ctx); +const stats = await t.host.get(Stats, { project: "p1" }, ctx); // stats.words.value: 3 +t.contributions(apiV3Worker.Indexers); // its entries in a registry nobody serves (the helper collects them) +t.unserved(); // contracts its requests wait for: stub these +await t.root(async (ctx) => ..., ctx, { principal: { kind: "user", id: "u1" } }); +await t.close(ctx); +``` + +The plugin is built as production builds it and loaded on an in-memory host (a `:memory:` store). Stubs are ordinary +facets with fake data. A test declares the contracts it reads again (contracts are data: the same name, version and +schema bind). + +## Checklist + +1. `package.json` with a valid name (or a single file `..ts`). +2. One facet file per kind it runs on; each default-exports `defineFacet(...)`. +3. Imports: the app's API at one version, own files, same-root contract files by relative path, other repositories' + by package name pinned in `dependencies`. +4. Contracts prefixed with the plugin's name; a version bump for every change of shape. +5. Files of one kind named `*..ts(x)`. +6. A test with `testPlugin` and stubs for every name `t.unserved()` lists. diff --git a/packages/chord/docs/next/plugin-model/book.md b/packages/chord/docs/next/plugin-model/book.md new file mode 100644 index 000000000..5717711a3 --- /dev/null +++ b/packages/chord/docs/next/plugin-model/book.md @@ -0,0 +1,186 @@ +--- +title: The Plugin Model: what a plugin is, what it may import, and how versions meet +cover: The Plugin / Model +kicker: A design proposal · Chord v2 · 2026-10-31 +subtitle: Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31 +stats: + - 4 | kinds of import a plugin may write + - 1 | integer per API version, in the import path + - 0 | package.json fields Chord reads besides name and dependencies + - 0 | type checks in the build +note: | + A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided + and what is proposed into one model. Decided points cite their decision in `decisions.md`; proposed ones are marked. + Nothing here is built yet beyond slices 7c and 10. +colophon: The Plugin Model · proposal · 2026-10-31 +--- + +# Before we begin {.preface} + +An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix. + +The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code. + +## Eight things to know + +:::findings +1. **A plugin is a directory, or a single file.** A directory has a `package.json` (its name, its npm dependencies) + and one file per host kind; a single file `hello.worker.ts` is a plugin named `hello` with no dependencies. +2. **Base code is internal unless it is an API.** The app exposes only versioned API modules to plugins; everything + else is unreachable. +3. **API versions are in the import path.** `import { Session } from "@acme/app/v3"`: the plugin says what it was + written for where it imports it; an unavailable version is a build error. +4. **APIs are split by host kind.** A shared module holds what every host may use (contracts, registry tokens, host + kinds); `/v3/web` and `/v3/tui` hold UI pieces; a worker never loads React. +5. **Contracts carry their own versions.** A plugin carries the contract version it was written for; the provider + binds it if it accepts that version, else the plugin waits with a clear reason. +6. **A plugin imports four things:** its own files, host packages, its npm dependencies, and the app's own plugins by + relative path. Nothing else. +7. **Settings are arguments, extensions are registries.** What the app decides at startup is passed to its components; + what plugins add is contributed. +8. **No type checking in the build.** Types come from where imports come from; editors and agents check, the build + only builds. +::: + +## Words used throughout + +**Base code** is the app's own program; its **API** is the part plugins may import, published as **API modules**, +one per version and host kind. A **host package** is a module the running process provides to every plugin (Chord, +the app's API modules). A **contract** is an object type's declaration; a **registry token** an extension point's. +A **facet** is a plugin's declarations for one host kind. + +# What a Plugin Is {#plugin lede="A directory or a single file, named once, with one file per host kind."} + +```text +plugins/notes/ a directory plugin + package.json { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } } + contract.ts the object types and registry tokens it offers + worker.ts server.ts web.tsx one facet per host kind it needs +plugins/hello.worker.ts a single-file plugin "hello" with one worker facet +plugins/hello.web.tsx the same plugin's web facet +``` + +A directory plugin's name is its `package.json` name; Chord reads only `name` and `dependencies` from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c). + +# Base Code: Internal and External {#base lede="What the app keeps to itself, what it exposes, and how exposure is versioned."} + +The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its **API**: + +```text +src/api/v3/index.ts shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens +src/api/v3/web.ts web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens +src/api/v3/tui.ts TUI only: its UI pieces +src/… internal: everything else +``` + +Each API module is a host package of the kinds that may use it: `@acme/app/v3` for every kind, `@acme/app/v3/web` for +the web kind only. A worker plugin importing `@acme/app/v3/web` is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with `import type`, which the build erases. + +:::note | Proposed +The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided. +::: + +# Versions in the Import Path {#versions lede="A plugin says what it was written for where it imports it."} + +The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides: + +```ts +// plugins/stats/worker.ts +import { defineFacet, Session, Panels } from "@acme/app/v3"; +import { remote, str } from "@earendil-works/chord/next/v2"; +``` + +- The app provides `/v3`, its current API. If version 2 plugins still work unchanged against it, `/v2` is listed too, + pointing to the same module. An app that needs a compatibility layer makes `/v2` a thin module over `/v3`. +- An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides + v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change. +- Nothing to fill in elsewhere: no `package.json` field, no npm ranges. Skeletons and examples import the current + version, so an agent copies it without thinking. +- Chord's own surface is versioned the same way (`@earendil-works/chord/next/v2`). + +Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how. + +# Contracts and Registries {#contracts lede="Finer versions inside the API, and what crosses between plugins."} + +Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole: + +- **An object type**: the consumer carries the contract version it was written for; the provider binds it at that + version if it accepts it (`accepts` maps arguments forward and results and state back), otherwise the consumer waits + "contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught. +- **A registry token**: a name, a version and TypeScript types for what is contributed and read. Values are in-process + things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining + plugin's rule, and its `accepts` adapts older contributions. +- **Host kinds on registries** (proposed): a token names the kinds that serve it (`hosts: ["web"]`), typed, so a worker + facet contributing to a web registry is a type error and a load error, not a silent no-op. + +A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only. + +# What a Plugin May Import {#imports lede="Four kinds of import, each with one rule."} + +| Import | Resolved to | Versioned by | +| --- | --- | --- | +| its own files (`./store.ts`) | copied into its script | the plugin itself | +| host packages (`@acme/app/v3`, `@earendil-works/chord/next/v2`, `react`) | the running process's instance, shared | the path's version; the app lists what it provides | +| npm packages in its `dependencies` (`slugify`, a third party's contracts package) | its own `node_modules`, bundled | the installed version | +| the app's own plugins (`../notes/contract.ts`), in the same root | copied into its script | the same revision: one repository | + +A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error. + +# Settings and Extensions {#settings lede="How the app configures its components without letting plugins override it."} + +Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it: + +```ts +const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx); +``` + +What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them. + +# Building and Evaluating {#build lede="Unchanged from slices 7c and 10: one build on the server, evaluation everywhere."} + +Only the server builds, because only the server has the sources. `createPlugins` builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at `plugins.load`, a worker's in `workerMain`, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload. + +# Types and Tests {#tools lede="Where types come from, and how a plugin is tested alone."} + +Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship `.d.ts` per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run `tsc` as a tool. + +Testing a plugin alone (proposed): `testPlugin(dir, { kind, stubs })` builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework. + +# Open Questions {.epilogue lede="What is proposed here and still to decide."} + +1. The external API split by host kind (`/v3`, `/v3/web`, `/v3/tui`). +2. Host kinds on registry tokens, typed and checked at load. +3. Single-file plugins (`hello.worker.ts`, named by the file). +4. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package. +5. `testPlugin` with stub facets. +6. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app + and npm dependencies across it. diff --git a/packages/chord/docs/next/plugin-model/index.html b/packages/chord/docs/next/plugin-model/index.html new file mode 100644 index 000000000..2549955ac --- /dev/null +++ b/packages/chord/docs/next/plugin-model/index.html @@ -0,0 +1,310 @@ + + +The Plugin Model: what a plugin is, what it may import, and how versions meet + + + + +

+
+
A design proposal · Chord v2 · 2026-10-31
+

The Plugin
Model

+

Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31

+
4kinds of import a plugin may write
1integer per API version, in the import path
0package.json fields Chord reads besides name and dependencies
0type checks in the build
+

A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided +and what is proposed into one model. Decided points cite their decision in decisions.md; proposed ones are marked. +Nothing here is built yet beyond slices 7c and 10.

+
+
+ + +
+

Before we begin

+

An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix.

+

The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code.

+

Eight things to know

+
    +
  1. A plugin is a directory, or a single file. A directory has a package.json (its name, its npm dependencies) +and one file per host kind; a single file hello.worker.ts is a plugin named hello with no dependencies.
    2. +
  2. Base code is internal unless it is an API. The app exposes only versioned API modules to plugins; everything +else is unreachable.
    3. +
  3. API versions are in the import path. import { Session } from "@acme/app/v3": the plugin says what it was +written for where it imports it; an unavailable version is a build error.
    4. +
  4. APIs are split by host kind. A shared module holds what every host may use (contracts, registry tokens, host +kinds); /v3/web and /v3/tui hold UI pieces; a worker never loads React.
    5. +
  5. Contracts carry their own versions. A plugin carries the contract version it was written for; the provider +binds it if it accepts that version, else the plugin waits with a clear reason.
    6. +
  6. A plugin imports four things: its own files, host packages, its npm dependencies, and the app's own plugins by +relative path. Nothing else.
    7. +
  7. Settings are arguments, extensions are registries. What the app decides at startup is passed to its components; +what plugins add is contributed.
    8. +
  8. No type checking in the build. Types come from where imports come from; editors and agents check, the build +only builds.
    9. +
+

Words used throughout

+

Base code is the app's own program; its API is the part plugins may import, published as API modules, +one per version and host kind. A host package is a module the running process provides to every plugin (Chord, +the app's API modules). A contract is an object type's declaration; a registry token an extension point's. +A facet is a plugin's declarations for one host kind.

+
+ +
I

What a Plugin Is

A directory or a single file, named once, with one file per host kind.

+
plugins/notes/                     a directory plugin
+  package.json                     { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } }
+  contract.ts                      the object types and registry tokens it offers
+  worker.ts  server.ts  web.tsx    one facet per host kind it needs
+plugins/hello.worker.ts            a single-file plugin "hello" with one worker facet
+plugins/hello.web.tsx              the same plugin's web facet
+

A directory plugin's name is its package.json name; Chord reads only name and dependencies from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c).

+
+ +
II

Base Code: Internal and External

What the app keeps to itself, what it exposes, and how exposure is versioned.

+

The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its API:

+
src/api/v3/index.ts       shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens
+src/api/v3/web.ts         web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens
+src/api/v3/tui.ts         TUI only: its UI pieces
+src/…                     internal: everything else
+

Each API module is a host package of the kinds that may use it: @acme/app/v3 for every kind, @acme/app/v3/web for +the web kind only. A worker plugin importing @acme/app/v3/web is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with import type, which the build erases.

+
Proposed

The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided.

+
+
+ +
III

Versions in the Import Path

A plugin says what it was written for where it imports it.

+

The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides:

+
// plugins/stats/worker.ts
+import { defineFacet, Session, Panels } from "@acme/app/v3";
+import { remote, str } from "@earendil-works/chord/next/v2";
+
    +
  • The app provides /v3, its current API. If version 2 plugins still work unchanged against it, /v2 is listed too, +pointing to the same module. An app that needs a compatibility layer makes /v2 a thin module over /v3.
    • +
  • An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides +v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change.
    • +
  • Nothing to fill in elsewhere: no package.json field, no npm ranges. Skeletons and examples import the current +version, so an agent copies it without thinking.
    • +
  • Chord's own surface is versioned the same way (@earendil-works/chord/next/v2).
    • +
+

Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how.

+
+ +
IV

Contracts and Registries

Finer versions inside the API, and what crosses between plugins.

+

Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole:

+
    +
  • An object type: the consumer carries the contract version it was written for; the provider binds it at that +version if it accepts it (accepts maps arguments forward and results and state back), otherwise the consumer waits +"contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught.
    • +
  • A registry token: a name, a version and TypeScript types for what is contributed and read. Values are in-process +things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining +plugin's rule, and its accepts adapts older contributions.
    • +
  • Host kinds on registries (proposed): a token names the kinds that serve it (hosts: ["web"]), typed, so a worker +facet contributing to a web registry is a type error and a load error, not a silent no-op.
    • +
+

A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only.

+
+ +
V

What a Plugin May Import

Four kinds of import, each with one rule.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ImportResolved toVersioned by
its own files (./store.ts)copied into its scriptthe plugin itself
host packages (@acme/app/v3, @earendil-works/chord/next/v2, react)the running process's instance, sharedthe path's version; the app lists what it provides
npm packages in its dependencies (slugify, a third party's contracts package)its own node_modules, bundledthe installed version
the app's own plugins (../notes/contract.ts), in the same rootcopied into its scriptthe same revision: one repository
+

A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error.

+
+ +
VI

Settings and Extensions

How the app configures its components without letting plugins override it.

+

Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it:

+
const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx);
+

What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them.

+
+ +
VII

Building and Evaluating

Unchanged from slices 7c and 10: one build on the server, evaluation everywhere.

+

Only the server builds, because only the server has the sources. createPlugins builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at plugins.load, a worker's in workerMain, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload.

+
+ +
VIII

Types and Tests

Where types come from, and how a plugin is tested alone.

+

Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship .d.ts per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run tsc as a tool.

+

Testing a plugin alone (proposed): testPlugin(dir, { kind, stubs }) builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework.

+
+ +
∴

Open Questions

What is proposed here and still to decide.

+
    +
  1. The external API split by host kind (/v3, /v3/web, /v3/tui).
    2. +
  2. Host kinds on registry tokens, typed and checked at load.
    3. +
  3. Single-file plugins (hello.worker.ts, named by the file).
    4. +
  4. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package.
    5. +
  5. testPlugin with stub facets.
    6. +
  6. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app +and npm dependencies across it.
    7. +
+
+ +

The Plugin Model · proposal · 2026-10-31

+
+ + diff --git a/packages/chord/docs/next/plugin-primer/book.md b/packages/chord/docs/next/plugin-primer/book.md new file mode 100644 index 000000000..361a043b2 --- /dev/null +++ b/packages/chord/docs/next/plugin-primer/book.md @@ -0,0 +1,273 @@ +--- +title: The Plugin Model, a Primer: what a plugin depends on, and how mismatches are caught +cover: Plugins / from the Ground Up +kicker: A primer · Chord v2 · 2026-10-31 +subtitle: One app, its own plugins and a stranger's plugin, followed through a year of changes, to show what the open decisions are about +stats: + - 4 | things a plugin depends on + - 3 | of them change without the plugin knowing + - 8 | points still to decide + - 1 | rule decided for base code: the version is in the import path +note: | + Written by the redesign session for the owner, after the proposal `docs/next/plugin-model/` did not explain enough. + Everything decided cites its decision in `decisions.md`; everything open is marked and ends with a recommendation. + Built behaviour is slice 7c and slice 10 (`src/next-node/plugins/`). +colophon: The Plugin Model, a Primer · first edition · 2026-10-31 +--- + +# Before we begin {.preface} + +Acme builds a notes app on Chord. The app is a server, a worker per project and browser tabs. Most features are +plugins: Acme's own `notes` plugin keeps the notes, its `search` plugin searches them. Bob, who does not work at +Acme, writes a `stats` plugin that counts words per project and shows a panel. Bob wrote it in March. It is now +November. + +Between March and November, three things changed without Bob: Acme released a new version of the app, Acme's `notes` +plugin changed what its notes look like, and Chord got a new release. Bob's plugin is still the March code. The +question this primer answers is the only question the plugin model has to answer: + +**When a plugin written against one state of the world runs against another, what does it get, and who notices the +mismatch?** + +The answer is different for each thing a plugin depends on, so the primer takes them one at a time. + +## What a plugin is, in one picture + +```text +plugins/stats/ Bob's plugin + package.json { "name": "@bob/stats", "dependencies": { "words-count": "2.1.0" } } + worker.ts runs in each project's worker: counts words + web.tsx runs in each tab: the panel +``` + +The server builds every plugin's files into one script per host kind (esbuild, a few milliseconds each). Each +process evaluates the scripts of its kind: the worker runs `worker.ts`'s script, a tab runs `web.tsx`'s. That much +is built (slices 7c and 10). + +# Four Things a Plugin Depends On {#four lede="Who changes each one, and when the plugin finds out."} + +Look at the top of Bob's `worker.ts`: + +```ts +import countWords from "words-count"; // 1. an npm package +import { defineFacet, Project } from "@acme/app"; // 2. the app's base code +import { obj, num, remote } from "@earendil-works/chord/next"; // 3. Chord +import { Notes } from "@acme/notes/contract"; // 4. another plugin +``` + +Each import is a different kind of dependency, and they behave differently over time: + +| Import | Who changes it | What Bob's plugin gets today | +| --- | --- | --- | +| `words-count` | its npm author | the version installed in Bob's plugin directory, copied into his script: frozen until Bob updates it | +| `@acme/app` | Acme, when it releases the app | whatever the running app is: always the newest | +| `@earendil-works/chord/next` | Chord, through Acme's upgrade | whatever the running app ships: always the newest | +| `@acme/notes/contract` | Acme, whenever it edits `notes` | the running `notes` plugin's file, copied at Bob's next build: always the newest | + +The first row is safe: npm packages are copied into the script at the version Bob installed, so they never change +under him. The other three rows always give the newest, and Bob's March code meets November's app. Whether that +works depends on what changed. The rest of the primer is about those three rows. + +:::note | Why not copy base code and Chord into the script too +Because they must be shared. If Bob's script carried its own copy of Chord, its `defineFacet` would make facets the +worker's host does not recognize, and its `Project` host kind would be a different object from the app's. Base code +and Chord are **host packages**: the process provides one instance and every plugin uses it. That is decided and +built. It is also why they can change under a plugin. +::: + +# Base Code: Versions in the Import Path {#base lede="Decided: a plugin says which app API it was written for, where it imports it."} + +In June, Acme renames the host kind `Project` to `Workspace`. Bob's March plugin imports `Project`; it no longer +exists. Without any versioning, Bob's plugin builds (the build does not type check), then fails when it runs with +something like "Cannot read properties of undefined", deep inside his code. + +Decided on 2026-10-31 (decision "API versions in the import path"): the app exposes only **API modules**, each with a +version in its path, and lists the versions it still provides. + +```ts +// Bob, in March +import { defineFacet, Project } from "@acme/app/v2"; + +// Acme's app, after June +apiVersions: { "@acme/app/v3": apiV3, "@acme/app/v2": apiV2 } // still provides v2, through a thin layer over v3 +``` + +Now there are two cases, both visible: + +- Acme still provides `/v2`: Bob's plugin gets the v2 module, in which `Project` still exists (Acme's thin layer maps + it to `Workspace`). It works. +- Acme dropped `/v2`: Bob's plugin fails to build with "@acme/app/v2 is not available: this app provides v3". The + plugin is left out (per plugin: nothing else breaks), and the message tells Bob, or an agent, exactly what to + change. + +Nothing outside the code says what Bob depends on: no `package.json` field, no version ranges. The import says it. + +**What is still open here** is what goes into the API and how Chord relates to it. That is point 2 below. + +# Other Plugins: the Contract Problem {#contracts lede="Why importing another plugin by name gives the wrong answer for a stranger's plugin."} + +Bob's `stats` asks Acme's `notes` for the notes of a project. Plugins talk only through **contracts** (the type of an +object: its state and methods) and registries. A contract has its own version: + +```ts +// plugins/notes/contract.ts (Acme), March +export const Notes = remote("notes", { + version: 1, + params: { project: str() }, + hostAddress: ({ project }) => Project({ id: project }), + variants: { reader: { state: { list: obj({ items: arr(str()) }) } } }, +}); +``` + +Bob's plugin imports this file, and the build copies it into his script. When his plugin asks for `Notes`, it says +"version 1, this shape"; the provider (the `notes` plugin's code in the worker) checks that it can serve that version. + +In September Acme changes notes: each note gets a timestamp, `items: arr(obj({ text: str(), at: int() }))`, and the +contract becomes version 2. Acme also writes an `accepts` mapping, so consumers of version 1 still get a list of +strings. Chord's contract versioning exists for exactly this case. + +Now the question is: which version of the contract file ends up in Bob's script? + +## Today: by name, the newest file + +Decided this morning and built (decision "Plugins import each other by name"): `@acme/notes/contract` resolves to the +running `notes` plugin's `contract.ts`. Trace what happens at Bob's next build after September: + +:::steps | A rebuild after Acme's change ++++ The build copies the new file +Bob's `stats` is rebuilt (a rollout, a restart, any edit anywhere). `@acme/notes/contract` is now the version 2 file, +and it is copied into Bob's script. ++++ Bob's plugin claims version 2 +At runtime his plugin asks for `Notes` version 2. The provider serves version 2 gladly: no `accepts` is used. ++++ Bob's code reads version 1's shape +Bob's code does `list.items.join(" ")` and `countWords` on it. The items are now objects. He counts the words of +`"[object Object] [object Object]"`. Nothing fails; the panel shows wrong numbers. +::: + +The contract version was supposed to protect Bob, and it did nothing: copying the newest file made Bob's plugin claim +a version his code was never written for. + +## Proposed: the version Bob wrote against + +The contract file must come from where Bob's code came from: the state of the world he wrote against. For a stranger's +plugin that is an npm dependency at a fixed version: + +```json +// plugins/stats/package.json +{ "name": "@bob/stats", "dependencies": { "@acme/notes-contracts": "1.0.0" } } +``` + +Now Bob's script carries the March contract file forever (npm packages are copied at the installed version, as row 1 +of the first table). His plugin asks for version 1; the provider maps it through `accepts`; Bob gets strings as +before. If Acme ever stops accepting version 1, Bob's plugin waits with "contract mismatch: v1, served v3", visible in +the UI, never wrong numbers. + +For Acme's own plugins (`search` using `notes`) the problem does not exist: they live in one repository and change in +the same commit. They import each other by relative path, `../notes/contract.ts`, the same file at the same revision. +The by-name rule also needed path aliases in the TypeScript configuration for every plugin (six were added for the +tests and examples of slice 10); relative paths need none. + +# The Open Points {#open lede="Eight decisions, each with the problem it answers and a recommendation."} + +## 1. Reverse "import by name" + +**Problem:** the trace above: a stranger's plugin silently claims a contract version it was not written for. +**Proposal:** the app's own plugins import each other relatively (`../notes/contract.ts`); everyone else gets +contracts as an npm dependency at a fixed version. Importing a running plugin by name is removed; a relative path +leaving the plugin's root stays a build error. +**Recommendation:** do it, in the plugin slice (slice 10 is committed with by-name imports; the change is small: +`build.ts` and the examples' imports). + +## 2. What the app's API is, and Chord in it + +**Problem:** a plugin imports two host packages that both change, `@acme/app/v3` and `@earendil-works/chord/next/v2`. +Acme must then say which combinations work: does a plugin importing app v3 and Chord v1 still run? Two version +numbers per plugin, and a matrix nobody tests. +**Proposal:** plugins import only the app's API. The app re-exports the part of Chord it supports: + +```ts +// src/api/v3/index.ts (Acme's base code) +export { obj, arr, str, int, remote, local, defineRegistry } from "@earendil-works/chord/next"; +export { Workspace, Panels, defineFacet } from "../../core.ts"; +``` + +One number, the app's, describes everything a plugin gets from its host. When Acme upgrades Chord in a way that +breaks plugins, Acme bumps its API version, and decides whether `/v2` survives as a thin layer. +**Generations:** if the app provides `/v3` and `/v2`, a facet must import from one of them only, never both (the +build checks it), because `/v2`'s `Panels` and `/v3`'s `Panels` may be different registries. +**Recommendation:** yes; app kit apps get it for free (`chord/next/app` provides the API module). + +## 3. The API split by host kind + +**Problem:** the API module holds everything plugins need, including web things: a `Sheet` component, a theme, the +`Panels` registry whose entries are React components. A worker plugin imports `@acme/app/v3` for `Workspace`, and the +worker process has to load the module, so it loads React. +**Proposal:** `/v3` holds what every host may use (contracts, registry tokens, host kinds, principals) and imports no +UI library at runtime; `/v3/web` and `/v3/tui` hold UI pieces. A worker facet importing `/v3/web` is a build error. +**Recommendation:** yes. + +## 4. Host kinds on registry tokens + +**Problem:** Bob contributes his panel from `worker.ts` by mistake (`self.contribute(Panels, …)`). The worker has no +`Panels` registry; the contribution sits in a table nobody reads. Nothing fails; the panel never appears. +**Proposal:** a token names the kinds that serve it, `defineRegistry("app.panels", { version: 1, hosts: ["web"] })`; +contributing from another kind is a type error (the facet's `self` is typed per kind) and a load error. +**Recommendation:** yes; together with point 3, `Panels` lives in `/v3/web` anyway. + +## 5. Contracts for third parties + +**Problem:** with point 1, Bob needs Acme's plugins' contracts as an npm package. Which package? +**Options:** (a) the app's API exports the contracts of its own plugins that others may use (`@acme/app/v3` has +`Notes`): one place, versioned with the app; (b) Acme publishes its app package to npm and Bob depends on it; (c) each +plugin publishes its own contracts package. (c) is what the owner called unpleasant. +**Recommendation:** (a) for contracts Acme wants others to use: the API is the app's promise. Note: in (a) the +contract file comes with the API version, so Bob's v2 import keeps v2's contract files (the thin layer keeps them). + +## 6. Testing a plugin alone + +**Problem:** testing Bob's `stats` needs a worker with a `Notes` provider, an app API, a store. +**Proposal:** `testPlugin("plugins/stats", { kind: "worker", stubs: [notesStub] })` builds the plugin as production +does, loads it on an in-memory host with the kind's real host packages, and loads stub facets that serve the +contracts it uses. A stub is an ordinary facet (`host.serve(Notes, () => ({ list: constant({ items: ["a b"] }) }))`). +**Recommendation:** yes, in the app kit. + +## 7. Single-file plugins + +**Problem:** a plugin that adds one route needs a directory and a `package.json` with a name. +**Proposal:** `plugins/hello.server.ts` is a plugin named `hello` with one server facet, no dependencies; when it +needs a package or a second file, it becomes a directory. +**Recommendation:** yes, cheap; agents write many such plugins. + +## 8. The app's own plugins follow the same rules + +**Problem:** are Acme's built-in plugins special? +**Proposal:** no. They import the app's API like everyone (not base code's internals), import each other relatively +(point 1), are built at startup and reload like any plugin. The only difference: they change in the same commit as +base code, so they always use the newest API version. +**Recommendation:** yes. It keeps base code small and means every built-in plugin is an example of a correct plugin. + +# What to Decide {.epilogue lede="In the order that makes each answer easier."} + +1. Reverse "import by name" (recommended: yes, in the plugin slice). +2. Plugins import only the app's API; Chord is re-exported in it; one generation per facet (recommended: yes). +3. Split the API by host kind (recommended: yes). +4. Host kinds on registry tokens (recommended: yes). +5. Contracts for third parties through the app's API (recommended: (a)). +6. `testPlugin` with stub facets (recommended: yes). +7. Single-file plugins (recommended: yes). +8. Built-in plugins follow the same rules (recommended: yes). + +Later, not part of these: project plugins (per working directory, with trust), and installing plugins from npm or git. + +# Words {.appendix} + +| Word | Meaning | +| --- | --- | +| base code | the app's own program, started by `node server.ts`; never reloads | +| API module | the part of base code plugins may import, one per version (`@acme/app/v3`) | +| host package | a module the process provides to every plugin (Chord, the API modules, React on the web) | +| contract | an object type: its state and methods, with a version; consumers name the version they use | +| `accepts` | the provider's mapping that serves an older contract version | +| registry token | an extension point (`Panels`); plugins contribute to it | +| facet | a plugin's code for one host kind (`worker.ts`) | +| generation | one set of API versions provided together (`/v3` with its web and tui modules) | diff --git a/packages/chord/docs/next/plugin-primer/index.html b/packages/chord/docs/next/plugin-primer/index.html new file mode 100644 index 000000000..0eb064640 --- /dev/null +++ b/packages/chord/docs/next/plugin-primer/index.html @@ -0,0 +1,430 @@ + + +The Plugin Model, a Primer: what a plugin depends on, and how mismatches are caught + + + + +
+
+
A primer · Chord v2 · 2026-10-31
+

Plugins
from the Ground Up

+

One app, its own plugins and a stranger's plugin, followed through a year of changes, to show what the open decisions are about

+
4things a plugin depends on
3of them change without the plugin knowing
8points still to decide
1rule decided for base code: the version is in the import path
+

Written by the redesign session for the owner, after the proposal docs/next/plugin-model/ did not explain enough. +Everything decided cites its decision in decisions.md; everything open is marked and ends with a recommendation. +Built behaviour is slice 7c and slice 10 (src/next-node/plugins/).

+
+
+ + +
+

Before we begin

+

Acme builds a notes app on Chord. The app is a server, a worker per project and browser tabs. Most features are +plugins: Acme's own notes plugin keeps the notes, its search plugin searches them. Bob, who does not work at +Acme, writes a stats plugin that counts words per project and shows a panel. Bob wrote it in March. It is now +November.

+

Between March and November, three things changed without Bob: Acme released a new version of the app, Acme's notes +plugin changed what its notes look like, and Chord got a new release. Bob's plugin is still the March code. The +question this primer answers is the only question the plugin model has to answer:

+

When a plugin written against one state of the world runs against another, what does it get, and who notices the +mismatch?

+

The answer is different for each thing a plugin depends on, so the primer takes them one at a time.

+

What a plugin is, in one picture

+
plugins/stats/                     Bob's plugin
+  package.json                     { "name": "@bob/stats", "dependencies": { "words-count": "2.1.0" } }
+  worker.ts                        runs in each project's worker: counts words
+  web.tsx                          runs in each tab: the panel
+

The server builds every plugin's files into one script per host kind (esbuild, a few milliseconds each). Each +process evaluates the scripts of its kind: the worker runs worker.ts's script, a tab runs web.tsx's. That much +is built (slices 7c and 10).

+
+ +
I

Four Things a Plugin Depends On

Who changes each one, and when the plugin finds out.

+

Look at the top of Bob's worker.ts:

+
import countWords from "words-count";                       // 1. an npm package
+import { defineFacet, Project } from "@acme/app";           // 2. the app's base code
+import { obj, num, remote } from "@earendil-works/chord/next";   // 3. Chord
+import { Notes } from "@acme/notes/contract";               // 4. another plugin
+

Each import is a different kind of dependency, and they behave differently over time:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ImportWho changes itWhat Bob's plugin gets today
words-countits npm authorthe version installed in Bob's plugin directory, copied into his script: frozen until Bob updates it
@acme/appAcme, when it releases the appwhatever the running app is: always the newest
@earendil-works/chord/nextChord, through Acme's upgradewhatever the running app ships: always the newest
@acme/notes/contractAcme, whenever it edits notesthe running notes plugin's file, copied at Bob's next build: always the newest
+

The first row is safe: npm packages are copied into the script at the version Bob installed, so they never change +under him. The other three rows always give the newest, and Bob's March code meets November's app. Whether that +works depends on what changed. The rest of the primer is about those three rows.

+
Why not copy base code and Chord into the script too

Because they must be shared. If Bob's script carried its own copy of Chord, its defineFacet would make facets the +worker's host does not recognize, and its Project host kind would be a different object from the app's. Base code +and Chord are host packages: the process provides one instance and every plugin uses it. That is decided and +built. It is also why they can change under a plugin.

+
+
+ +
II

Base Code: Versions in the Import Path

Decided: a plugin says which app API it was written for, where it imports it.

+

In June, Acme renames the host kind Project to Workspace. Bob's March plugin imports Project; it no longer +exists. Without any versioning, Bob's plugin builds (the build does not type check), then fails when it runs with +something like "Cannot read properties of undefined", deep inside his code.

+

Decided on 2026-10-31 (decision "API versions in the import path"): the app exposes only API modules, each with a +version in its path, and lists the versions it still provides.

+
// Bob, in March
+import { defineFacet, Project } from "@acme/app/v2";
+
+// Acme's app, after June
+apiVersions: { "@acme/app/v3": apiV3, "@acme/app/v2": apiV2 }   // still provides v2, through a thin layer over v3
+

Now there are two cases, both visible:

+
    +
  • Acme still provides /v2: Bob's plugin gets the v2 module, in which Project still exists (Acme's thin layer maps +it to Workspace). It works.
    • +
  • Acme dropped /v2: Bob's plugin fails to build with "@acme/app/v2 is not available: this app provides v3". The +plugin is left out (per plugin: nothing else breaks), and the message tells Bob, or an agent, exactly what to +change.
    • +
+

Nothing outside the code says what Bob depends on: no package.json field, no version ranges. The import says it.

+

What is still open here is what goes into the API and how Chord relates to it. That is point 2 below.

+
+ +
III

Other Plugins: the Contract Problem

Why importing another plugin by name gives the wrong answer for a stranger's plugin.

+

Bob's stats asks Acme's notes for the notes of a project. Plugins talk only through contracts (the type of an +object: its state and methods) and registries. A contract has its own version:

+
// plugins/notes/contract.ts (Acme), March
+export const Notes = remote("notes", {
+	version: 1,
+	params: { project: str() },
+	hostAddress: ({ project }) => Project({ id: project }),
+	variants: { reader: { state: { list: obj({ items: arr(str()) }) } } },
+});
+

Bob's plugin imports this file, and the build copies it into his script. When his plugin asks for Notes, it says +"version 1, this shape"; the provider (the notes plugin's code in the worker) checks that it can serve that version.

+

In September Acme changes notes: each note gets a timestamp, items: arr(obj({ text: str(), at: int() })), and the +contract becomes version 2. Acme also writes an accepts mapping, so consumers of version 1 still get a list of +strings. Chord's contract versioning exists for exactly this case.

+

Now the question is: which version of the contract file ends up in Bob's script?

+

Today: by name, the newest file

+

Decided this morning and built (decision "Plugins import each other by name"): @acme/notes/contract resolves to the +running notes plugin's contract.ts. Trace what happens at Bob's next build after September:

+
A rebuild after Acme's change

The build copies the new file

Bob's stats is rebuilt (a rollout, a restart, any edit anywhere). @acme/notes/contract is now the version 2 file, +and it is copied into Bob's script.

+

Bob's plugin claims version 2

At runtime his plugin asks for Notes version 2. The provider serves version 2 gladly: no accepts is used.

+

Bob's code reads version 1's shape

Bob's code does list.items.join(" ") and countWords on it. The items are now objects. He counts the words of +"[object Object] [object Object]". Nothing fails; the panel shows wrong numbers.

+
+

The contract version was supposed to protect Bob, and it did nothing: copying the newest file made Bob's plugin claim +a version his code was never written for.

+

Proposed: the version Bob wrote against

+

The contract file must come from where Bob's code came from: the state of the world he wrote against. For a stranger's +plugin that is an npm dependency at a fixed version:

+
// plugins/stats/package.json
+{ "name": "@bob/stats", "dependencies": { "@acme/notes-contracts": "1.0.0" } }
+

Now Bob's script carries the March contract file forever (npm packages are copied at the installed version, as row 1 +of the first table). His plugin asks for version 1; the provider maps it through accepts; Bob gets strings as +before. If Acme ever stops accepting version 1, Bob's plugin waits with "contract mismatch: v1, served v3", visible in +the UI, never wrong numbers.

+

For Acme's own plugins (search using notes) the problem does not exist: they live in one repository and change in +the same commit. They import each other by relative path, ../notes/contract.ts, the same file at the same revision. +The by-name rule also needed path aliases in the TypeScript configuration for every plugin (six were added for the +tests and examples of slice 10); relative paths need none.

+
+ +
IV

The Open Points

Eight decisions, each with the problem it answers and a recommendation.

+

1. Reverse "import by name"

+

Problem: the trace above: a stranger's plugin silently claims a contract version it was not written for. +Proposal: the app's own plugins import each other relatively (../notes/contract.ts); everyone else gets +contracts as an npm dependency at a fixed version. Importing a running plugin by name is removed; a relative path +leaving the plugin's root stays a build error. +Recommendation: do it, in the plugin slice (slice 10 is committed with by-name imports; the change is small: +build.ts and the examples' imports).

+

2. What the app's API is, and Chord in it

+

Problem: a plugin imports two host packages that both change, @acme/app/v3 and @earendil-works/chord/next/v2. +Acme must then say which combinations work: does a plugin importing app v3 and Chord v1 still run? Two version +numbers per plugin, and a matrix nobody tests. +Proposal: plugins import only the app's API. The app re-exports the part of Chord it supports:

+
// src/api/v3/index.ts (Acme's base code)
+export { obj, arr, str, int, remote, local, defineRegistry } from "@earendil-works/chord/next";
+export { Workspace, Panels, defineFacet } from "../../core.ts";
+

One number, the app's, describes everything a plugin gets from its host. When Acme upgrades Chord in a way that +breaks plugins, Acme bumps its API version, and decides whether /v2 survives as a thin layer. +Generations: if the app provides /v3 and /v2, a facet must import from one of them only, never both (the +build checks it), because /v2's Panels and /v3's Panels may be different registries. +Recommendation: yes; app kit apps get it for free (chord/next/app provides the API module).

+

3. The API split by host kind

+

Problem: the API module holds everything plugins need, including web things: a Sheet component, a theme, the +Panels registry whose entries are React components. A worker plugin imports @acme/app/v3 for Workspace, and the +worker process has to load the module, so it loads React. +Proposal: /v3 holds what every host may use (contracts, registry tokens, host kinds, principals) and imports no +UI library at runtime; /v3/web and /v3/tui hold UI pieces. A worker facet importing /v3/web is a build error. +Recommendation: yes.

+

4. Host kinds on registry tokens

+

Problem: Bob contributes his panel from worker.ts by mistake (self.contribute(Panels, …)). The worker has no +Panels registry; the contribution sits in a table nobody reads. Nothing fails; the panel never appears. +Proposal: a token names the kinds that serve it, defineRegistry("app.panels", { version: 1, hosts: ["web"] }); +contributing from another kind is a type error (the facet's self is typed per kind) and a load error. +Recommendation: yes; together with point 3, Panels lives in /v3/web anyway.

+

5. Contracts for third parties

+

Problem: with point 1, Bob needs Acme's plugins' contracts as an npm package. Which package? +Options: (a) the app's API exports the contracts of its own plugins that others may use (@acme/app/v3 has +Notes): one place, versioned with the app; (b) Acme publishes its app package to npm and Bob depends on it; (c) each +plugin publishes its own contracts package. (c) is what the owner called unpleasant. +Recommendation: (a) for contracts Acme wants others to use: the API is the app's promise. Note: in (a) the +contract file comes with the API version, so Bob's v2 import keeps v2's contract files (the thin layer keeps them).

+

6. Testing a plugin alone

+

Problem: testing Bob's stats needs a worker with a Notes provider, an app API, a store. +Proposal: testPlugin("plugins/stats", { kind: "worker", stubs: [notesStub] }) builds the plugin as production +does, loads it on an in-memory host with the kind's real host packages, and loads stub facets that serve the +contracts it uses. A stub is an ordinary facet (host.serve(Notes, () => ({ list: constant({ items: ["a b"] }) }))). +Recommendation: yes, in the app kit.

+

7. Single-file plugins

+

Problem: a plugin that adds one route needs a directory and a package.json with a name. +Proposal: plugins/hello.server.ts is a plugin named hello with one server facet, no dependencies; when it +needs a package or a second file, it becomes a directory. +Recommendation: yes, cheap; agents write many such plugins.

+

8. The app's own plugins follow the same rules

+

Problem: are Acme's built-in plugins special? +Proposal: no. They import the app's API like everyone (not base code's internals), import each other relatively +(point 1), are built at startup and reload like any plugin. The only difference: they change in the same commit as +base code, so they always use the newest API version. +Recommendation: yes. It keeps base code small and means every built-in plugin is an example of a correct plugin.

+
+ +
∴

What to Decide

In the order that makes each answer easier.

+
    +
  1. Reverse "import by name" (recommended: yes, in the plugin slice).
    2. +
  2. Plugins import only the app's API; Chord is re-exported in it; one generation per facet (recommended: yes).
    3. +
  3. Split the API by host kind (recommended: yes).
    4. +
  4. Host kinds on registry tokens (recommended: yes).
    5. +
  5. Contracts for third parties through the app's API (recommended: (a)).
    6. +
  6. testPlugin with stub facets (recommended: yes).
    7. +
  7. Single-file plugins (recommended: yes).
    8. +
  8. Built-in plugins follow the same rules (recommended: yes).
    9. +
+

Later, not part of these: project plugins (per working directory, with trust), and installing plugins from npm or git.

+
+ +
A

Words

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
WordMeaning
base codethe app's own program, started by node server.ts; never reloads
API modulethe part of base code plugins may import, one per version (@acme/app/v3)
host packagea module the process provides to every plugin (Chord, the API modules, React on the web)
contractan object type: its state and methods, with a version; consumers name the version they use
acceptsthe provider's mapping that serves an older contract version
registry tokenan extension point (Panels); plugins contribute to it
faceta plugin's code for one host kind (worker.ts)
generationone set of API versions provided together (/v3 with its web and tui modules)
+
+ +

The Plugin Model, a Primer · first edition · 2026-10-31

+
+ + diff --git a/packages/chord/docs/next/slice-1.md b/packages/chord/docs/next/slice-1.md new file mode 100644 index 000000000..e0f17f077 --- /dev/null +++ b/packages/chord/docs/next/slice-1.md @@ -0,0 +1,236 @@ +# Slice 1: one host, local objects (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-22)**, after Astra's and Fable's spec reviews. The single source for building slice 1, +together with the feature map in `slices.md` (entries assigned to slice 1) and `design.md`, read in full (the most +relevant sections: §3 host, §5-§7, §8, §9, §11, §12, §16, §18); rules come from the decision log +(`packages/chord/docs/next/decisions.md`); code style: the section "Chord v2 Code Style" of the root `AGENTS.md`. Builds on +L1 (`src/next/contracts`, `src/next/data`) and Chord's shared watch handle (`src/watch.ts`). + +## What an app can do after this slice + +```ts +const Notes = local("notes.store", { + version: 1, + params: {}, + members: { + notes: obj({ list: arr(Note) }), + titles: arr(str()), + count: int(), + add: method(obj({ text: str() })), + saveDraft: method(obj({ conversation: int(), body: str() })), + }, +}); +const Draft = local("notes.draft", { + version: 1, + params: { conversation: int() }, + members: { summary: str() }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(Note) }), version: 1, initial: () => ({ list: [] }) }); +const Drafts = defineState("drafts", { schema: obj({ body: str() }), version: 1, initial: () => ({ body: "" }), keyed: true }); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { // empty params: lives as long as the host + const notes = self.state(NotesDoc); + const drafts = self.state(Drafts); // keyed: one document per conversation (P3 pattern) + const title = mapShared((note: Note) => note.text.slice(0, 20)); + return { + notes, + titles: self.derive([notes], ([n]) => title(n.list)), + count: self.derive([notes], ([n]) => n.list.length), + add: async ({ text }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(notes)).list.push({ text }); }, ctx); + return null; + }, + saveDraft: async ({ conversation, body }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(drafts, String(conversation))).body = body; }, ctx); + return null; + }, + }; + }); + host.serve(Draft, ({ conversation }, self) => { // params: lives while referenced + const store = self.get(Notes, {}); // declared here, live before this object is + return { summary: self.derive([store.count], ([c]) => `conversation ${conversation}, ${c} notes`) }; + }); +}); + +const host = createHost({ onError: (error) => log(error) }); // no address: local types only +await host.load([{ name: "@app/notes", facet }], ctx); +const notes = await host.get(Notes, {}, ctx); // live and hydrated +await notes.add({ text: "milk" }, ctx); +notes.count.value; // 1 +const w = await notes.count.watch(ctx); +w.start(async (value) => console.log(value)); +notes.release(); // ends w; the object lives on (empty params) +await host.close(ctx); +``` + +## 1. Host and facets + +- `createHost({ telemetry?, onError? })`: a host without an address; it serves only `local` types in this slice. + `telemetry` is the adapter (NOOP by default); `onError` receives background errors (a throwing watch listener). +- `defineFacet

(setup)` returns a facet typed for the app's principal type `P` (base code binds it once; nothing is + checked at runtime). `setup(host, self)` runs synchronously inside `host.load` and only declares: `host.serve(Type, + factory)` and facet documents through `self.state(Def)` (C16, owner: built now). `host.serve` outside a setup + throws. +- **Facet documents (design §9, §16):** the facet is a pure object with the host's lifetime; its documents live as long + as the host. Objects the facet serves may read the facet's handles and draft them in their own `self.tx`, together + with their own documents, atomically (lock order: the object's own line, then the facet's line). The facet never + drafts a served object's documents; objects of other facets cannot use them. +- `host.load(facets, context)`: `facets` is `[{ name, facet }]` with npm-style plugin names; duplicate names or a type + served twice are refused. After every setup ran, every served `local` type with empty params is activated (Q-A, the + start barrier); `load` resolves when all are live. If one fails, `load` ends everything it activated and rejects; + the host is then empty. A `host.get` made before `load` resolves waits for it. +- `host.close(context)`: rejects new calls, ends every object (in reverse activation order) and resolves when all have + ended. + +## 2. Objects + +- **Identity:** one object per `(type, params key)` (L1's `checkParams` and `identityKey`; local has no principal). +- **Lifetime (Q-A):** a local type with empty params is created at `load` and lives until `close`; a type with params + is created by the first `get` and ends when its last reference is released. +- **Factory:** `(params, self) => implementation`, synchronous and declarative, run once per object. The + implementation has exactly the contract's members: a state member is a JSON `Watchable` of its value (a document + handle, a derive, a member of a reference), a method is `(args, ctx) => Promise`. L1's `Implementation` + types state members as `Watchable>` (owner, 2026-10-22). +- **Activation (design §6):** run the factory; acquire the references it declared with `self.get` (activating their + objects if needed) and wait until they are live; check the members (exactly the contract's, state members JSON + watchables, Q14); compute every derive's first value; then the object is live and calls proceed. A throw anywhere + in activation (factory, a dependency's activation, a member check, a first derive compute, `initial()` failing its + schema) rejects the `get` (or `load`) with that error, releases everything acquired and leaves no object. Failure + states, backoff and retries are slice 2. +- **Ending:** new calls on the object fail `released`; the context of each of its running calls is aborted; from then + on its `self.tx` throws `released` (a still-running method can no longer change anything); its derives are detached + from their sources (no source keeps a reference to an ended object's derive); its references are released; its + documents are dropped; every watch on its members ends `ended`. Ending does not wait for running calls (the drain + arrives with natives in slice 2, design §3). + +## 3. References + +- `host.get(Type, params, context)` returns a reference once its object is live; `self.get(Type, params)` declares one + in a factory: it is live by the time the object goes live, and released when the object ends. +- A reference is a plain object built from the contract's member list (no proxy): `ref.` is a `Watchable` + (`value`, `watch(ctx)`), `ref.(args, ctx)` calls the method, `ref.release()` releases it (idempotent). After + release, every use throws `released`. +- **Calls:** arguments and results pass by reference; arguments are checked at the provider with L1's `checkArgs` + against the current version (`bindProvider(contract, {})` at serve): unknown method `invalid/member`, bad arguments + `invalid/args`; results are not checked; no-argument methods take `null`. +- **Per-reference watches** (owner): watches obtained through a reference end (`ended`) when it is released, even + while other references keep the object alive. +- **Contract match** (owner): at `get`, the caller's contract must have the provider's version and fingerprint (the + same declaration, or an identical one), otherwise `contract_mismatch`. Older versions through `accepts` come with + slice 7. +- **Reserved member names** (owner): `then`, `release`, `state` and `retry` are refused as member names at + declaration (an L1 check in `remote`, `local` and `evolve`), because a reference has them (`release` now, `state` and + `retry` in slice 2) and `then` would make an awaited reference thenable. +- **Cycles (Q5, one host; owner):** at `self.get` during an activation, Chord follows the chain of objects waiting for + activation; reaching the object being activated fails that activation with `invalid/cycle`, naming the cycle + (`a → b → a`); nothing is kept. +- `ctx` in this slice is a plain `Context` (cancellation and telemetry); `ctx.get` and `ctx.principal` come with + slice 3. + +## 4. Documents in memory (§9, P2) + +- `defineState(name, { schema, version, initial, keyed? })`: the root is a JSON object. `checkpointWhen` is required + only for persisted documents (slice 6; owner, 2026-10-22). `migrate` comes with slices 6 and 7. +- `self.state(Def)` returns a handle bound to this object; every document is in memory, instance state of the object + (P2), gone when it ends. +- **Singleton handle:** `handle.value` is the current value, `initial()` until written or after a retire (Q11); the + handle is a `Watchable`. +- **Keyed handle:** `await handle.get(key)` gives the value or `undefined`; `handle.keys()` is an `AsyncIterable` of + the present keys (the shape persistence keeps); keys are strings. A keyed handle is not a `Watchable` (a live + collection is a singleton holding a list or map). +- `initial()` is checked against the schema once, the first time a definition's `initial()` is used (R8). + +## 5. Transactions (§9, R12, R13) + +- `self.tx(fn, ctx)`: an object's transactions run one at a time (its line). `fn` gets `tx`: `await tx.draft(handle, + key?)` returns the document's draft (Delta's `track`, `beginChange`; the same document gives the same draft within + one transaction); `tx.retire(handle, key?)` removes a document (a no-op when absent). `self.tx` returns what `fn` + returns. +- **Commit:** when `fn` resolves, if `ctx` is aborted the transaction is discarded; otherwise every draft is prepared + (Delta's `prepare`: ops recorded, only placed values checked), unchanged drafts commit nothing, and all changed + documents are adopted (pointer swap) in **one batch** (section 6). A throw discards everything. +- Drafts are revoked when the transaction settles (touching one throws, R12); a transaction returns plain values and + data is read from the handle after the commit. +- A transaction drafts only its own object's documents and its facet's (any other handle throws). Inside `fn`: no + second `self.tx`, no calls to other objects (Durable's rule; documented, not detected). +- **Shorthands (C14, owner: built now):** `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) runs one + transaction that loads the document and calls the synchronous `fn(draft)`; `handle.retire(ctx)` (keyed: + `handle.retire(key, ctx)`) retires it. Both are thin wrappers over `self.tx`. + +## 6. Derives and the batch (W1, §11, §12) + +- `self.derive(sources, compute)`: sources are JSON `Watchable`s (this object's handles and derives, members of + its `self.get` references); only the declared sources are tracked. `compute` is a synchronous pure function of their + values; its result is JSON. +- **Pull model.** A commit only marks: every derive that depends on a changed document, directly or through other + derives, including derives of other objects on the host, is flagged dirty (O(dependents), flags only). A derive's + `value` recomputes when it is dirty, after reading its sources' values (which recompute first if they are dirty); + so each derive computes at most once per batch and always after its sources, diamonds included. +- **The batch** (one per host): a commit's adoptions happen inside it; after the last adoption, still synchronously, + every watched value that changed is read (watched derives recompute through the pull), its ops computed + (`diffRevisions(old, new)` for a derive, the commit's ops for a document), and its watches get one frame each. A + derive is recomputed in the batch only if something watched depends on it; otherwise it stays dirty until read. + Reading `.value` once does not make it watched. +- A derive that throws fails only itself: its `value` throws the error and its watches end `failed`; a derive over it + throws the same error when read. A derive that fails during activation fails the activation (section 2). A failed + derive stays failed until the object is rebuilt. +- **Reentrancy:** inside the batch (adoptions, derive computes, including lazy computes outside a commit), `self.tx`, + `watch`, `release`, `get`, `load` and `close` throw `invalid/reentrant`. One flag per host, cleared in a `finally`. +- `mapShared(f)`: maps an array calling `f` only for object elements not seen before (by identity), returning the + previous result for unchanged ones; primitive elements are mapped every time; the cache is a `WeakMap`. +- `constant(value)`: a JSON `Watchable` that never changes (start data, §18); usable as a state member and a source. +- State is JSON only (L2 decision): no bytes in documents or derives. + +## 7. Watching + +`watchable.watch(ctx)` returns Chord's shared `Watch` (`src/watch.ts`): the value at acquisition, then one frame per +batch that changed it, `(value, ops, ctx)`, in order, asynchronously, with its overflow, `stop` and end semantics. +Each frame is consistent with its commit; two different watches are not ordered relative to each other. A retired +singleton gives the frame `[["r", initial]]` (it changes back to `initial()`; a singleton always has a value). `ctx`'s +abort cancels the watch. A listener that throws ends its watch `listener_error` and is reported to `onError` (taken +from the carry-over). + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a call | L1's three lookups and one compiled check; arguments by reference | +| a commit | Delta prepare per changed document, pointer swaps, dirty flags per dependent, then per watched changed value one read (derives recompute once) and one frame per watch, by reference | +| an unwatched derive | nothing until read | + +Counters (telemetry, where the work happens): L1's, plus `chord.derive.computes` and `chord.derive.diffs`. + +## Carry-over taken by this slice + +From `carry-over.md` (L4 section): watch listener errors reported to `onError`; native watchables refused as state +(by the member check: only JSON watchables exist in this slice); `defineFacet` typed by `P`. Left for +later: plugin error shape, old-version calls (7), `broken()` (2), cuts C5, C6, C7, C13 (2). + +## Tests + +- Host: load runs setups, activates empty-params types, rejects duplicates and double serves; a failing activation + makes `load` end what it activated; `get` during load waits; `serve` outside setup throws; close ends everything. +- Objects: one object per params key; params type created on first get and ended on last release (documents gone, + watches end `ended`, its derives detached from other objects' members); empty-params type lives across releases; + activation failures (factory, dependency, member check, first derive, `initial()`) reject the get and leave nothing; + a method running while its object ends cannot commit (`self.tx` throws `released`) and sees its `ctx` aborted. +- References: hydrated on get; members watchable; calls checked; released reference throws; `self.get` dependency + kept alive and released on end; per-reference watches; contract mismatch; reserved member names refused at + declaration; a two-object and a three-object cycle fail with `invalid/cycle` and leave nothing. +- Documents: singleton initial, retire back to initial; keyed get, keys, retire; `initial()` checked once. +- Transactions: serialized per object; same draft twice; unchanged commits nothing; throw and abort discard; drafts + revoked after settle; another object's handle refused; a served object's transaction over its own and its facet's + documents commits both in one batch; shorthands. +- Batch and derives: two documents in one transaction reach a derive over both in one compute and a watch in one + frame; a derive over another object's member updates in that object's commit's batch; diamond computes once; + unwatched derive stays lazy and a one-off read does not make it eager; a throwing derive fails only itself and its + dependents; reentrancy throws in commits and lazy computes and the flag clears after a throw; `mapShared` identity. +- Watching: frames in order, retire frame, cancellation, listener error reported. +- Cost guards per the table. GC: an ended object with its documents, derives and watches is collected; a released + reference and a stopped watch hold nothing; a detached derive is not held by its former sources. + +## Size + +About 1,100 source lines. diff --git a/packages/chord/docs/next/slice-10.md b/packages/chord/docs/next/slice-10.md new file mode 100644 index 000000000..6bbaea50a --- /dev/null +++ b/packages/chord/docs/next/slice-10.md @@ -0,0 +1,285 @@ +# Slice 10: HTTP and workers (spec) + +Status: **built** (2026-10-31; build reviews by Fable, Opus, Astra and Sol, security audits by Fable, Opus and Astra, four fix rounds with rechecks; the shape after the fix rounds is in §2, §3 and design §23). Approved by the owner (2026-10-30: "just build it"; D1-D12, D10b, round 1's R1-R6 and round 2's Q1-Q6 in the +decision "Slice 10's points"; spec reviews round 1 by Fable and Opus, `/tmp/s10-spec-review-{fable,opus}.md`, round 2 +by Fable, Opus, Astra and Sol, `/tmp/s10-spec-review2-*.md`, folded in). +Slice 10 builds two Node components of the app kit (decision "Slice 7b's open points" (6)): `chord/next/http` (the +server's HTTP and WebSocket entry, routes contributed by plugins through a registry, responses as descriptions) and +`chord/next/workers` (a worker process per host address, started on demand, stopped when idle, its plugins shipped from +the server). Both live in `packages/chord/src/next-node/` (Node-only); the kernel stays portable. The web page host and +the TUI are slice 11, telemetry slice 9. + +Read in full before writing code: this spec; `decisions.md`; `design.md` (§3 hosts, routes, `host.activity`, §17, +§18, §21 "Loading plugins and the reload loop", §23 in full); `slices.md`; `slice-7c.md` and +`src/next-node/plugins/README.md`; the 7b brief `slice-7b/book.md` (chapters "Assets and Files", "HTTP", "A Mini p2 +from Chord"); p2's `src/hosts/server.ts`, `src/runtime/server.ts`, `src/hosts/worker.ts`, `src/hosts/socket-limits.ts`, +`src/hosts/principals.ts`, `plugins/files/server.ts`, `plugins/terminals/server.ts`, `plugins/radius/server.ts`; Chord +v1's `src/manager.ts` (`processLock`, `forkDriver`, `createHostManager`, `childControl`). + +## 1. What p2 has, and what moves into Chord + +p2's server host (`src/hosts/server.ts`, 742 lines; `src/runtime/server.ts`, 720) does, besides p2's own policy: + +| p2 today | Moves to | +| --- | --- | +| a Node `http` server on loopback, requests parked until `load` resolved, upgrades captured before listening | `http` | +| `Routes` (`"METHOD /path"`, `/prefix/*`), handlers writing Node's `ServerResponse` themselves (ranges, download headers, `HEAD` by hand in `files`) | `http`: routes returning descriptions | +| `Upgrades` (a WebSocket by path for terminals and voice), `/chord` (`acceptWebSockets` with per-socket limits) | `http` | +| `Authenticators`, `RequestResolvers` (cookies, devices, the Radius relay), Fetch Metadata, CSP and the loopback cookie | the app (p3's base code), through `http`'s `authenticate` and `headers` (D3) | +| loopback and allowed hosts, the `Origin` check on upgrades | `http` (R4), allowed hosts contributable by plugins | +| `SessionHosts`: a worker per session, `createHostManager` + `forkDriver`, a kernel lock per session, idle stop from reported counts, crash notices, sticky failed starts, ready timeout | `workers` (failures stay until Retry or a rollout, R2) | +| relocation and pause of a worker, the catalog, `WorkerInitValues` | the app (built on `workers`' `stop` and start data) | +| `hosts/worker.ts`: the child's control channel, lock, start data, connect, load, report counts, shutdown | `workers`' child side | +| the dev reload across server and workers, worker plugin lists | `plugins` (7c) plus `workers`' rollout step (§4) | + +Every option that is not bootstrap is changeable by plugins (decision "The app kit is malleable by plugins"): either a +contribution to `Http` or a function the component calls per use, which the app backs with its own registries; +nothing is copied out of a contribution or an option at startup. Bootstrap (changed by a restart): the listen address +and port, the links path, the worker entry with its `execArgv` and `env`, the host kind workers serve. + +## 2. `chord/next/http` + +As built after the fix rounds (decisions "The HTTP core is a fetch handler", "The HTTP server is a native object", +"Settings are arguments, extensions are registries", "The HTTP component's security baseline", "allowed hosts only +through contributions", "A CSP nonce per response", "three small points", "a route's overhead accepted"); design §23 +states every rule, `src/next-node/http/README.md` the API. + +```ts +const http = createHttp(server, { + listen: { host: "127.0.0.1", port: 8788 }, // settings: arguments, a restart to change + links: { path: "/chord", options: (request, principal) => ({ limits: linkLimits(principal), maxPayload: 64 * MiB }) }, + closeTimeout: 30_000, + onError: (error) => log(error), // `HttpError`, below +}, ctx); +const app = defineFacet((_host, self) => { + self.contribute(Authenticators, (request, ctx) => principalFromCookie(readCookie(request, "session", secret))); + self.contribute(Http, { hosts: ["notes.example.com"], relays: [radiusRelay] }); +}); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }, { name: "@app/core", facet: app }], modules }, ctx); +… +await http.close(ctx); // first: no new requests, sockets closed, running requests waited for up to `closeTimeout` +await workers.close(ctx); +await server.close(ctx); +``` + +- **The core** is `http.fetch(request, { localAddress?, localPort?, remoteAddress? })`, a Web `Request` to a Web + `Response`; `http.upgrade` is Node's upgrade adapter; `nodeRequest(message, res)` converts a Node request with the + listener's checks (`{ kind: "request", request, connection } | { kind: "refused", status }`, never a throw) for apps + mounting on their own Node server. Chord's own listener (with `listen`) is a host-lifetime object of `chord.http` + whose native is a Node server, on a fast path that skips the Web `Request` unless a route or authenticator reads + it. +- **Registries:** `Http` (`"chord.http"`, version 1: `routes`, `wraps`, `sockets`, `maxPayload`, `hosts`, `framers`, + `relays`, `csp`; a value or a writable) and `Authenticators` (in contributor order, the first principal wins). + Requests before the publication wait for it (503 past `readyTimeout`). +- **Admission, foreign requests, origins, default headers, cookies, downloads:** the security baseline (design §23). + Refusals before any plugin code: a non-loopback socket without an allowed host, forwarded headers without a + verified relay, duplicate or missing `Host`, userinfo in `Host`, absolute-form targets (403), unparsable targets + (400), `TRACE`/`TRACK`/`CONNECT` (501; a route for them refused at compose). +- **A request's scope, `HttpRequest`, patterns, wraps, responses, `fileResponse`, sockets, links, revocation, + `close`:** design §23. The spec's earlier text (an `httpServer` that listens first, `allowedHosts`, one + `authenticate` and a `headers` option, 64 KiB reads) is superseded by those decisions. +- **Errors** (`onError`, `HttpError`): `route_failed { pattern, plugin, error }`, `wrap_failed { name, plugin, error }`, + `socket_failed { pattern, plugin, error }`, `response_failed { pattern, error }`, `hook_failed { hook, plugin?, + error }` (`authenticate`, `relay`, `links.options`). +- `ws` is an optional peer dependency (D7), imported only here. + +## 3. `chord/next/workers` + +```ts +// server.ts +const workers = createWorkers(server, Session, { + kind: "worker", // the facet kind it ships + entry: "./worker.ts", execArgv: ["--import", resolver], env: { … }, // bootstrap + authorize: (principal, address) => app.mayReach(principal, address), // the route's `authorize` (synchronous) + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: (address, ctx) => app.startData(address, ctx), // JSON, before each start + lock: (address) => `app:${realpath(dataDir)}:worker:${address.key}`, // one live worker per address + idleMs: () => app.idleMs(), + plugins, // the plugins component + onOutput: (address, stream, chunk) => logs.append(address.key, stream, chunk), + onError: (error) => log(error), // `WorkersError`, below +}, ctx); +workers.state // JSON watchable (R5) +await workers.stop(Session({ id }), ctx); // archive: suspended while held (Q3), then stopped +await workers.retry(Session({ id }), ctx); // the Retry button: a failed or suspended address resolves again +await workers.close(ctx); // no new starts, every child stopped +``` + +```ts +// worker.ts +await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(start.dir, "session.sqlite"))), ctx); + onClose((ctx) => store.close(ctx)); // closers run in reverse order + const worker = createHost

({ address, principal: { kind: "worker", session: start.id }, store, onError }); + onClose((ctx) => worker.close(ctx)); + worker.connect(parent, { principal: { kind: "server" }, trusted: true }, ctx); // `started` goes out now: connect + // before anything slow (a store opens in ms) + await plugins.load(worker, { kind: "worker", core: [{ name: "@app/core", facet: coreWorker(start) }] }, ctx); + return { busy: harness.activeTasks }; // extra keep-alive counts +}, { + hostPackages: { "@earendil-works/chord/next": chordNext, "@app/core": appCore }, // the child's own instances + redact: (error) => ({ message: redactSecrets(String(error)), ...codeOf(error) }), // what crosses as `fatal` + onError: (error) => log(error), +}); +``` + +- **The route:** `createWorkers` calls `host.route(Kind, { authorize, resolve }, ctx)` itself and keeps the + `RouteHandle` (`retry`, `suspend`: D10b, R1). `authorize` is synchronous (the kernel's): a plugin backing it keeps an + index. +- **Per address, events and operations:** a child's events (its exit, its link's close, `started`, `ready`, `fatal`) + are observed as they happen and recorded at once on that child (a late event of an old child never changes a new + one); the operations (start, stop) run in one queue per address: a start waits for the previous child's exit, a + stop during a start waits for it then stops, and a start queued behind an exit checks the address's state after it + (a crash recorded meanwhile makes it reject with that failure). Stops record their intent before `shutdown`, so the + exit they cause is no crash. +- **A start** (`resolve(address)`): calls `start(address, ctx)` (abortable; a throw rejects the start), checks that the + component is not closing, forks `entry` (`serialization: "advanced"`, a detached process group, stdout and stderr + piped: the stderr tail kept, mirrored to the server's when there is no `onOutput`), makes `ipcLink(child)` at once + (so the child's `hello` and early holds are queued, never lost), and sends the start message `{ address, start, + scripts, lock }` (`scripts`: the worker kind's complete current set, read when the message is sent). The child takes + the lock (when an old worker still holds it, it waits by connecting to the lock's socket and trying again when that + connection closes: no polling), and sends `started` when the app's host sets its listener on `parent` (`parent` is + a thin wrapper of the IPC link, so `connect` is the signal: the app connects before anything slow). `resolve` then + returns `{ link, principal: principal(address), trusted: true }`, and the handshake and the worker's server traffic + run during its `load`. The start timer (60 s from the fork) covers fork, lock and connect only (Q4): past it the tree is + killed and the start fails. `ready` (after the app's function returned, carrying the idle state and its load report) sets `state` to + `running` and lets idle stops begin; a `load` that never ends shows `starting` until `stop`. +- **Workers never restart by themselves** (R2): a crash (an exit nobody asked for), a `fatal`, a link lost while its + child lives, or a failed start (`start` threw, the start timer, a lock not taken in time) leaves the address `failed` + with its error (`worker_failed` reported once). A link lost while its child lives (the heartbeat after a blocked + event loop, a protocol error) starts the stop: the grace (5 s), then the tree is killed; its exit is the crash ("link + lost"). `resolve` of a failed address rejects at once with that error; its holders and new holds see `failed { + waiting }` with it as cause (Q1) and stay on the address's path, nothing forks. `workers.retry(address)` + (`RouteHandle.retry`) or a rollout reaching workers clears it (Q2, §4). An idle stop is no failure: the next hold + starts the worker on demand. +- **Keep-alive** (D9, R3, Q5): the child's idle state is all of `host.activity`'s `references`, `jobs` and `lingering` + and the app's `busy` (a JSON watchable of numbers) at zero, not `calls` (a remote call runs on a hold; local calls + come from routines, jobs and tasks counted elsewhere). `ready { idle }` carries the first state, then `{ idle }` on + each flip; a rollout applying in the child counts as busy. The server stops a worker idle since `idleMs()` (5 min by + default; checked every second), never before `ready`. A stop (idle, `stop`, `close`): `shutdown`; the child aborts its + `ctx`, runs the closers registered with `onClose` in reverse order and exits, forcing its own exit after 3 s; the + server kills the tree after 5 s. +- **`state`** (R5, decision "worker errors in public"): per address key `{ status: "starting" | "running" | + "stopping", pid, startedAt }` or `{ status: "failed", error: { message, code?, reason? }, at }` with a public error + (a fixed message, or the child's redacted `fatal`); no entry when stopped or never started. The full error and the + stderr tail go only to `onError` (`worker_failed`); every app hook of a start (`start`, `principal`, `lock`, the + fork) is inside that boundary. +- **`stop(address, ctx)`** (R1, Q3): `RouteHandle.suspend(address)`, then the stop. The suspension lasts while the + address has holders (they see `failed { waiting }`, "suspended"); with the last holder it goes, and a later hold + starts the worker on demand; `retry(address)` lifts it at once. Whether an archived session may be opened is the + app's (`authorize`, its views). +- **`close(ctx)`:** cancels active and queued starts first (a `start(address)` waiting on the app's barrier is + aborted; nothing forks after it), then stops every child (grace, then kill), clears the timers; new starts reject + `released/closed`; the caller's cancellation ends only its own wait. `createWorkers`'s `ctx` aborting (the host + closing aborts the route's context) closes it too, and from then on every exit is a stop, never a crash (no + `worker_failed` at shutdown). +- **The child** (`workerMain(fn, { hostPackages, redact, onError })`): reads the start message, takes the lock, makes + `ipcLink(process)` and its wrapper `parent`, creates its plugin runtime with `hostPackages` (the child's own module + instances; functions cannot cross IPC), calls `fn({ address, start, parent, plugins, onClose }, ctx)`. `shutdown`, + the parent's disconnect, `SIGTERM` and `SIGINT` alike abort `ctx` and run the closers registered so far, also while + `fn` still runs (its startup is cancelled through `ctx`). An uncaught error, or `fn` throwing, sends `fatal` with + `redact(error)` (never the original message) and exits; after `started` that is a crash like any other. +- **Control messages** (JSON beside the link's `Uint8Array` frames; the IPC link skips them): parent to child `start`, + `shutdown`, `reload { scripts }`; child to parent `started`, `ready { idle }`, `idle { idle }`, `fatal { error }`, + `reloaded { report } | { error }`. One `reload` is outstanding per child at a time: a reload whose caller aborted + keeps the child's slot until its `reloaded` or the child's exit, and the next waits for it. Errors cross as `{ + message, code?, reason? }`. +- **Errors** (`onError`, `WorkersError`, which includes `BatchError` for a `state` watch listener that throws): `worker_failed { address, error, stderrTail? }` (private), + `authorize_failed { address, error }` (a throwing `authorize`: the hold is refused `forbidden`), `worker_exited { + address, code, signal }` (only for an exit nobody asked for: not after a stop or a start the component failed). + +### Plugins in workers (D11) + +Workers never build. The plugins component (7c) gets `scripts(kind, ctx)`: the complete current set of one kind (every +discovered plugin's facet of it, built on first use, then updated by every rollout that ships that kind), each with its +plugin record (`name`, `dir`, `chord.external`). The component's per-host loading (its loaded map, `#prepare`, `adopt`, +the queue per host) moves into a file without esbuild or the watcher, `createPluginRuntime()` +with `load(host, { kind, core, scripts, modules }, ctx)` (`modules`: the host packages' module objects of this process) and `reload(host, scripts, ctx)` (a complete set: removed = loaded − +shipped; same digests unchanged); `createPlugins` uses it, and the child's `plugins` is a `WorkerPlugins` over one +(`load(host, { kind, core })` loads the start message's scripts; `reload` messages wait until `fn` returned); `import` +of `chord/next/plugins/runtime` loads no esbuild. Each kind is defined once in `createPlugins({ kinds: { : { +platform, hostPackages } } })` (decision "Slice 11's points", Kinds). + +## 4. The rollout across server and workers + +```ts +const built = await plugins.build(plugins.state.value.dirty, ctx); +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +// result.steps: [{ name: "server", hosts: [{ host: "server", report }] }, +// { name: "workers", hosts: [{ host: "session|S1", report }, { host: "session|S2", stopped: true }, +// { host: "session|S3", error }] }] +``` + +`RolloutStep.reload(built, ctx)` returns `{ hosts: readonly ({ host; report } | { host; stopped: true } | { host; error +})[] }` (D12; 7c's in-process step returns a list of one, its `host` the name given to `step`). `workers.step(name)`: + +1. replaces the workers' current set (`plugins.scripts(kind)` updated from `built`); +2. clears the failures recorded before this rollout and calls `RouteHandle.retry` for those addresses that have holders + (Q2): they start with the new set and are listed; an unheld one starts at its next hold; +3. ships the complete set to every running or starting worker whose start message or last reload carried an older one + (a child queues a reload that arrives before its `ready` until its load is done); workers reload in parallel. + +A worker's entry settles on its `reloaded` or on its exit or link loss, whichever comes first. A worker stopped cleanly +during the step (idle, `stop`) is listed `stopped` and dirties nothing (its next start gets the current set); one that +crashed or failed to apply is listed with its error and does not stop the rollout, and every plugin of its kind in the +build counts as failed there (stays dirty); a worker failing during this rollout stays failed. A plugin's dirty mark +clears only when it applied on every host of every step, and a kind built in the rollout that no step shipped keeps its +plugins dirty. + +## 5. Not in slice 10 + +The web page host, the browser's evaluation of scripts and serving them (slice 11); the TUI (slice 11); relocation and +pause of workers (the app's: `start(address)` is where the app awaits its own barrier); request resolvers and +authenticators as registries (the app's); chunked transfers as a Chord helper (an owner pattern, design §23); the +channel component (D6); telemetry spans for requests (slice 9). + +## 6. Kernel changes + +- `host.route(Kind, options, ctx)` returns a `RouteHandle { retry(address), suspend(address) }` (D10b, R1, Q3). +- A route has no backoff timer (R2): a rejected `resolve`, or a link that closes before its handshake completed, fails + the route's holders `failed { waiting }` with its error (they stay on the address's path, Q1) and resolves nothing + more; the route resolves again only for a new hold, on `retry`, or once after a link that had completed its handshake + dropped while holds exist; a suspended address resolves nothing while it has holders and forgets the suspension with + the last one. +- `host.root(fn, ctx, { principal: undefined })` runs anonymous (`ctx.principal` undefined); only an omitted + `principal` takes the host's (a public-API change of `RootOptions`: `"principal" in options`). +- Origins only from relays (decision, 2026-10-31, security audit): `origin` is read only on parent and route links; + on a child link it is a protocol error; a per-origin record goes once it has no holds, calls or messages in flight. +- An anonymous `host.root` scope gets no remote object served on its host (`forbidden`). +- Design §3: the paragraph on a `resolve` waiting for the worker's `load` is updated (done), and "Chord never closes a + route link itself except at `host.close`" becomes "except at `host.close`, the heartbeat and protocol errors". + +## 7. Tests, examples, cost + +- One test file per source file. HTTP: listening before the publication (a request waits, then is served; 503 past + `readyTimeout`); the router resolved once; `Host` (loopback forms, allowed from the option and from a writable + contribution changing at runtime) and `Origin` checks; anonymous requests (`ctx.principal` undefined on a server with + a principal); authentication per request and upgrade; throwing hooks; patterns, decoding, precedence, `HEAD`, 404; + conflicts reported; wraps (phases, order, scopes, a missing route, a throwing wrap); every response kind (ranges + 206/416, multi-range, `HEAD`, download names, chunked without `size`, a short `read`, a disconnect stopping `read`, + header merging); `json`/`body` limits and their statuses, an unread body; a throwing route; the links endpoint + (`host.accept`, options per socket, a refused socket, the output cap); plugin sockets (an oversized frame closes only + that socket; a reload removing the contribution closes its sockets 1012); a removed allowed host closing its + connections; a reload swapping a route while a request runs on the old router; `fileResponse` closing its handle on + every path; `close` (refusals, waiting, then sealing). Workers: a start on the first hold; the link made at the fork + (a load longer than the heartbeat's 30 s binds); `started` on `connect`; the start timer covering only the start (a + slow `load` is not killed); the lock (an old worker's lock waited for without polling); `start` throwing; a crash, + a `fatal`, a link lost with a live child (killed after the grace), and a failed start each staying failed (no fork for + a new hold; holders `failed { waiting }`), `retry` and a rollout clearing them and rebinding the holders; idle stop, + a worker idle from its start, a hold keeping it; `busy`; 1,000 sequential local calls sending no message; + `stop` with holders (suspended, then forgotten with the last holder); stop during start and start during stop; a + crash while a resolve is queued; `close` (cancelling a start waiting in `start(address)`, no `worker_failed` at + shutdown); the rollout step (complete sets, a worker starting during it, a clean stop listed `stopped`, a crash during + it, an aborted reload's late reply, a kind no step shipped staying dirty); the child's shutdown during its startup, + disconnect and signals; `redact` (a secret in the original message reaches neither `fatal` nor `state`). Kernel: + `RouteHandle.retry` and `suspend`; a link closing before its handshake (no re-resolve loop); no re-resolve without a + hold, a retry or one drop of a healthy link; `host.root` with an explicit `undefined` principal. +- GC: requests (their scopes, bodies, file handles) and stopped workers' state collected; no listener left on a closed + socket; 1,000 addresses started and stopped leave no `state` entries, no per-address queue records and no + suspensions. +- Examples (each a folder, as 7c's): `40-http` (routes, a wrap, a download with ranges, a socket, an allowed host + added at runtime), `41-workers` (a server routing a session kind to forked workers, idle stop, a crash staying failed + until Retry, a rollout reaching the workers and restarting a held failed one). +- Benchmark: a route's overhead per request against a plain Node handler (bar: within 10% of a keep-alive round + trip; Opus measured 2.5 µs of 54 µs); `bytes` throughput for a 100 MiB file against `pipeline`; a worker's cold start + (fork to `started` and to `ready`) with shipped scripts; memory per idle worker on the server. diff --git a/packages/chord/docs/next/slice-11.md b/packages/chord/docs/next/slice-11.md new file mode 100644 index 000000000..e13899000 --- /dev/null +++ b/packages/chord/docs/next/slice-11.md @@ -0,0 +1,99 @@ +# Slice 11: web and TUI surfaces (spec) + +Status: **approved** (owner, 2026-10-30/11-01: P1-P9 in the decisions "Slice 11's points" and "Slice 11, P3" to "P9", +then "go ahead, just do all the things"). Built after slices 12 and 13: the surfaces are generic plumbing that the app +kit wires in (`startApp` serves the web surface; a TUI client entry). The app keeps everything visible: HTML shell, +CSS, components, themes, navigation, layout, keys. + +Read in full before writing code: this spec; `decisions.md` (slice 11's entries, the plugin model, the app kit); +`design.md` §3 (clients), §21, §23 (all, "Web and TUI" included); `slice-10.md`, `slice-12.md`, `slice-13.md`; p2's +`src/hosts/web.ts`, `src/hosts/web/main.tsx`, `src/web/reconnect.ts`, `src/runtime/web.tsx`, `src/hosts/bundles.ts`, +`src/hosts/tui.ts`, `src/hosts/tui-client.ts`, `src/hosts/tui-sandbox.ts`, `src/hosts/tui-bootstrap.mjs`, +`src/hosts/tui-worker-runtime.ts`, `src/runtime/tui.ts` (`TuiScope`). + +## 1. `chord/next/web` (server side) + +```ts +// inside startApp (slice 13), or by hand: +const web = webSurface(server, { + kind: "web", plugins, + entry: "./src/web/main.tsx", // the app's web start code: the base bundle's entry + page: (request, assets) => shell(request.nonce, assets), // the app's HTML; assets: base script, plugins file, styles +}, ctx); +// its facet (routes to Http, the WebScripts object) goes into the server's core facets +``` + +- **Base bundle** per client kind: built on the server by esbuild from `entry`, with the kind's host packages inlined + once (the app's API modules for the kind, Chord, React), content-hashed (`/_chord/web/base..js`), cached + forever (`Cache-Control: immutable`, overriding the baseline's `no-store` for these routes only); rebuilt when base + code changes (the server restarts anyway: built at start). +- **Plugin scripts:** each web facet served as a script that registers itself (`chordFacet("", "", + function (require, module, exports) { … })`, P2), at `/_chord/web/plugin/..js`, immutable; the + current set as one combined file `/_chord/web/plugins..js` (P3); stylesheets and assets of the build + likewise, under `publicPath`. +- **`WebScripts`** (a `remote` object the surface serves, params `{}`, any principal that may load the app): its state + is the current set `{ plugins: [{ name, digest, url, css? }], base }`; a rollout that ships the web kind updates it + (the plugins component's `scripts("web")` changes; the surface follows them) without waiting for pages (P5). +- Routes and the page are `Http` contributions of the surface's facet; the page route is `page: true` (navigation from + another site allowed) and public or not as the app says. + +## 2. `chord/next/web` (browser side) + +```ts +// src/web/main.tsx (the app's entry, in the base bundle) +startPage

({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@acme/app/v3": api, "@acme/app/v3/web": apiWeb, react: React, "react/jsx-runtime": jsx }, + core: [{ name: "@acme/web", facet: webCore }], + canRetry: async () => true, + onConnection: (state) => setBanner(state), + render: (host) => createRoot(document.getElementById("root")!).render(), +}); +``` + +- A client host (no address); the connection loop (P6, p2's `keepConnected`: reconnect 250 ms to 5 s, wake on + visibility, `online` and network change, replace a silent socket, stop on an outdated protocol (reload the page) or + when `canRetry` says no); `onConnection` reports `connecting | connected | waiting`. +- Loads the combined plugins file, evaluates each registered facet with `modules` (the `require` they get answers only + host packages and `@earendil-works/chord/next/plugins`'s browser half if any), loads them with `core` in one + `host.load`, then holds `WebScripts`: on a new set it fetches the changed scripts (` + diff --git a/packages/chord/docs/next/slice-7c.md b/packages/chord/docs/next/slice-7c.md new file mode 100644 index 000000000..f70fa38f0 --- /dev/null +++ b/packages/chord/docs/next/slice-7c.md @@ -0,0 +1,165 @@ +# Slice 7c: plugins (spec) + +Status: **ready to build** (owner, 2026-10-30: decisions "Slice 7c's points" and "Per plugin at startup and reload"; +spec reviews by Fable and Opus, `/tmp/s7c-spec-review-{fable,opus}.md`, folded in). 7c builds `chord/next/plugins`: the +code that turns plugin directories into facets, loads and reloads them per plugin, and watches them. A Chord app is +self-modifying (the owner or an agent edits a plugin while it runs), so this is core machinery, not a dev tool (no +dev/prod split). It also changes the kernel's `load` and `reload` to per-plugin failure (§4). HTTP and workers are slice +10, the web page host and the TUI slice 11: 7c builds scripts for every surface and evaluates them on Node; its build +results are plain data those slices ship. + +Read in full before writing code: this spec; `decisions.md` (2026-10-29/30: "Slice 7b's open points, as recommended", +"A plugin's npm dependencies are bundled", "Per plugin at startup and reload", "Slice 7c's points"); `design.md` (§3 +`load`/`reload`, §16, §18, §21 in full); the 7b brief `slice-7b/book.md` (chapters "A Plugin, File by File", "Building +and Loading", "Assets and Files", "Performance", "A Mini p2 from Chord"); the two spec reviews; p2's +`apps/p2/src/hosts/{bundles,plugins}.ts` and `apps/p2/docs/plugin-loading.md` §4.1, §7.1, §8, §9, §14; coding-agent's +`packages/coding-agent/src/core/package-manager.ts` (dependency installs). Code lives in +`packages/chord/src/next-node/plugins/` (Node-only: `fs`, esbuild, `vm`; the kernel in `src/next` stays portable), +exported as `./next/plugins` when v2 gets its entry point; until then tests import the source. + +## 1. A plugin on disk + +```text +plugins/notes/ + package.json "name": the plugin's name (required, npm style); "dependencies": bundled; "peerDependencies": + host packages; "chord": { "external": ["better-sqlite3"] } (optional, Node facets only) + server.ts worker.ts web.tsx tui.ts one file per facet kind the app names; default export: a facet + contract.ts store.ts NoteList.tsx notes.css assets/… plain files + node_modules/ its own npm dependencies (`installDependencies`) +``` + +- The plugin's name is its `package.json` `name`, never the directory name. A directory without a `package.json`, or + with an invalid name, is reported, not a plugin. +- Facet kinds are the app's (`["server", "worker", "web", "tui"]` in p3), each `.ts`, `.tsx`, `.js` or `.mjs`; two + files of one kind are reported (not chosen by extension). A plugin with none of the kinds (contracts only) is a + plugin with no facets: imported by others, never loaded. +- Discovery lists each root one level deep, reads only `package.json`, follows symlinks to their real paths, and keeps + the first root's plugin of a duplicate name (the other reported). + +## 2. Building + +```ts +const plugins = await discoverPlugins(["./plugins", join(dataDir, "plugins")], { kinds: KINDS }, ctx); +const built = await buildFacet(plugins[0], "worker", { hostPackages: HOST_NAMES, platform: "node" }, ctx); +// { kind: "script", plugin, facet: "worker", source, digest, inputs: [{ file, hash }], copied: [file], +// css?: { name, text }, assets: [{ name, bytes }] } | { kind: "failed", plugin, facet, errors } +``` + +One esbuild build per (plugin, kind), from cold (no kept contexts: about 10 ms per facet; p2 measured 237 ms for 25 +worker facets). A failed build is a result, never a throw. The result is plain data (strings, bytes, arrays): the same +value a worker (slice 10) or a page (slice 11) evaluates where it runs. + +| An import of | Becomes | +| --- | --- | +| the plugin's own files | inlined | +| a file of another discovered plugin (`../sessions/contract.ts`) | inlined (a copy), listed in `copied`; it may import only host packages and files of its own plugin, never packages (a build error otherwise): contracts and side-effect-free modules only (decision 4) | +| a package in the plugin's `dependencies` | inlined from its `node_modules` | +| a package in `chord.external` (Node builds) | external, `require`d at evaluation from the plugin's `node_modules`; a browser build fails | +| a host package (`options.hostPackages`, e.g. `@earendil-works/chord/next`, `react`, `react/jsx-runtime`, `@app/core`) | external, from the map at evaluation | +| `@earendil-works/chord/next/plugins` | external always (its `pluginFile` is per facet, §3) | +| a Node builtin (`node:fs`, `fs`) | external (Node builds); a browser build fails | +| CSS (browser builds) | one stylesheet per facet (`css`) | +| an asset (`import url from "./icon.svg?url"`, `.wasm`, browser builds) | a content-hashed file in `assets`; the import is its URL under the app's `options.publicPath` | +| `import.meta.url`, `__dirname`, `__filename`, a literal dynamic `import("x")`, top-level `await` | a build error pointing to `pluginFile` (§3); a computed `import(x)` cannot be seen at build and fails when it runs, through the facet's `require` naming the facet | +| anything else (a relative path into base code, a package not in `dependencies`) | a build error naming the file and the rule | + +- Output: `format: "cjs"`, wrapped `(function (require, module, exports) { … })`; TypeScript stripped, JSX automatic + (`react/jsx-runtime`); no type check; minified; an inline source map whose `sources` are relative to the plugin's + directory (so the script and its `digest`, a hash of the source, are the same on every machine and checkout). +- `inputs` are captured where esbuild loads each file (an `onLoad` that hashes the exact bytes it hands esbuild, as p2's + `captureBundleInputs`; the metafile has no hashes), absolute paths, for the watcher. + +## 3. Evaluating (Node) + +```ts +const facet = evaluateFacet(built, { hostPackages: HOST, plugin }); // returns the facet, or throws its error +``` + +- `vm.Script` under a stable filename per (plugin, kind, digest), path-like (`/chord-plugins/@pi/notes/worker.ts` + plus the digest), run in this context; the default export, checked to be a facet (a declaration error otherwise). +- **The `require` it passes answers, in order:** the host-package map (each value is what Node's `require` would give: + a CommonJS package's exports, an ESM package's namespace wrapped once with `__esModule: true`, so default and named + imports both work); `@earendil-works/chord/next/plugins` → `{ pluginFile }` bound to this plugin's directory; + `chord.external` packages and Node builtins through `createRequire(join(plugin.dir, "package.json"))`; anything else + throws naming the facet. +- **`pluginFile(path)`** gives the absolute path of a file inside the plugin's directory; `..` out of it is refused. +- **No module map, no cache:** an old facet's closures are collected once nothing holds them. The component turns V8's + compilation cache off when it is created (`v8.setFlagsFromString("--no-compilation-cache")`, once per process; an + app may pass the flag itself). +- **Stack traces** (decision Q1 (b)): `mapStack(error)` returns the error's stack with plugin scripts' positions mapped + through their source maps (a `module.SourceMap` per evaluated script, kept while the script is) to `notes/worker.ts:12`; + the app's reporters call it. Nothing global (no `Error.prepareStackTrace`, no process-wide source-map switch). + +## 4. Loading and reloading, per plugin (kernel change) + +Decision "Per plugin at startup and reload" changes `host.load` and `host.reload` (slice 7b) and the component's +`load`/`reload` helpers: + +- **`host.load(facets, ctx)`** resolves with `{ failed: [{ plugin, error }] }`: a plugin whose setup throws (or that is + refused at declaration: invalid name, a type or registry served twice, a remote type on a host without principal or + address, `checkAccepts`) is left out and reported; the others publish; its consumers wait `failed { waiting }`. `load` + never closes the host for a plugin (base code decides whether its core failing is fatal). A plugin left out is + loaded by the next reload after an edit rebuilt it (it is dirty). +- **`host.reload(facets, ctx)`**: staging per plugin. A plugin whose new code fails in staging (setup, a factory against + the recording `self`, a declaration error, `checkAccepts`) keeps its loaded facet (as if passed unchanged), and + staging runs again without its new code. **The closure rule:** a changed plugin whose recorded requests (staging + records them, for its existing objects, with their contract versions) would not bind (the same check as a real + bind, the provider's host address included) because the plugin they need was kept old, or because nobody serves the + name while any plugin of this reload failed (a failed new setup may be what would have served it: owner, 2026-10-30, + "keep"), is kept old too (repeat until stable). The report gains `failed: [{ plugin, error }]` (the same list as `load`'s; an added plugin that fails is left out); the commit applies the rest. +- **The component** builds and evaluates per plugin: a plugin whose build fails or whose evaluation throws is passed to + the host as its loaded facet at a reload, with `reload(facets, context, { failed })` naming it (the host applies the + closure rule to it and lists it in `failed`), left out at startup and reported with the host's report. + `plugins.load(host, ctx)` and `plugins.reload(host, names, ctx)` keep, per host, the facet loaded for every plugin, so + `reload` always passes the complete set (unchanged plugins as the same facet object, removed ones left out) and never + restarts an unchanged plugin. + +## 5. Watching and rollout + +```ts +const plugins = createPlugins({ roots: ["./plugins"], kinds: KINDS, hostPackages: HOST, base: ["./src"] }, ctx); +plugins.state // a JSON watchable: { dirty: ["@pi/notes"], install: ["@pi/notes"], set: false, base: false } +const next = await plugins.build(["@pi/notes"], ctx); // built results per (plugin, kind), plain data +await rollout([serverStep, ...workerSteps, ...clientSteps], next, ctx); // the app's order (decision 3) +``` + +- **The watcher** (`fs.watch` recursive over roots and `base`) ignores `node_modules` and `.git`. A changed file marks + dirty every plugin whose facets used it (`inputs`; a copied contract dirties every user) or whose directory contains + it; a changed `package.json` marks `install`; a plugin directory added or removed sets `set` (discovery runs again + at the next build); a file under `base` sets `base` (the app restarts its hosts; Chord restarts nothing). A change + during a build keeps its mark. If the watcher cannot watch (the system's limit), it reports once and the app builds + on demand. It marks, never applies. +- **`installDependencies(plugin, ctx)`**: `npm install --omit=dev --legacy-peer-deps --ignore-scripts --no-audit + --no-fund` in the plugin's real directory, the command overridable (decision 3); it clears `install`. +- **`rollout(steps, built, ctx)`**: each step is `{ name, kinds, reload(built, ctx) }` and returns the host's report; + steps run in order; a step's `reload` evaluates the facets of its kinds where its host runs (an in-process step through + the component's per-host state, §4; a remote one, slice 10/11, gets the built data). It stops at a step that rejects + (a host closed, an abort); kept plugins do not stop it. Marks clear for plugins applied on every step; a plugin kept + on any host stays dirty. + +## 6. Not in 7c + +Installing plugins from npm or git (p2 §8), project plugins with trust and snapshots (design §26 #14), a base bundle per +surface, evaluating in a browser (slice 11), shipping built scripts to workers and pages (slices 10, 11), static checks +of cross-plugin imports beyond §2's rule. + +## 7. Tests, examples, cost + +- One test file per source file; fixture plugins under `test/next-node/fixtures/plugins/`. Discovery (names, duplicates, + missing or invalid `package.json`, two files of a kind, contracts-only plugins, symlinks); build (every row of §2's + table, failures as results, relative source maps and stable digests across two checkout paths, inputs hashed at + load); evaluation (host packages as the host's instances: `instanceof ChordError` across a plugin; default and named + imports of ESM and CommonJS host packages; `pluginFile` and its `..` refusal; an import-time throw; a non-facet + export); `mapStack` naming plugin files and lines; per plugin at startup and reload (§4: a failing setup, factory, + declaration, build, evaluation; the closure rule with a provider and consumer changed together; the report's `failed` + and `failed`; `load` never closing the host); the watcher (dirty marks from inputs and directories, `install`, `set`, + `base`, a change during a build, `node_modules` ignored); `rollout` (order, stopping at a rejecting step, marks). +- **GC:** 200 reloads of an evaluated plugin facet: old closures collected, heap flat after warm-up. +- **End to end:** a server and a worker (memory links) loading plugin directories, a plugin edited on disk, built and + rolled out (server, then worker), a holder seeing one frame; a broken edit kept old on both hosts and reported. +- Examples (owner, 2026-10-30: full coverage, each a folder `NN-topic/` with the hosts in `main.ts` and real plugin + directories under `plugins/`): `35-plugins` (load, edit, build, rollout), `36-plugin-failures` (per plugin at startup + and reload, the closure rule, an abort after the commit), `37-plugin-imports` (copied contracts, dependencies, + `chord.external`, host packages, `pluginFile`, the refusals), `38-watch-and-rollout` (the marks, plugins added and + removed, a stopped rollout), `39-browser-build` (CSS, assets, the built result as plain data). Benchmark (`benchmarks.md`): cold build of a 20-plugin set, a rebuild of one facet, + evaluation per facet, heap after 100 reloads. diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md new file mode 100644 index 000000000..7a2eff98b --- /dev/null +++ b/packages/chord/docs/next/slice-8.md @@ -0,0 +1,194 @@ +# Slice 8: registries and `host.root` (spec) + +Status: **built** (2026-10-28; build review and two fix rounds by the same four reviewers, +`/tmp/s8-build-review-*.md`, `/tmp/s8-build-rereview-*.md`). Before: ready to build (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, +`/tmp/s8-review{,2}-{astra,fable,opus,sol}.md`, found no remaining blocker; S3 refined after the second: with no value +after `load`, a reader fails at once, as `self.get` of a type not served here) + +8 builds the extension point between plugins (registries), in-process values that change (`self.writable`), wake +signals for failed objects (`retryOn`), and the call scope for code outside objects (`host.root`). The harness object +is p3's base code; Chord shows it only in an example. + +## What an app can do after this slice + +```ts +// the defining plugin (e.g. http): the token, exported for other plugins +export const Http = defineRegistry("chord.http", { version: 2 }); // shared: identity only +export default defineFacet((host, self) => { + host.serve(Http, { // the one provider (Q8), as for object types + compose: (table, { report }) => buildRouter(table, report), // pure; always a value; conflicts reported + accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, + }); + host.serve(HttpServer, (_, self) => { + const router = self.registry(Http); // a NativeWatchable of the composed value + self.on(router, ({ value }, ctx) => …); + return { … }; + }); +}); + +// any plugin: a facet or a host-lifetime object contributes one value per registry +export default defineFacet((host, self) => { + self.contribute(Http, { routes: { "GET /notes/export": exportNotes } }); +}); +host.serve(McpStore, (_, self) => { // local, empty params (R21) + const ext = self.writable(mcpExtension([])); // an in-process value that changes + self.contribute(HarnessExtensions, ext); + self.on(mcp.tools, ({ value: tools }) => ext.set(mcpExtension(tools))); + self.retryOn(self.registry(HarnessModels)); // a model contribution wakes a failed attempt + return {}; +}); + +// code outside objects (a Durable tool, a CLI): a call scope +await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx), context); +``` + +## 1. Registries + +- **Token and provider (D1, owner, 2026-10-27):** the token `defineRegistry(name, { version })` is shared, as a + contract: its name (as types, `[a-z0-9@/._~-]`), the contribution type `C`, the composed value type `V`, and the + version the bundling plugin was built against. The defining plugin's facet provides it with `host.serve(R, { + compose, accepts? })`, as it serves object types (adapters with the provider, never in the shared token, as + `checkAccepts` for contracts); a second serve on a host is refused; a reload replaces it (slice 7). + +- **Contributors** (R21): facets (by plugin name) and host-lifetime objects (`local`, empty params: by plugin name and + type), at most one value per registry each; `self.contribute(R, value | Watchable | NativeWatchable)` in a facet setup + or such an object's factory (a type error elsewhere; a declaration error at runtime). A facet's contribution is in the + table while its plugin is loaded; an object's from `live` until it ends, fails or restarts. +- **The table:** an immutable list of `{ contributor, plugin, value }`, sorted by plugin name, then type (never load + order); a contribution carries its token's version: an older one is adapted once with the declaration's `accepts` + when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` + `contribution_refused { registry, plugin, version }`). +- **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member; another + registry's value too, a cycle of registries reading each other refused when the second one serves or contributes (superseded by audit A2: caught at the first composition by the runtime guard, reported `contribution_failed`)), followed as a dependent of the table (its change recomposes), recognised by Chord's own mark; anything + else is a plain value, whatever methods it has (a harness, a conversation handle or a runtime is contributed as a plain + value, not followed; build review 2026-10-27, replacing "a foreign watchable is refused"). The table's entry is `{ plugin, type?, version, value }` (`type` for + an object contributor; served types carry their plugin's name). +- **`compose(table, { report })`:** pure, lazy as a derive: the registry's value is marked when its table changes (a + contributor goes live or ends, a watchable contribution changes) and composed when read or, while watched, in the + owner's flush (W4: never by another owner's adoption itself; the flush after a commit step is where it runs). The first + composition happens at the serve: the facets' setups of one `load` serve and contribute, then every served registry + composes once in one batch at the end of the setups (an empty table composes too), so a served registry always has + a value before any object activates. `report(conflict)` goes to `onError` as `registry_conflict { registry, message, + plugins }`, once per conflict episode. +- **Failures (owner, 2026-10-27):** contributions never fail the registry. A contribution that is a watchable and fails + is left out of the table and reported (`contribution_failed { registry, plugin, error }`), the table recomposes without + it, and it is back once it heals. A `compose` that throws (the owner's bug) is reported (`registry_failed { registry, + error }`), the composed value's watches get a `failed` call, and `.value` keeps the last good composition (an extension + point stays available; derives keep their own rule, `.value` throwing while failed); the next good composition + replaces it. Before any good composition `.value` throws the error. +- **Without a provider** (the defining plugin is not loaded): the registry has no value; contributions wait in the + table; its serve composes it. +- **Reading is a dependency (S3, owner, 2026-10-27: "isn't self.registry meant to wait like self.get?"):** + `self.registry(R)` in a factory declares the registry as a dependency of the attempt, as `self.get`: activation waits + until it has a value; once `load`'s setups ran, a registry with no value cannot get one before a reload (a provider + comes only with a plugin), so a reader then fails at once `transient` with `unavailable/undeclared` as the + dependency's cause (as `self.get` of a type not served here fails at once, and the start barrier waits for no + contribution) and is retried the moment the registry gets its first value (R18); a first `compose` that throws fails + its readers the same way. It returns a `RegistryValue` (a native source: a derive source, an `on` source, a `retryOn` source; never a + state member); methods and routines read `.value` at use. Once the object is live, a later failure of the registry + (a throwing `compose`) changes nothing for it: `.value` keeps the last good composition. Code outside objects: + `await host.registry(R, ctx)` waits the same way. +- **Older readers (S2, owner, 2026-10-27):** a reader whose bundled token has an older version than the provider's gets + the composed value mapped back with `accepts[old].value` (computed once per composition and version, while such a + reader exists); without that mapping, or for a reader newer than the provider, its dependency fails + `contract_mismatch` (reported). `accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } }`. + Adapters are the provider's code, guarded as `compose`: a `contribution` adapter that throws leaves that contribution + out (reported `contribution_failed`); a `value` adapter that throws fails only that version's value, by the rules of a + throwing `compose` (its readers keep the last good mapped value; before any, a reader of that version waiting at + activation fails `transient`, retried when a mapped value arrives); current readers are never affected. +- **Helpers:** `keyed(table, entriesOf, report)` (a map, a duplicate key reported through `report`, the first by plugin + name wins) and + `ordered(table, itemsOf, orderOf?)`. +- **Use sites guard contributions** (Q8): Chord's helpers keep each entry's plugin name; the guide shows catching a + contribution's throws and rejections, reporting the plugin, and continuing. +- **Registries carry natives** (design §17): a contribution may be any in-process value; an entry lives exactly as long + as its contributor. + +## 2. `self.writable(initial)` and native sources + +A `NativeWatchable` with `set(value)` (identity compared: `set` of the same value notifies nobody; a new value is a +new frame), owned by the attempt (dropped with it; `set` after the attempt's seal throws `released`); never a state +member; usable as a contribution, a derive source and an `on` source. `set` inside a batch marks it there; outside, it +opens its own batch. `writable`, `RegistryValue` and `ref.state` (a `NativeWatchable` since slice 3b, until now not derivable: design audit, +2026-10-27) are native sources: one base (`NativeSource`) the batch marks and derives depend on; `self.derive` takes JSON and native sources (its type and the source resolver widen), while the +member check still refuses a native as a state member. + +## 3. `self.retryOn(...sources)` + +In any object's factory, anywhere: Chord follows each source with watches owned by the object (under its context, +replaced by each attempt's declaration, stopped at the object's end; GC-tested), installed right after the factory +returns, before natives open, so a failed open still has them; a `value` call of a source while the object is failed +(`transient` or `broken`) retries it as `ref.retry()` does (backoff skipped, an attempt now, holders told `loading`); +a call while it activates is remembered and retries it at once if that attempt fails (decision S1); ignored while live. Sources +are those that live independently of the attempt: a `RegistryValue`, `constant`; anything else (a reference's member, +a `writable` of the same attempt) is refused at runtime (declaration policy; TypeScript cannot tell them apart). + +**Which calls retry (D2, owner, 2026-10-27):** only `value` calls: a new value of a source wakes a failed object; a +`failed` call (a throwing `compose`) does nothing, its next good composition retries. + +## 4. `host.root(fn, context, options?)` + +**Signature and guarantees (D3, owner, 2026-10-27; design §26 #25):** `host.root(fn: (ctx: Ctx) => Promise, +context: Context, options?: { principal?: P }): Promise`. `fn` gets a call context like a handler's: the host's +principal (or `options.principal`, a trusted same-host caller's choice, as base code acting for an authenticated +user), this host's incarnation, `context`'s telemetry and abort signal linked until `fn` settles +(`withLinkedAbortSignal`), and `ctx.get`. When `fn` settles or `context` aborts, the call context is sealed: +`ctx.abortSignal` aborted, `ctx.get` rejects `released`, every reference it took released (their calls reject +`released`, their watches end), so an uncooperative `fn` fails at its next use of Chord, as a handler after its caller +cancelled; work outside Chord runs on until then. `ctx.get` before `load` resolved parks like any reference; +`host.close` seals every open root scope (the host keeps them), so an `fn` waiting in `sleep(ms, ctx)` ends; `host.root` +after `host.close` rejects `released`. No span of its own until slice 9. + +- `chord(fn, context)` is p3's helper over `host.root` (design §24); Chord ships none. + +## 5. Durable on registries (examples, p3's pattern) + +- Durable's `HarnessOptions.registry` takes a `RegistryReader` (`snapshot()`, `subscribe(listener)`, the listener + called synchronously after every publication). A `RegistryValue` offers a synchronous observer of its compositions + (`observe(observer, ctx)`, as Durable's `observeXXX`: `advance` inside the flush that composed, record only; an + attached observer counts as watching, so the lazy compose runs for it); Durable's + `registryReader(value)` (in its `src`) reads `.value` in `snapshot()` and forwards `observe` to `subscribe`. The + harness object declares the extensions registry with `self.registry(...)`, so it activates (and opens Durable) only + once the registry has a value. p3's `compose` builds Durable's snapshot without throwing (a duplicate task name is + reported and the later extension left out). +- **Durable's views from other objects** (carry-over from 6b): the object that opens Durable contributes its harness to + a registry (`HarnessRegistry`, p3's); another object declares it in its factory and follows a view through its value: + + ```ts + const harness = self.registry(HarnessRegistry); // a dependency: activation waits for it + self.retryOn(harness); // the new harness's contribution wakes the failed follower + const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); + ``` + + When that harness closes, Durable ends the observer `failed` (S1, a Durable change in this slice: a Session close + ends observers `failed`, `ended` only when the observed owner ended), so the follower fails `transient` and its retry + follows the new harness. +- Examples: Chord's `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a + `writable` contribution, a missing provider healing) and `30-host-root.ts`; Durable's `test/examples/37-harness- + registry.ts` (a Durable harness on the store, its extensions from a registry, its views followed by a second object) + and the `registryReader` tests in Durable's tree. Durable's `subscribe` listeners then run inside Chord's flush: + they only wake Durable's scheduler (no Chord call; a reentrant one would throw `invalid/reentrant`), which the + `registryReader` test pins. + +## 6. Carry-over + +From `slices.md` "Slice 8": `host.root` (D3); `retryOn` (section 3); registries recomposing in the owner's flush (W4); +`host.activity` on the facet's host: not built (nothing needs it; dropped from the carry-over); Durable's views from +other objects (section 5). Left to slice 7: registries on reload (a reload replaces a contribution and a provider +atomically; staging composes nothing, decision D1 of "Review round 1"); `accepts` of contracts and jobs; a `writable` +across a pure swap; older registry readers across a reload of the provider. + +**New names** (design §3, §5, the pinned reason list): the reason `unavailable/undeclared`; `HostError` kinds +`registry_conflict`, `registry_failed`, `contribution_failed`, `contribution_refused`, each reported once per episode. + +## 7. Tests and cost + +- One `describe` per mechanism: tokens and providers (two serves refused, a missing one healing), contributors + (R21's type and runtime refusals, facet and object lifetimes, one value each), the table (order by plugin, versions + adapted, refused versions reported), `compose` (in the flush, after a commit that changed a watchable contribution, + conflicts reported, a throw as a failure call and its heal), reading (`self.registry` in derives and `on`, `.value` at + use), `writable` (identity, batching, as a contribution), `retryOn` (a failed object retried by a contribution, value + calls only, sources installed before natives), `host.root` (principal, release at settle, abort, before load, after + close), `registryReader` with a Durable harness; GC for every table, writable and root scope. +- Cost guards: one compose per flush per changed table; no compose for an unchanged table; a `writable.set` of the same + value costs one comparison. diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md new file mode 100644 index 000000000..b965386f0 --- /dev/null +++ b/packages/chord/docs/next/slices.md @@ -0,0 +1,225 @@ +# Chord v2: the slice plan + +Status: approved (owner, 2026-10-22); kept current. `design.md` states the rules; this file says which slice builds +what, and (section "Carry-over") what a slice deferred to a later one. + +## Why slices + +Building bottom-up made each layer guess the needs of callers that did not exist yet; L2 grew generic machinery +(nodes, sinks, deliveries, publications) that nobody could follow, and was deleted. A **slice** is the thinnest +end-to-end path through every layer a feature needs, so every API has a real caller from the first line. Later slices +extend earlier code; reworking it when a real need appears is expected, guessing ahead is not. + +Per slice (owner, 2026-10-22): a one-page spec written by this session (`slice-N.md`, including the carry-over +entries it takes), the owner's approval, a build by a fresh Opus 5.5 subagent, reviews of the uncommitted code by this +session and by the reviewers Astra and Fable (`v2-review-astra`, `v2-review-fable`), fixes by the builder, the owner's +review of the uncommitted diff, commit. Deferred work goes into the section "Carry-over" below, under the slice +that must do it, in the same change (owner, 2026-10-22; moved here from `carry-over.md` on 2026-10-24). Every slice that touches a hot path (links, the store, relays) also adds a light benchmark script +(`packages/chord/test/next/bench/`, run by hand) for timing, memory and wire size of realistic payloads, with results +in `benchmarks.md` (owner, 2026-10-22). Every slice adds one or more runnable +examples (`packages/chord/test/next/examples/NN-topic.ts`, numbered across slices, in the format of Durable's +`test/examples/`), each on one topic: for the owner to run and dig around in, and as references for Chord's users +(owner, 2026-10-22). Implementers and reviewers read +the slice spec, the feature map below and the whole of `design.md` in full, and check the work against all three; a +slice spec's section citations are for orientation, not a reading limit (owner, 2026-10-22). + +## What exists + +- **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, + errors. `index.ts` for apps, `internal.ts` for Chord's own code. +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b, 8, 7a** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28, 8 2026-10-28, 7a + 2026-10-29): `src/next/documents`, `state`, `host`, `jobs`, `link`, `remote`, `store`, `registries`; examples 01-32 + (Durable's 35-37); benchmarks in `benchmarks.md`. +- **Slice 7b** (built 2026-10-29, `slice-7b.md`): `host.reload`, staging (`host/staging.ts`), fates, adapted members + (`host/adapted.ts`), references, jobs and registries across a reload; examples 33-34; `test/next/bench/reload.ts`. +- **Slice 7c** (committed 2026-10-30, `slice-7c.md`): `src/next-node/plugins/` (discovery, builds, evaluation, + `pluginFile`, `mapStack`, `installDependencies`, the watcher, `rollout`), per-plugin `load` and `reload`; examples 35-39. +- **Slice 10** (built 2026-10-31, `slice-10.md`): `src/next-node/http/` (a `fetch` core with the security baseline, + Chord's Node listener, `nodeRequest`, plugin sockets, links), `src/next-node/workers/`, `createPluginRuntime`, kinds, + the kernel's `RouteHandle` and anonymous `host.root`, origins only from relays; examples 40-48. +- **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, + `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), + context, JSON, output, SQLite drivers. + +## The slices + +Order (owner, 2026-10-22: routines and jobs before the store and reload; 2026-10-27: slice 8 before slice 7, whose +reload has to reload real registries; 2026-10-29: slice 7 split into 7a, 7b, 7c; the app kit as slices 10 and 11; +telemetry last, since it instruments the app kit too; 2026-11-01: the plugin model (12) and the app kit (13) before web and TUI (11), which builds into the kit). + +| # | Slice | In short | +| --- | --- | --- | +| 1 | **One host, local objects** | host, facets, local objects, references, in-memory documents, transactions, derives, watches | +| 2 | **Failures and natives** | `self.open`, activation failures and recovery, backoff, the drain, timeouts | +| 3a | **Remote objects over one link** | remote contracts, principals, variants, a memory link, the message format, holds, calls, one message per commit (no relay, no drops) | +| 3b | **Surviving drops** | re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger` | +| 4a | **Links and relays** | host kinds, host addresses (`placement` then), routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | +| 4b | **Authority and housekeeping** | revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression | +| 5 | **Routines and jobs** | `self.routines`, `self.job` | +| 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | +| 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | +| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, Durable's `registryReader`, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | +| 7a | **The dependency model** | requests that outlive providers, parked attempts without timers, wakes, the publication gate, getters, registries as dependencies, `parked_long` (`slice-7a.md`, `dependencies/model.md`) | +| 7b | **Reload in the kernel** | `host.reload`: staging, commit, rollback; swaps, restarts, removal and re-serving; holds and `accepts` across a reload; jobs and registries across a reload; per-name wakes (`slice-7b/` brief) | +| 7c | **Plugins** | `chord/next/plugins` (discovery, `buildFacet`, `evaluateFacet`, `pluginFile`, `installDependencies`, the watcher, `rollout`, `mapStack`; no dev/prod split; the kernel's `load`/`reload` per plugin; `slice-7c.md`) | +| 10 | **HTTP and workers** | `chord/next/http` (server, `Http` registry, responses as descriptions, WebSocket entry) and `chord/next/workers` (a worker per address, IPC, idle stop) | +| 12 | **The plugin model** | imports (relative within a repository, package names pinned across), versioned app API with Chord re-exported, per-kind API modules and contract files with the build's import check, single-file plugins, git installs, `testPlugin`, a required root (`slice-12.md`) | +| 13 | **The app kit** | `chord/next/app`: `startApp`, `startWorker`, layout, worker kinds, owner link, deny by default, sessions, CLI tokens, storage, the files plugin, export per address, plugin roots and "apply changes", base code as API, bootstrap and a start-data facet (`slice-13.md`) | +| 11 | **Web and TUI** | `chord/next/web` (page host, React hooks) and `chord/next/tui` (launcher, sandbox), from p2's code | +| 9 | **Telemetry** | spans, events, metrics across hosts | + +The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and +carry-over entries are decided in the slice that first needs them. + +## Feature map + +Every feature of `design.md`, by section, assigned to the slice that builds it. A slice's spec covers every entry +assigned to it; **implementers and reviewers check the slice spec, this map and the whole design** (owner, 2026-10-22). A feature missing here is a planning bug: add it before the slice that needs it. "Doc" means a documented +pattern, no code. Decided cuts are noted where they apply. + +| § | Feature | Slice | +| --- | --- | --- | +| 3 | `createHost` with `telemetry`, `onError`; host without address | 1 | +| 3 | `createHost` with `principal`, `principalKey` (3a), `address` (4a), `revokedBy` (4b), the store (6a) | 3a, 4a, 4b, 6a | +| 3 | `load([{ name, facet }])` with static facets; start barrier (host-lifetime objects live before `load` resolves) | 1 (failed plugins reported: 2) | +| 3 | `connect`, `accept`, memory link pair, incarnations in the handshake, protocol-version handshake | 3a | +| 3 | host kinds, addresses, `route` (`authorize`, `resolve`), relayed holds and calls, re-resolve after a drop | 4a | +| 3 | IPC and WebSocket links, trusted links, the ack-clocked window per origin (K unacked state messages, pending queues, call window N per origin), relays with flow control per origin (`origin`, `closed`, senders FIFO), heartbeat | 4a | +| 3 | timeouts: `closeTimeout`, call drain, `readyTimeout` (C13: two host timeouts) | 2 | +| 3 | provider-side `linger` (C7: kept) | 3b | +| 3 | park deadline (= `readyTimeout`, C13) | 3b | +| 3 | `host.activity` (keep-alive counts) | 4b | +| 3 | `host.root(fn, …)` for code outside objects | 8 | +| 3 | requests that outlive providers: the destination by address, the publication gate, the unbound index, the check against the selected provider only; the start barrier's first outcomes (decision (a)) | 7a | +| 4 | principal type `P`, links stamp principals, principal keys in identity | 3a | +| 4 | revocation (`revokedBy`, `host.revoke`, propagation to children) | 4b | +| 5 | `local` types: params, members, checked arguments (Q15), results unchecked | 1 | +| 5 | version and fingerprint match at acquisition; reserved member names (`then`, `release`, `state`, `retry`) | 1 | +| 5 | `remote` types, variants chosen by the factory (`{ variant, members }`), identity with principal key, calls checked against the binding's variant | 3a | +| 5 | `accepts` mappings at runtime (old consumers), keyed members per key, `contract_mismatch` | 7b | +| 6 | lifecycle, one object per identity, lifetime from kind and params (Q-A), factory, activation, ending | 1 | +| 6 | failure causes, backoff per identity, dependency recovery (R18) | 2 | +| 6 | `retryOn` | 8 (first independently living source: registries) | +| 6 | parked attempts (no dependency deadline or backoff, wakes on every change, retry passing on), `FailureCause` `waiting`, `parked_long` and the authoring rules against cycles | 7a | +| 6 | cycle rejection on one host (Q5) | 1 | +| 6 | cycle rejection across hosts (activation chain in hold headers) | 4b | +| 6 | shared store with per-principal views; worker-wide native owners; natives following a relocation | doc (1, 3, 2) | +| 7 | `self.get`, `host.get`, hydrated references, members as watchables, `release` | 1 | +| 7 | per-reference watches (released with their reference) | 1 | +| 7 | `ctx.get` (call-scoped references) | 3a | +| 7 | `ref.state` (tagged), `ref.retry()`, `host.hold`, calls parking, failed references | 3b | +| 7 | getters wait through `waiting`; `host.hold` throws only for invalid params | 7a | +| 7 | `ref.variant` as discriminant, `RefIn` (C15: no `inVariant`/`expectVariant`) | 3a | +| 7 | `state: false` (C8: cut) | - | +| 7 | optional dependencies as registries | doc (8) | +| 8 | `ctx` with telemetry and signal | 1 | +| 8 | `ctx.principal`, `ctx.incarnation` (R20), `ctx.get` (3a); relayed incarnations (Q13, 4a) | 3a, 4a | +| 8 | `ctx.trace` (C11) | 9 | +| 8 | a job's `ctx.jobId`, `ctx.state` | 5 | +| 9 | `defineState`, singleton and keyed handles, `self.state`, in-memory documents (P2) | 1 | +| 9 | `self.tx`, `tx.draft`, `tx.retire`, drafts revoked (R12), no existence check (R13), lines per object | 1 | +| 9 | `handle.change`/`handle.retire` shorthands (C14: kept) | 1 | +| 9 | `initial()` checked once per definition (R8, D3) | 1 | +| 9 | `persist`, `checkpointWhen` (required for persisted documents), persisted only in local empty-params objects (P3), eviction | 6a | +| 9 | keyed watches (`handle.watch(key, ctx)`), `handle.at(key)`, retirement as a flag (decision "One retirement mechanism") | 6a | +| 9 | versions, `migrate`, schema fingerprints on records | 6a (in-memory migrate on reload: 7b) | +| 9 | ~~facet documents, a served object's transaction spanning its facet's documents~~ (removed: C16 accepted, owner, 2026-10-26) | 1, removed in 6a | +| 9 | rules across owners, reconciling routines | doc (5) | +| 10 | the store: backends, tables, query runner, writes, commit queue, outbox, migrations, export/import | 6a | +| 11 | `Watchable`, watches (Chord's `Watch`), the batch (W1), reentrancy | 1 | +| 11 | `NativeWatchable`, `self.writable` | 8 | +| 11 | Durable's views as sources (`self.follow` over Durable's `observeXXX`) | 6b | +| 12 | `self.derive`, failures per member, lazy when unwatched, `mapShared` | 1 | +| 12 | container-width benchmark (R24) | 9 | +| 13 | `self.open`, close reasons, order, restart patterns (R16) | 2 (restarts on reload: 7b) | +| 14 | `self.on` (W2b), `self.onLive`, `sleep`, `untilAborted` (C4 decided); the handover per object on reload | 5 (handover: 7b) | +| 15 | `defineJob`, `self.job`, intent/run/finish, state documents, dedupe (Q9), `busy`, `running`, blocked `finish` (C3 decided); persistence, resume, restart limit; `accepts` in use | 5 (persistence: 6b; `accepts` and reload: 7b) | +| 16 | `defineFacet

`, facet as declarations only (documents and routines removed in 6a) | 1, 6a | +| 17 | registries: `defineRegistry`, contributions, `compose`, versions (Q8), use-site guards | 8 | +| 17 | registry readers as parked dependencies (`waiting`), `host.registry` waiting, the last good value dropped when a contribution leaves | 7a | +| 18 | `constant(value)`, start data as a core object (`SessionInfo`) | 1 (core facet pattern: 4a) | +| 18 | platform bindings as core objects | doc (8) | +| 19 | `host.references` (presence) | 4b | +| 20 | flows kept by their owner, the `steps` helper, the Questions service | doc (5); deferred decisions #9, #11 | +| 21 | `host.reload`: staging, atomic commit, pure swaps, native restarts, removal and re-serving | 7b | +| 21 | loading plugins (loader D), dev loop; the rollout across hosts is the app's (decision "Slice 7b's open points" (3)) | 7c | +| 22 | message format, header schema, unknown ids, one message per commit per link, consumers apply ops, plugin errors | 3a | +| 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7b | +| 22 | trace carriers (R5) | 9 | +| 23 | HTTP (routes, wraps, responses as descriptions, request bodies, plugin sockets), workers; channels and chunked transfers stay owner patterns until a second user | 10 | +| 24 | Durable integration: `chord(fn, context)` (Q6), a `RegistryReader` over composed registries; the harness object and its model and env registries are p3's (base code, owner, 2026-10-27) | 8 (Chord parts), p3 | +| 24 | `StoreStorage` | 6b | +| 27 | versions table: contracts (L1 done), registries 8, documents 6, jobs 5, link protocol 3a | as listed | +| 29 | telemetry spans, events, metrics, carriers | 9 (cost counters: every slice) | +| 30 | cost guards per hop | every slice | + +## Size + +About 9-10k source lines in total, as estimated for the layers (owner: fine, on the order of Durable). + +## Carry-over + +Obligations a slice deliberately left to a later one, under the slice that must do them, with where they came from. +Rules (owner, 2026-10-22): whoever defers something (this session, a subagent, a reviewer) adds it here in the same +change; a slice's spec includes every entry of its section, and its build and review check them off; a done entry is +deleted (the slice's spec and code are the record). Moved here from `carry-over.md` (owner, 2026-10-24). + +### Any slice (protocol and checks) + +- **Bump `PROTOCOL_VERSION`** for any message or reason change once p3 has shipped (slice 3b added `retry`, `loading` + and `failed`'s `retryIn`; slice 4a parts, the list `bound`, `hello`'s `calls`, `ack`, `ping`, `pong`, `closed`, + `origin` and the hold's `address`, `principal` and `incarnation`; all without a bump: nothing is deployed). (Owner, + slice 3b.) +- **The park deadline's per-type override** is not built: a parked call waits the reference's `readyTimeout`. Build + it only if a real case needs it. (Slice 3b spec.) +- No CBOR; revisit only if benchmarks show encoding as a bottleneck (slice 3a's and 4a's do not). (L2 decisions.) +- **One N on every host:** a relay forwards calls at once, so a worker announcing a smaller N than the relay's could + see more running calls of one origin than it accepts (a protocol error). Every host announces 256; per-link N, if + ever needed, must bound the relay's forwarding too. (Slice 4a rework.) +- **The Durable conversation view's size on re-hold** is a known cost until pi #11327 (windowed view) lands. (Slice 4.) + +### Later: reload (when a case needs it) + +- **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a + real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) + +### Slice 13: the app kit + +- **Deny by default in the HTTP component** (decision "The app kit, point 3: authentication"): routes and sockets + refuse requests without a principal unless marked public; the owner link with a one-time token; a session helper. +- **Base code reduced to the API, bootstrap and a start-data facet** (decision "The plugin model, points 3, 4, 5 and + 8, imports and builds", point 8): the harness and the providers of core registries as plugins of the app's required + root, the kit's start-data objects serving process values. (Slice 12 built the required root, the API checks and + the agents' document `plugin-authoring.md`; the reduction itself is the kit's. Owner's main agent, 2026-11-01.) + +### After slice 11 + +- **Small examples** (owner, 2026-10-30: the 7c examples are complete but complex): once HTTP, workers, web and TUI + exist, a pass of small, easy-to-read examples, each showing one thing a plugin author or app does. + +### Slice 9: telemetry + +- Spans for routine calls and job runs (from slice 5). +- The use of a persisted run's stored start trace carrier (`trace`, stored since slice 6b). + +- Round-3 cut to decide: C11 (`ctx.trace`). + +### When v2 goes upstream + +- `mapShared` lives in `src/next/state/derive.ts` and moves into Delta. (L2 decisions.) +- `withLinkedAbortSignal` lives in `src/next/host/ctx.ts` and moves next to `withAbortSignal` in `src/context` (pi + first): a call's context linked to long-lived abort signals until the call settles, instead of `AbortSignal.any`, + which keeps cleanup entries per call while its sources live before Node 24.16.0 (nodejs/node#62367). (Slice 4a.) +- The `./next` package export returns when the new code has an entry point (removed with the frozen-kernel move). +- A `./next/node` export for `nodeCompression` (`src/next-node/`), with `./next`. (Slice 4b.) + +### The sync layer (not a slice; when Radius sync is built) + +- Upload, acknowledgement and `store.outbox.trim`, epochs, pull-back, the split (S5, S7); a store opened with an + outbox (decision "The outbox is optional, internal and versioned") starts its sync from an archive. +- Snapshot reads for large exports on a second SQLite connection (S4), if exports get large. + +### Outside the slices + +- The playground's runtime tests fail on a stale `node_modules` copy of chord 1.3.0; relink with + `npm install --ignore-scripts` when the playground is used again. (Frozen-kernel move.) diff --git a/packages/chord/docs/next/store/book.md b/packages/chord/docs/next/store/book.md new file mode 100644 index 000000000..4f8a53f69 --- /dev/null +++ b/packages/chord/docs/next/store/book.md @@ -0,0 +1,665 @@ +--- +title: One Store: how a p3 session keeps, exports and syncs everything it owns +cover: One / Store +kicker: A design book · Chord v2 and p3 · 2026-10-21 +subtitle: Why Durable's transcript, Chord's documents and every plugin's own tables live in one transactional store, and how that store moves a session to another machine without coming apart +stats: + - 17 | review items decided (S1–S17) + - 1 | commit queue per store, one seq for every owner + - ~150 B | outbox write for renaming 10,000 rows (instead of ~40 MB) + - 0 | lines of plugin code a replica needs +note: | + Written for the owner from the discussion of 2026-10-21 and the store review by Fable and Astra. Every decision is in + the decision log (`packages/chord/docs/next/decisions.md`, entries "The store", "S1" to "S17", "Context last"); the + review is summarized in `apps/p2/docs/p3-reviews/store-review-summary.md`, bulk operations and migrations in + `store-bulk-migrations.md`, the earlier discussion with measurements in `session-data-scenarios.md`. Numbers were + measured on slayer with Node 24.4.1 and `node:sqlite` unless marked as estimates. Nothing here is built yet. Notes + marked *review* are points the reviewers raised that the design answers. +colophon: One Store · first edition · 2026-10-21 · based on the redesign decision log and the store review +--- + +# Before we begin {.preface} + +Mario works in session S1 on his laptop. The agent edits files, and a plugin called **file-history** remembers every +edit: which file, which conversation entry made it, the content before and after. It keeps those rows in its own table, +because it needs queries: every version of `src/app.ts`, every file the last turn touched, every edit under `a/`. When +Mario renames the directory `a/` to `b/`, ten thousand rows change. + +In the evening he wants to continue S1 on his desktop. Another day he exports S1 to a file and imports it into a +colleague's p3. Each time the other machine must end up with the conversation (Durable), the plugins' documents (Chord) +and the file-history table, and all three must agree: a row that says "entry 812 made this edit" must find entry 812, +the same entry 812. + +Copying files does not do that, and neither does syncing each storage on its own. This book explains the problem, the +solutions that were tried and dropped, and the design that came out: one transactional store per session, under +Durable, Chord and every plugin. + +## Eight things to know + +:::findings +1. **A session is self-contained.** Everything it owns lives in its store and its directory; it can be exported from + the session alone, imported elsewhere, synced and continued on another machine (P1). +2. **One store holds everything.** Durable's records (through an adapter), Chord's persisted documents and plugin + tables commit into one store, through one commit queue, numbered by one seq. +3. **Order is guaranteed where it is observable.** Disk, outbox and export follow the seq; in-memory adoption runs + inside the commit step (S1), so no reader ever sees seq 41 without seq 40. +4. **The outbox records what changed, not code.** Rows, column updates, bulk operations and table changes, written in + the same transaction as the data. A replica applies them without any plugin code. +5. **Tables are real tables with a small query language.** Keys and indexed columns are typed; text is UTF-8 and + compares by code point on every backend. +6. **Bulk writes stay small.** `updateWhere` records one operation instead of one row per change, and every replica runs + the same operation. +7. **Migrations run only where the code is.** Changes inside `json(...)` columns migrate on access; key and index changes run eagerly on the + driver and ship as ordinary writes. +8. **Losing control never rolls back.** A device that learns another one took over turns its copy into a separate + session. +::: + +## Words used throughout + +A **session** is one agent workspace: its conversations, tasks and plugin data. An **owner** is the code that writes a +piece of data: Durable's Session, a Chord object, a plugin's object. The **store** is the transactional store of one +session; a **backend** is what it runs on (SQLite, including in the browser as WebAssembly; tests use SQLite `:memory:`). A **table** is a set of +rows owned by one object. A **commit** is one atomic write; the **commit step** is the short, serialized part of a commit +in which the store checks, writes and numbers it; the **seq** is that number. The **outbox** holds a record of every +commit, written in the same backend transaction. The **driver** is the machine that writes a session; a **replica** is +a copy on another machine that applies the driver's records. **Adoption** is an owner updating its in-memory state to a +committed write. + +# A Session Is More Than Its Transcript {#problem lede="What has to travel with a session, and why separate storages cannot keep it consistent."} + +A p3 session holds three kinds of data written by three kinds of owners. Durable writes the transcript: conversations, +entries, tasks and its own documents. Chord objects write persisted documents: settings, drafts, plugin state. And a +plugin may need data shaped for queries, like file-history's edits. + +The owner's rule for p3 (**P1**) is that a session is self-contained. Everything of it lives in its storage plus its +directory; the server only mirrors it (the session catalog) and keeps instance data (shares, devices, settings). That +makes three things possible: export a session from its files alone, import it into another p3, and sync it to Radius +so another device can continue it. + +Durable alone already supports that shape: one storage, every change in an atomic commit, numbered by a seq. Armin's +sync design builds on it: a decorator captures each committed batch into an outbox, batches go to Radius, a replica +replays them. It breaks as soon as a session has a second storage. + +:::steps | One edit, two storages, synced separately ++++ On the laptop +The edit tool records the edit in file-history's own `history.sqlite` (row: path `x.ts`, entry 812). Durable commits +the tool's result. Both exist and agree. ++++ The history arrives first +The desktop has received `history.sqlite` up to the edit, but Durable's log only up to the commit before. The row says +entry 812 made the edit; the desktop has no entry 812. ++++ Mario takes over now +The laptop is closed; the desktop becomes the driver and writes its next entry, which gets id 812: a different +message. The row now points to the wrong entry, for good. +::: + +Each storage is consistent on its own. What is missing is one order across them: which change of `history.sqlite` +happened before which commit of the transcript. Every solution below is judged by whether it provides that order, +lets a replica work without the plugin's code, and keeps plugins able to model what they need. + +# What We Tried and Dropped {#dropped lede="The solutions discussed on the way, and the reason each one failed."} + +The first answers tried to avoid giving plugins storage at all. The later ones gave them storage and tried to sync it. + +| Idea | Why it was dropped | +| --- | --- | +| Plugins' own databases hold only derived data (indexes, caches), rebuilt after an import | too limiting: some plugins need their own write model and queries (owner) | +| Export plugin directories as opaque files | needs a stopped session; a changing database cannot sync in small pieces | +| Plugin export and import hooks | every plugin would build its own incremental log for sync | +| Plugin record logs in the session log, the plugin's database rebuilt from them by an `apply` function | the plugin must keep `apply` deterministic and every record version forever | +| Several storages, each with an outbox inside its own transaction, ordered by one shared position counter | works (prototype), but many outboxes and a "safe cut" rule; SQLite's row-change records are per row (a 10k-row rename is 259–418 KB); schema changes are not captured | +| A SQLite store that records the plugin's SQL statements, checked by a hash of their effect | SQLite only; arbitrary SQL can give different results on replicas (`random()`, unordered `INSERT … SELECT`, SQLite versions) | +| Raw SQL reads with controlled writes | ties every backend to SQLite and needs a guard against SQL that writes | +| An IndexedDB-shaped API only (keys, indexes, ranges) | too weak for real queries (owner) | +| Chord's documents inside Durable's existing storage | covers Chord, not plugin tables | + +The store design kept something from almost each: one order for everything (from the shared counter), recorded +operations instead of rows where possible (from the statement idea, made safe because the operations are the store's +own), and a small query language (from the IndexedDB shape, but over real tables). + +Inside the store, more choices were dropped during the review: + +| Choice | Why dropped | +| --- | --- | +| all rows of all tables in one generic table `rows(tbl, key, v, row JSON)` | the owner wants real tables; with table definitions in the outbox a replica needs no plugin code anyway | +| index entries maintained by the store in a separate table | an extra row per index per write; SQLite could no longer filter natively | +| ~~code point order (SQLite's native order) with lone surrogates replaced~~ | rejected at first, then chosen (owner, 2026-10-26, "Text is plain UTF-8, in code point order") | +| text only for equality, order only over integers (Durable's own approach) | ordered text, ranges and prefixes must be possible (owner) | +| JSON-field operands in bulk updates | they lean on SQLite's JSON functions; added only when a real case needs them | + +# The Store {#store lede="Layers, tables owned by objects, and the one transaction API."} + +The store sits between the backends and everything that persists data in a session. + +:::flow wide | The layers. Durable keeps its existing storages and gets one more, the adapter; Chord and plugins use the store directly. +row "Owners" blue +box durable "Durable Session" "`StoreStorage` adapter" +box chord "Chord objects" "persisted documents" +box plugin "plugin objects" "`self.table(...)`" +row "Store" accent +box docs "documents engine" "records, revisions, versions" accent +box tables "tables" "query language, writes" accent +box commit "commit step" "seq, outbox, adoption" accent +row "Backends" muted +box sqlite "SQLite" "Node, Durable Objects, browser (WebAssembly)" +box memory "SQLite :memory:" "tests" +arrow durable -> docs +arrow chord -> docs +arrow plugin -> tables +arrow docs -> commit +arrow tables -> commit +arrow commit -> sqlite +note "Durable's memory, SQLite and JSONL storages stay (S15), with only the adopt argument added (S1); StoreStorage joins them." +::: + +**A table belongs to one object**, like a document: only that object reads and writes it; its address is (host kind, +object type, table name), never a host instance such as the session id (W7), so a params change, a split or an import +never orphans data; other plugins see the data only through the object's +state and methods. + +```ts +const Edits = defineTable("edits", { + version: 2, + columns: { id: str(), path: str(), entry: int(), at: int(), meta: json(EditMeta, { version: 1 }) }, // flat + key: ["id"], + indexes: { by_path: ["path", "at"], by_entry: ["entry"] }, +}); + +host.serve(FileHistoryStore, (_, self) => { // a local object with empty params + const edits = self.table(Edits); + return { + record: (edit, ctx) => self.tx((tx) => { tx.put(edits, edit); }, ctx), + versions: ({ path }, ctx) => + edits.page(query(Edits).where((e) => eq(e.path, path)).orderBy((e) => e.at, "desc"), 20, ctx), + }; +}); +``` + +- A table's `columns` are a flat record: each column is a scalar (`str`, `int`, `bool`, `num`) or `json(...)`; nesting + exists only inside `json(...)`. Every column is a real SQLite column. Scalars can be filtered and ordered; keys and + indexed columns must be `str` or `int`. A `json(...)` column holds JSON text: typed when read, never queryable or + indexable (the type checker refuses it). +- Tables and documents of an object are written in the same transaction API, `self.tx(fn, ctx)` (S14). The context + comes last in every signature (owner's rule). +- Index **definitions** are recorded like other table changes: when the owning object starts on the driver, the store + creates or drops the difference between declared and existing indexes, and every replica applies it. Index contents + never travel; every backend maintains them from the rows. So every copy of the store is the same file. + +# Writing It Down {#commit lede="One commit queue per store, one seq for every owner, and adoption inside the step."} + +Two different things are serialized, and they must not be confused. A **line** serializes a whole transaction +callback: read, decide, write. Chord has one line per object, Durable one per Session. The **commit queue** serializes +only the final step: checks, writes, seq, the outbox's commit. It is short and there is one per store. + +:::sequence animate | Two Chord objects and a Durable turn commit at the same time. Their callbacks never wait for each other; only the short commit steps queue. +actor a "object A" "its own line" +actor b "object B" "its own line" +actor d "Durable Session" "its own line" +actor q "commit queue" "one per store" accent +a -> a: draft notes … +b -> b: draft drafts … +d -> d: phase writes … +a -> q: commit +q => q: seq 41: write, outbox, adopt A +b -> q: commit +q => q: seq 42: write, outbox, adopt B +d -> q: commit +q => q: seq 43: write, outbox, adopt Durable +::: + +Durable's seq is the store's seq. Durable sees gaps (other owners' commits in between), and that is harmless: every use +of a seq in Durable compares (`created_at <= at`, `seq > base`); nothing computes `seq + 1` to find a neighbour (checked +in the storages and the Session). + +## Adoption inside the step (S1) + +After a commit is on disk, each owner swaps its in-memory state to the committed version. If that happens later, in +the owner's own continuation, the order can flip: + +``` +disk: 40 file-history, 41 Durable +memory: Durable adopts 41 first (one await fewer) + a routine on "entry 812 committed" reads file-history → still at 39 +``` + +So adoption runs inside the commit step, synchronously, before the next commit starts. Durable's storage interface +carries it as a new required argument: + +```ts +commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; +``` + +Every storage calls `adopt(seq)` once its commit is durable and before its next commit. The Session catches its own +adoption errors; a rejection after adoption fails the Session without discarding the adopted changes. + +## Adopt and mark in the step, flush right after (W1) + +The step itself stays short: it writes, adopts (pointer swaps of every changed value, including Durable's views the +commit touched) and **marks** (dependent derives dirty, ops appended by reference to each touched hold's batch and each +watch's queue). Right after the step, synchronously and before any other code runs, the committing owner **flushes**: +it recomputes the dirty derives someone holds or watches, encodes one part per touched hold into one message per link, +and queues watch callbacks. So a source and the derives over it always travel in one message; a throwing derive fails +only its own member, never the commit. + +``` +step 40: write · adopt notes · mark count dirty, notes' ops into tab 1's batch +flush 40: count = 3 → its ops join the batch · ONE message to tab 1 { notes: +1, count: 3 } +step 41: write · adopt Durable's view · mark lastReply … +``` + +Watch listeners run afterwards, each in order; their order across owners may differ from the seq, but every listener +reads a complete prefix of the log. + +## When a commit counts as done (S3) + +The step ends only on the backend's terminal success: SQLite's `COMMIT` returned (with an asynchronous backend, the +transaction's final completion event, not its individual requests). Any backend error in the step stops the store +(`storage_failed`, Durable's rule): an ambiguous `COMMIT` cannot be told from a failed statement, so nothing more is +written or published. + +:::review | Review: request success is not commit completion +Astra's trace: IndexedDB reports the puts of seq 40 successful, the store publishes 40, the transaction aborts later +(quota), 50 commits on top: memory has 40, disk does not. The rule above is the answer. +::: + +## Ids + +Each owner's id counter starts, when the store opens, above that owner's highest stored id: one index lookup per table +(S10). Chord's document ids are assigned in the commit step; for Durable's `mintId` (6b), an id handed out to a +transaction that never committed may be handed out again after a restart. Nobody can +notice, because nothing leaves a transaction before it commits (S2), so Durable's `mintId` contract is narrowed to say +exactly that. + +# Reading {#reading lede="A small query language, one text order on every backend, and cursors."} + +Queries are immutable data built in TanStack DB's style; S12 starts without joins, grouping and projections: + +```ts +query(Edits).where((e) => eq(e.path, "src/app.ts")).orderBy((e) => e.at, "desc") +query(Edits).where((e) => and(gte(e.at, from), lt(e.at, to))) +query(Edits).where((e) => prefix(e.path, "a/")) +``` + +`where` takes `eq ne lt lte gt gte inArray isNull prefix and or not`. `limit` and the cursor are arguments of the +runner, not part of the query. `count(q)` is the one aggregate; others are computed in JavaScript over indexed scans +until a real case needs more. + +## One text order + +Text compares by Unicode code point on every backend (owner, 2026-10-26; this replaced the first design, UTF-16 BLOBs +in JavaScript order, S16 and W8). That is the order SQLite compares UTF-8 text in natively, and Durable Objects, +sqlite-wasm, Postgres (`COLLATE "C"`) and MySQL (`utf8mb4_0900_bin`) do too. Every `str()` column is plain UTF-8 +`TEXT`: strings go to the driver and come back as they are, readable in `sqlite3` and in any other tool. + +It differs from JavaScript's `<` and `sort()` in one range only: characters U+E000-U+FFFF (fullwidth forms such as +`～`, private use, U+FFFD) against characters above U+FFFF (emoji): + +``` +JS sort(): "测试😀" < "测试（一）" +store order: "测试（一）" < "测试😀" +``` + +The store exports `compareText(a, b)` for plugins that sort store text themselves. Page with the store's cursors and use `prefix()`; a hand-built bound such as +`p + "\uffff"` misses every `p` followed by an emoji. + +UTF-8 cannot hold a lone surrogate, so every `str()` value written, and every query operand, goes through +`toWellFormed()`: a lone surrogate becomes U+FFFD, as `TextEncoder` does. A string cut in the middle of an emoji by +`slice()` is stored with U+FFFD in its place; cut text with `[...s].slice()` to keep it whole. A `str()` value may not +contain NUL (`invalid/args`); content belongs in `json(...)` columns, which escape both. There is no Unicode +normalization: "é" written as one code point and "e" plus a combining accent are two keys. + +| 100k path-like keys | UTF-16 BLOB (the first design) | UTF-8 TEXT | +| --- | --- | --- | +| insert, one transaction | 125 ms | 113 ms | +| read | 106 ms | 32 ms | +| bytes of an 18-character ASCII path | 36 | 18 | + +## Running queries + +A query is immutable data; the table handle runs it, and the context comes last: + +```ts +const edits = self.table(Edits); +const byPath = query(Edits).where((e) => prefix(e.path, "src/")).orderBy((e) => e.path); + +const page = await edits.page(byPath, 100, ctx, { after: cursor }); // { rows, next? } +for await (const row of edits.rows(byPath, ctx)) { … } // all pages +const e1 = await edits.get({ id: "e1" }, ctx); // keys are objects of the key columns + +await self.tx(async (tx) => { + for await (const row of tx.rows(edits, byPath)) { // committed rows, stable during the tx + tx.update(edits, { id: row.id }, { dir: dirname(row.path) }); // writes are synchronous, buffered + } + const again = await tx.get(edits, { id: "e1" }); // sees this transaction's own writes +}, ctx); +``` + +Inside a transaction, `tx.page` and `tx.rows` return committed rows: a stable view, because only the owning object +writes its tables and its transactions run one at a time. They do not show the transaction's own writes; `tx.get` does. +The rule in one sentence: queries inside a transaction see the state before it; your own writes become visible after +the commit. After an `updateWhere` or `deleteWhere` on a table, `tx.get` on that table throws `read_after_bulk` +(W10): the store would need a second evaluator to know what a bulk operation did to one row. + +## Cursors and consistent reads + +Every index ends with the primary key, so order is total and cursors are stable. A cursor is opaque, carries its +direction and its last key as logical values, is bound to its query's order, and is refused by another query or the +other direction. + +Reads outside a transaction see committed state, one query at a time. A read that needs several queries, like a +document assembled from its base and later revisions outside its owner's line (Durable's reads, 6b), runs in one backend +read transaction (S4); Chord's own loads run on the object's line and need none. Two separately +awaited reads in object code are not a snapshot, and the guide says so. + +# Documents {#documents lede="One documents engine in the store, used by Durable's adapter and by Chord."} + +Durable and Chord both persist documents the same way: a record per incarnation, revisions made of a base and later +deltas, versions, `checkpointWhen`, one live document per address. The store implements it once (S13): + +```ts +type DocAddress = { namespace: string; owner: string; kind: string; key?: string }; + +interface Documents { // reads, committed state + find(address: DocAddress, at: At, context: Context): Promise; + load(id: number, at: At, context: Context): Promise; // base + deltas, applied once + list(query: DocListQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise>; +} +interface DocumentWrites { // inside the commit step + create(record: NewDocRecord, base: BaseContent): number; + change(id: number, content: DocContent): void; + retire(id: number): void; +} +``` + +The `owner` string is opaque to the engine; each adapter fills it: + +| | namespace | owner | kind | key | +| --- | --- | --- | --- | --- | +| Durable session document | `pi-durable` | `session` | `pi.settings` | | +| Durable conversation document | `pi-durable` | `conversation:7` | `pi.provider` | | +| Durable task document | `pi-durable` | `task:31` | `pi.codemode` | | +| Chord persisted document | `chord` | `session:notes.store` | `notes` | `n-42` | + +What the owner string means (a conversation's documents fork and rewind with it, a task's are retired with the task) +stays in the adapter. The engine keeps nothing in memory; Durable's Session and Chord's objects keep their Delta +trackers, cache, pin and evict exactly as Durable does today. A fork's document copies are reconstructed by +`StoreStorage` before it enters the commit step (S11), so the step never replays revisions while every owner waits; +the outbox only ever carries the resulting documents. + +# Writing Many Rows {#bulk lede="Bulk operations recorded as operations, and migrations that run only where the code is."} + +A full-row put per changed row makes the rename of `a/` to `b/` over ten thousand 4 KB rows cost about 40 MB in the +outbox. The store's writes are therefore five: + +| Write | Records | +| --- | --- | +| `tx.put(table, row)` | the whole row | +| `tx.update(table, key, set)` | the changed columns of one row | +| `tx.delete(table, key)` | the key | +| `tx.updateWhere(table, where, set)` | one operation, plus the affected row count | +| `tx.deleteWhere(table, where)` | one operation, plus the affected row count | + +```ts +await self.tx((tx) => { + tx.updateWhere(edits, (e) => prefix(e.path, "a/"), (e) => ({ path: replacePrefix(e.path, "a/", "b/") })); +}, ctx); +``` + +```sql +-- on SQLite: one statement; path is UTF-8 TEXT, "a/" is 2 code points +UPDATE "session:file-history.store:edits" SET path = :to || substr(path, 3) WHERE path >= :lo AND path < :hi +``` + +``` +outbox, seq 57 (about 150 bytes): +{ kind: "updateWhere", table: "session:file-history.store:edits", where: { prefix: ["path", "a/"] }, + set: { path: { replacePrefix: ["a/", "b/"] } }, count: 10000 } +``` + +A replica runs the same operation, without plugin code, and compares the number of affected rows. A different number +means divergence: it stops applying that store and restores it from a snapshot. The `set` expressions touch only real +columns (plain values, or column references in a callback): `add`, `sub`, `max`, `min`, `concat`, `replacePrefix`, +`coalesce`, `when`. They are the store's +own, specified once and checked on every backend by one conformance suite, which is why there are no SQL subqueries, +joins, case folding or floats: each of those behaves differently somewhere. + +## Migrations (S17) + +Adding, renaming or dropping a column is a recorded table change (`alter`); a new column gets its default without a +bulk rewrite. A change of the shape **inside a `json(...)` column** (a field added, renamed, restructured) **migrates on +access** (W11): each `json(...)` column has its own version and a value-only migration, +`json(MetaV2, { version: 2, migrate: (old, from) => … })`, stored with its value; reading an older value migrates it, +writing the column stores the new shape and version. Nothing is rewritten in bulk and indexes are unaffected. The row's +`$v` counts only eager steps (below); `update`, `updateWhere` and `deleteWhere` never set it. Old code refuses a newer +table version or a newer JSON column version. + +A change to a key or an indexed column must be **eager**, because an index over a column filled only on access would +silently miss rows. It runs when the owning object starts on the driver, before the object goes live: + +```ts +const EditsV4 = evolveTable(EditsV3, (t) => + t.add("dir", str({ default: "" })).rows((row) => ({ dir: dirname(row.path) })).index("by_dir", ["dir"])); +const EditsV5 = evolveTable(EditsV4, (t) => + t.add("size", int({ default: 0 })).update({ where: (e) => lt(e.size, 0), set: { size: 0 } })); +``` + +Each version is its own materialized definition (as contracts' `evolve`), so a store at version 3 replays 4 and then 5, +each with exactly its own columns. + +:::steps | Migration 4 on the driver ++++ The table change +Commit 80 records `alter: add column dir` and marks table version 4 as `migrating`. Every replica adds the column +from the record. ++++ The scan +The store reads rows with `$v < 4` in key order, 100 at a time (about 1 ms per commit step), applies `rows` to each and writes the changed columns +plus `$v = 4` in the same write. One commit per chunk, so other owners' commits interleave and the commit step stays +short. (A full-row `put` also sets `$v`; partial and bulk writes never do.) ++++ Ready +The last chunk marks version 4 `ready`. The store creates the `byDir` index and records it, and `FileHistoryStore` +goes live. ++++ After a crash in the middle +The next start finds `migrating`, scans again from the first key and skips rows with `$v >= 4`. Each row is +transformed exactly once, even when the transform is not idempotent. A key change is never a `rows` step: it is a +recorded `rebuild` that every replica performs itself. +::: + +A replica, with or without the plugin, applies the recorded column changes and the column updates; it never runs a +migration itself. Chord documents keep their own per-document `migrate` on access, and +Durable versions its records itself. + +# The Easy Parts {#easy lede="Blobs and derived data: why they need none of the store's ordering machinery."} + +Two kinds of session data stay outside the store's tables, and both are easy for the same reason: nothing about them +ever changes in place. A blob is final once written; derived data has no data of its own. + +## Blobs + +Artifact versions, uploads and sent files are **blobs**: files in the session directory, named by the hash of their +content (`blobs/sha256/9f/9f3a…`), never changed. They are app layer, not store: any process may write one, because a +file named by its content cannot conflict with another writer. The order is the whole trick: the blob is written first, +then the row or document that names it. + +:::sequence animate | Saving artifact "chart" version 3. The blob exists before anything refers to it. +actor tool "artifact tool" +actor blobs "blobs/" muted +actor store "store" blue +tool -> blobs: write sha256:9f3a… (tmp, fsync, rename) +tool => store: seq 41: chart.versions[2] = { blob: "sha256:9f3a…" } +tool => store: seq 42: entry 812 "chart v3 saved" (Durable) +::: + +- **Export** copies `blobs/` together with the store snapshot. Every blob a record up to the export's `lastSeq` names was + written before that record, so it is in the directory. **Import** copies them back and checks each hash. +- **Sync:** before uploading a batch, the sync layer uploads every blob file that is present and not yet uploaded. No + reference tracking is needed: a blob is always written before the commit that names it, so a replica never sees a + hash whose file is missing. +- **Deleting:** no blob is deleted before its session is deleted. A blob that nothing references any more (a crash + between the write and the commit, a retired document) only costs disk; a garbage collector would have to know every + reference in every table. + +## Derived data + +A full-text index, a cache, a plugin's own search database: **derived data** can be recomputed from the store and the +blobs, so it is never exported or synced. Its owner keeps it current with a routine that compares it with the store. +The artifacts plugin's full-text index in its own SQLite file: + +```ts +host.serve(ArtifactStore, (_, self) => { + const artifacts = self.state(Artifact); // keyed documents in the store + const blobs = self.get(SessionBlobs, {}); + const index = self.open(async () => openIndex(join(dir, "derived/artifacts/index.sqlite")), (db) => db.close()); + + self.onLive(async (ctx) => { // the index repairs itself + for await (const name of artifacts.keys(ctx)) { + const a = await artifacts.get(name, ctx); + if (a && index.value.versions(name) !== a.versions.length) + index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx)); + } + }); + … +}); +``` + +:::steps | One routine, three situations ++++ Normal save +`save` writes the blob, commits the document, then updates the index. The routine finds nothing to do. ++++ Crash after the commit +The process dies between the commit and the index update. The document says version 3, the index still has version 2. +On the next start the routine sees the difference, reads the blob of version 3 and indexes it. ++++ Import or sync +On the desktop `derived/artifacts/` does not exist. The routine finds every artifact missing and indexes them all from +the documents and blobs that came with the session. +::: + +This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store keeps +the last seq it applied and reads only what changed after it. + +# Moving a Session {#sync lede="Outbox, replicas (no compaction yet), takeover, export and import."} + +With an outbox (optional: a store opened with `{ outbox: true }`), every commit's writes go into it, written in the +same backend transaction as its data, so a crash keeps both or neither. Sync is a layer above the store; it follows Armin's transport and fencing with the rules the review added. + +- **Uploading:** the driver may upload batches in parallel, resubmits each until Radius acknowledges it, and trims a + batch from its outbox only after its acknowledgement (S5). +- **Applying:** every batch carries `from` and `to`. A replica applies a batch only when its `from` equals the + replica's `lastSeq` (the seq of the last commit it applied), as a whole, in one backend transaction together with + its new `lastSeq`. A later batch + waits for a missing one. A crash leaves all of a batch or none of it. +- **No compaction at first (W9):** batches carry the ordered operations exactly as committed. "Keep the last write per + row" would lose rows built from `put` plus partial `update`s; a later compaction must merge those and treat bulk + operations as barriers. +- **After a power loss** the driver's local `lastSeq` may be behind what Radius has; it first pulls back its own uploaded + batches, then continues (S3). +- **Blobs** go up before the batch that may name them: every blob file present and not yet uploaded (chapter VIII). + +## Losing the driver role (S7) + +:::sequence animate | The laptop learns the desktop took over. Nothing is rolled back. +actor laptop "laptop" "driver, epoch 3" +actor radius "Radius" blue +actor desktop "desktop" +desktop -> radius: take over S1 +radius => desktop: epoch 4 +laptop -> laptop: commits 40, 41 locally (a routine, a late tool result) +laptop -> radius: upload 40–41 (epoch 3) +radius --> laptop: refused: epoch 4 +laptop => laptop: the copy becomes session S1′ (new id) +note laptop..desktop: S1 continues on the desktop; S1′ on the laptop; they never sync again +::: + +Rolling back is not possible: tools ran, files were written, natives and derived data followed the tail. The laptop's +copy becomes a separate session with a new id and keeps running; commits in flight land in it. The catalog shows both, +the split-off copy marked with its origin. + +## Export and import (S4) + +An export holds the commit queue while it reads, so the archive is the state after one seq, its `lastSeq`: Durable's records, Chord's +documents and every plugin table, including tables of plugins that are not installed, plus the session directory's +blobs (chapter VIII). The archive is written in the outbox's own operations (a manifest, `table`, `index`, `put` and `document.create` +records, an `end` record), as one batch from 0 to its `lastSeq`, so an import is applied by the same code as a +replica's batches. An import restores the archive into a fresh store in one transaction; the server then mirrors the +session into its catalog. The encoding is pluggable (the store yields and takes records); plain JSONL ships first. +Derived data (full-text indexes, caches) is never exported; it is rebuilt where it is needed. + +# Rules for Plugin Authors {#rules lede="What the store does not do for you."} + +**Nothing leaves a transaction before it commits (S2).** Ids are handed out inside the transaction that creates them. +Reference what is committed: the file-history plugin references the tool-call entry, which is committed before the tool +runs, never the tool-result entry, which does not exist yet. + +**Work that finishes out of order is yours (S8).** A routine that starts a slow job per change can let an old result +overwrite a newer one. The handler calls of one routine never overlap, so either chain through the routine or drop +stale results inside the destination's transaction: + +```ts +self.on(conversation.entries, async ({ value: entries }, ctx) => { + const run = await summarize.start({ upTo: entries.at(-1)!.id }, ctx); + await run.outcome(ctx); // the next handler waits for this run +}); +``` + +**Text compares by code point.** Do not sort keys yourself to page through a table; use the store's cursors, or +`compareText`. Normalize text yourself if two spellings must be one key. + +**Bulk changes are operations.** Prefer `updateWhere` and `deleteWhere` over loops of puts; a loop records every row. + +# What's Next {.epilogue lede="What is decided, and the order in which it gets built."} + +The direction and all seventeen review items are decided (decision log, 2026-10-21). The store is new code; the parts +outside it are small changes in pi: + +- **pi, one change:** `Storage.commit(writes, adopt, context)` with every storage calling `adopt` (S1), the + Session's handling of rejections after adoption, the narrowed `mintId` comment (S10), `traceIn(options, fn, + context)` (context last), Durable's `WatchHandle`/`WatchEnd` moved into Chord with generic end reasons and explicit + retirement (W2), and Durable's state getters and the `replicatedState` family removed (W14). +- **The store** in `packages/chord/src/next`: commit step and outbox, tables and the query language, the SQLite + backend with the text encoding, the documents engine, bulk operations and migrations, export and import; + about 1,650 lines by Fable's estimate, plus its tests. +- **`StoreStorage`**, passing Durable's existing storage conformance suite. +- **The sync layer** (upload, apply, pull-back, epochs, splitting a session) on top, when Radius sync is built. + +Open, to settle while building: how a split-off session gets its new id (store metadata, directory, catalog), and a Postgres backend once a serverless target needs one (W14). + +# Decisions {.appendix} + +| # | Decision | +| --- | --- | +| The store | one transactional store per session under Durable (adapter), Chord documents and plugin tables | +| S1 | adoption inside the commit step; `Storage.commit(writes, adopt, context)` | +| S2 | nothing leaves a transaction before it commits; no mechanism | +| S3 | a commit is done on the backend's terminal success; ambiguous outcomes stop the store; pull-back after restart | +| S4 | export holds the commit queue; multi-query reads in one read transaction; no global snapshots | +| S5 | parallel uploads, resubmit until acknowledged, trim after acknowledgement; apply whole batches in order | +| S6 | (superseded by W9) | +| S7 | losing the driver role splits the session; no rollback | +| S8 | out-of-order work is an app concern; routine handlers never overlap | +| S9 | `lastSeq` (the seq of the last commit) is persisted with every commit | +| S10 | id counters derived at open; `mintId` contract narrowed | +| S11 | fork copies reconstructed by `StoreStorage` before the commit step | +| S12 | the query language starts without joins, grouping and projections | +| S13 | one documents engine in the store, opaque owner | +| S14 | one transaction API, `self.tx(fn, ctx)` | +| S15 | Durable's existing storages are kept | +| S16, text | real tables; text as plain UTF-8 in code point order (replaced W8's UTF-16 BLOBs) | +| S17, W11 | five writes, bulk operations recorded with a row count; JSON columns migrate on access with their own version; eager steps by scan with row `v` | +| W1 | adopt and mark in the commit step, one synchronous flush right after (derives, wire, watches) | +| W6, W7 | workers connect before loading; store addresses carry the host kind, never an instance | +| W9, W10 | no compaction at first; no `tx.get` on a table after a bulk operation in the same transaction | +| W14 | no IndexedDB backend (SQLite in WebAssembly in the browser); no `replicatedState` | +| index definitions | recorded in the outbox and applied by every replica; contents never travel | +| blobs, derived data | blobs are app-layer files named by hash, written before what names them, uploaded before each batch, deleted only with the session; derived data is rebuilt, never shipped | + +# Glossary {.appendix} + +**Adoption:** an owner updating its in-memory state to a committed write. **Blob:** a file in the session directory +named by the hash of its content, never changed. **Derived data:** data recomputed from the store and blobs, never +shipped. **Backend:** what the store runs on (SQLite, +memory). **Write:** one change of the store (`put`, `update`, `updateWhere`, `table`, `document.create`, …), the same in the +outbox and in an archive. **Commit** (in the outbox): the writes of one seq. **Batch:** the commits from `from` to `to`, +uploaded and applied as a whole. **Archive:** a manifest, the writes from 0 to `lastSeq`, and an `end`. +**Commit step:** the short, serialized part of a commit in which the store checks, writes, numbers and records it, and +owners adopt. **Driver:** the machine that writes a session. **Line:** the serialization of one owner's transactions +(per Chord object, per Durable Session). **Outbox:** the record of every commit, written in the same transaction. +**Owner:** the code that writes a piece of data. **Replica:** a copy of a session that applies the driver's batches. +**Seq:** the number of a commit, one sequence per store. **Store:** the transactional store of one session. +**Table:** a set of rows owned by one object, with a flat record of typed columns (scalars or `json(...)`). diff --git a/packages/chord/docs/next/store/index.html b/packages/chord/docs/next/store/index.html new file mode 100644 index 000000000..067a77772 --- /dev/null +++ b/packages/chord/docs/next/store/index.html @@ -0,0 +1,861 @@ + + +One Store: how a p3 session keeps, exports and syncs everything it owns + + + + +

+
+
A design book · Chord v2 and p3 · 2026-10-21
+

One
Store

+

Why Durable's transcript, Chord's documents and every plugin's own tables live in one transactional store, and how that store moves a session to another machine without coming apart

+
17review items decided (S1–S17)
1commit queue per store, one seq for every owner
~150 Boutbox record for renaming 10,000 rows (instead of ~40 MB)
0lines of plugin code a replica needs
+

Written for the owner from the discussion of 2026-10-21 and the store review by Fable and Astra. Every decision is in +the decision log (packages/chord/docs/next/decisions.md, entries "The store", "S1" to "S17", "Context last"); the +review is summarized in apps/p2/docs/p3-reviews/store-review-summary.md, bulk operations and migrations in +store-bulk-migrations.md, the earlier discussion with measurements in session-data-scenarios.md. Numbers were +measured on slayer with Node 24.4.1 and node:sqlite unless marked as estimates. Nothing here is built yet. Notes +marked review are points the reviewers raised that the design answers.

+
+
+ + +
+

Before we begin

+

Mario works in session S1 on his laptop. The agent edits files, and a plugin called file-history remembers every +edit: which file, which conversation entry made it, the content before and after. It keeps those rows in its own table, +because it needs queries: every version of src/app.ts, every file the last turn touched, every edit under a/. When +Mario renames the directory a/ to b/, ten thousand rows change.

+

In the evening he wants to continue S1 on his desktop. Another day he exports S1 to a file and imports it into a +colleague's p3. Each time the other machine must end up with the conversation (Durable), the plugins' documents (Chord) +and the file-history table, and all three must agree: a row that says "entry 812 made this edit" must find entry 812, +the same entry 812.

+

Copying files does not do that, and neither does syncing each storage on its own. This book explains the problem, the +solutions that were tried and dropped, and the design that came out: one transactional store per session, under +Durable, Chord and every plugin.

+

Eight things to know

+
    +
  1. A session is self-contained. Everything it owns lives in its store and its directory; it can be exported from +the session alone, imported elsewhere, synced and continued on another machine (P1).
    2. +
  2. One store holds everything. Durable's records (through an adapter), Chord's persisted documents and plugin +tables commit into one store, through one commit queue, numbered by one seq.
    3. +
  3. Order is guaranteed where it is observable. Disk, outbox and export follow the seq; in-memory adoption runs +inside the commit step (S1), so no reader ever sees seq 41 without seq 40.
    4. +
  4. The outbox records what changed, not code. Rows, column updates, bulk operations and table changes, written in +the same transaction as the data. A replica applies them without any plugin code.
    5. +
  5. Tables are real tables with a small query language. Keys and indexed columns are typed; text is UTF-8 and +compares by code point on every backend.
    6. +
  6. Bulk writes stay small. updateWhere records one operation instead of one row per change, and every replica runs +the same operation.
    7. +
  7. Migrations run only where the code is. Changes inside json(...) columns migrate on access; key and index changes run eagerly on the +driver and ship as ordinary writes.
    8. +
  8. Losing control never rolls back. A device that learns another one took over turns its copy into a separate +session.
    9. +
+

Words used throughout

+

A session is one agent workspace: its conversations, tasks and plugin data. An owner is the code that writes a +piece of data: Durable's Session, a Chord object, a plugin's object. The store is the transactional store of one +session; a backend is what it runs on (SQLite, including in the browser as WebAssembly, or a memory backend for tests). A table is a set of +rows owned by one object. A commit is one atomic write; the commit step is the short, serialized part of a commit +in which the store checks, writes and numbers it; the seq is that number. The outbox holds a record of every +commit, written in the same backend transaction. The driver is the machine that writes a session; a replica is +a copy on another machine that applies the driver's records. Adoption is an owner updating its in-memory state to a +committed write.

+
+ +
I

A Session Is More Than Its Transcript

What has to travel with a session, and why separate storages cannot keep it consistent.

+

A p3 session holds three kinds of data written by three kinds of owners. Durable writes the transcript: conversations, +entries, tasks and its own documents. Chord objects write persisted documents: settings, drafts, plugin state. And a +plugin may need data shaped for queries, like file-history's edits.

+

The owner's rule for p3 (P1) is that a session is self-contained. Everything of it lives in its storage plus its +directory; the server only mirrors it (the session catalog) and keeps instance data (shares, devices, settings). That +makes three things possible: export a session from its files alone, import it into another p3, and sync it to Radius +so another device can continue it.

+

Durable alone already supports that shape: one storage, every change in an atomic commit, numbered by a seq. Armin's +sync design builds on it: a decorator captures each committed batch into an outbox, batches go to Radius, a replica +replays them. It breaks as soon as a session has a second storage.

+
One edit, two storages, synced separately

On the laptop

The edit tool records the edit in file-history's own history.sqlite (row: path x.ts, entry 812). Durable commits +the tool's result. Both exist and agree.

+

The history arrives first

The desktop has received history.sqlite up to the edit, but Durable's log only up to the commit before. The row says +entry 812 made the edit; the desktop has no entry 812.

+

Mario takes over now

The laptop is closed; the desktop becomes the driver and writes its next entry, which gets id 812: a different +message. The row now points to the wrong entry, for good.

+
+

Each storage is consistent on its own. What is missing is one order across them: which change of history.sqlite +happened before which commit of the transcript. Every solution below is judged by whether it provides that order, +lets a replica work without the plugin's code, and keeps plugins able to model what they need.

+
+ +
II

What We Tried and Dropped

The solutions discussed on the way, and the reason each one failed.

+

The first answers tried to avoid giving plugins storage at all. The later ones gave them storage and tried to sync it.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdeaWhy it was dropped
Plugins' own databases hold only derived data (indexes, caches), rebuilt after an importtoo limiting: some plugins need their own write model and queries (owner)
Export plugin directories as opaque filesneeds a stopped session; a changing database cannot sync in small pieces
Plugin export and import hooksevery plugin would build its own incremental log for sync
Plugin record logs in the session log, the plugin's database rebuilt from them by an apply functionthe plugin must keep apply deterministic and every record version forever
Several storages, each with an outbox inside its own transaction, ordered by one shared position counterworks (prototype), but many outboxes and a "safe cut" rule; SQLite's row-change records are per row (a 10k-row rename is 259–418 KB); schema changes are not captured
A SQLite store that records the plugin's SQL statements, checked by a hash of their effectSQLite only; arbitrary SQL can give different results on replicas (random(), unordered INSERT … SELECT, SQLite versions)
Raw SQL reads with controlled writesties every backend to SQLite and needs a guard against SQL that writes
An IndexedDB-shaped API only (keys, indexes, ranges)too weak for real queries (owner)
Chord's documents inside Durable's existing storagecovers Chord, not plugin tables
+

The store design kept something from almost each: one order for everything (from the shared counter), recorded +operations instead of rows where possible (from the statement idea, made safe because the operations are the store's +own), and a small query language (from the IndexedDB shape, but over real tables).

+

Inside the store, more choices were dropped during the review:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
ChoiceWhy dropped
all rows of all tables in one generic table rows(tbl, key, v, row JSON)the owner wants real tables; with table definitions in the outbox a replica needs no plugin code anyway
index entries maintained by the store in a separate tablean extra row per index per write; SQLite could no longer filter natively
code point order (SQLite's native order) with lone surrogates replacedlossy ("\uD800" and "\uFFFD" collide) and needs a comparator in JS
text only for equality, order only over integers (Durable's own approach)ordered text, ranges and prefixes must be possible (owner)
JSON-field operands in bulk updatesthey lean on SQLite's JSON functions; added only when a real case needs them
+
+ +
III

The Store

Layers, tables owned by objects, and the one transaction API.

+

The store sits between the backends and everything that persists data in a session.

+
OWNERSSTOREBACKENDSDurable Session`StoreStorage` adapterChord objectspersisted documentsplugin objectsself.table(...)documents enginerecords, revisions, versionstablesquery language, writescommit stepseq, outbox, adoptionSQLiteNode, Durable Objects, browser(WebAssembly)memorytestsDurable's memory, SQLite and JSONL storages stay (S15), with only the adopt argument added (S1); StoreStorage joinsthem.
Figure 1. The layers. Durable keeps its existing storages and gets one more, the adapter; Chord and plugins use the store directly.
+

A table belongs to one object, like a document: only that object reads and writes it; its address is (host kind, +object type, table name), never a host instance such as the session id (W7), so a params change, a split or an import +never orphans data; other plugins see the data only through the object's +state and methods.

+
const Edits = defineTable("edits", {
+  version: 2,
+  columns: { id: str(), path: str(), entry: int(), at: int(), meta: json<EditMeta>() },   // flat
+  key: ["id"],
+  indexes: { byPath: ["path", "at"], byEntry: ["entry"] },
+});
+
+host.serve(FileHistoryStore, (_, self) => {          // a local object with empty params
+  const edits = self.table(Edits);
+  return {
+    record: (edit, ctx) => self.tx(async (tx) => { await tx.put(edits, edit); }, ctx),
+    versions: ({ path }, ctx) =>
+      edits.page(query(Edits).where((e) => eq(e.path, path)).orderBy((e) => e.at, "desc"), 20, ctx),
+  };
+});
+
    +
  • A table's columns are a flat record: each column is a scalar (str, int, bool, num) or json(...); nesting +exists only inside json(...). Every column is a real SQLite column. Scalars can be filtered and ordered; keys and +indexed columns must be str or int. A json(...) column holds JSON text: typed when read, never queryable or +indexable (the type checker refuses it).
    • +
  • Tables and documents of an object are written in the same transaction API, self.tx(fn, ctx) (S14). The context +comes last in every signature (owner's rule).
    • +
  • Index definitions are recorded like other table changes: when the owning object starts on the driver, the store +creates or drops the difference between declared and existing indexes, and every replica applies it. Index contents +never travel; every backend maintains them from the rows. So every copy of the store is the same file.
    • +
+
+ +
IV

Writing It Down

One commit queue per store, one seq for every owner, and adoption inside the step.

+

Two different things are serialized, and they must not be confused. A line serializes a whole transaction +callback: read, decide, write. Chord has one line per object, Durable one per Session. The commit queue serializes +only the final step: checks, writes, seq, outbox record. It is short and there is one per store.

+
object Aits own lineobject Bits own lineDurable Sessionits own linecommit queueone per storedraft notes …draft drafts …phase writes …commitseq 41: write, outbox, adopt Acommitseq 42: write, outbox, adopt Bcommitseq 43: write, outbox, adopt Durable
Figure 2. Two Chord objects and a Durable turn commit at the same time. Their callbacks never wait for each other; only the short commit steps queue.
+

Durable's seq is the store's seq. Durable sees gaps (other owners' commits in between), and that is harmless: every use +of a seq in Durable compares (created_at <= at, seq > base); nothing computes seq + 1 to find a neighbour (checked +in the storages and the Session).

+

Adoption inside the step (S1)

+

After a commit is on disk, each owner swaps its in-memory state to the committed version. If that happens later, in +the owner's own continuation, the order can flip:

+
disk:    40 file-history, 41 Durable
+memory:  Durable adopts 41 first (one await fewer)
+         a routine on "entry 812 committed" reads file-history → still at 39
+

So adoption runs inside the commit step, synchronously, before the next commit starts. Durable's storage interface +carries it as a new required argument:

+
commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise<Seq>;
+

Every storage calls adopt(seq) once its commit is durable and before its next commit. The Session catches its own +adoption errors; a rejection after adoption fails the Session without discarding the adopted changes.

+

Adopt and mark in the step, flush right after (W1)

+

The step itself stays short: it writes, adopts (pointer swaps of every changed value, including Durable's views the +commit touched) and marks (dependent derives dirty, ops appended by reference to each touched hold's batch and each +watch's queue). Right after the step, synchronously and before any other code runs, the committing owner flushes: +it recomputes the dirty derives someone holds or watches, encodes one part per touched hold into one message per link, +and queues watch callbacks. So a source and the derives over it always travel in one message; a throwing derive fails +only its own member, never the commit.

+
step 40:  write · adopt notes · mark count dirty, notes' ops into tab 1's batch
+flush 40: count = 3 → its ops join the batch · ONE message to tab 1 { notes: +1, count: 3 }
+step 41:  write · adopt Durable's view · mark lastReply …
+

Watch listeners run afterwards, each in order; their order across owners may differ from the seq, but every listener +reads a complete prefix of the log.

+

When a commit counts as done (S3)

+

The step ends only on the backend's terminal success: SQLite's COMMIT returned (with an asynchronous backend, the +transaction's final completion event, not its individual requests). The memory backend stages a +commit and swaps it in at the end. An ambiguous outcome, an error during COMMIT, stops the store +(storage_failed): nobody knows whether the commit landed, so nothing more is written or published.

+
Review: request success is not commit completion

Astra's trace: IndexedDB reports the puts of seq 40 successful, the store publishes 40, the transaction aborts later +(quota), 50 commits on top: memory has 40, disk does not. The rule above is the answer.

+
+

Ids

+

Each owner's id counter starts, when the store opens, above that owner's highest stored id: one index lookup per table +(S10). An id handed out to a transaction that never committed may be handed out again after a restart. Nobody can +notice, because nothing leaves a transaction before it commits (S2), so Durable's mintId contract is narrowed to say +exactly that.

+
+ +
V

Reading

A small query language, one text order on every backend, and cursors.

+

Queries are immutable data built in TanStack DB's style; S12 starts without joins, grouping and projections:

+
query(Edits).where((e) => eq(e.path, "src/app.ts")).orderBy((e) => e.at, "desc")
+query(Edits).where((e) => and(gte(e.at, from), lt(e.at, to)))
+query(Edits).where((e) => prefix(e.path, "a/"))
+

where takes eq ne lt lte gt gte inArray isNull prefix and or not. limit and the cursor are arguments of the +runner, not part of the query. count(q) is the one aggregate; others are computed in JavaScript over indexed scans +until a real case needs more.

+

One text order

+

Text compares by Unicode code point on every backend (owner, 2026-10-26; this replaced the first design, UTF-16 BLOBs +in JavaScript order, S16 and W8). That is the order SQLite compares UTF-8 text in natively, and Durable Objects, +sqlite-wasm, Postgres (COLLATE "C") and MySQL (utf8mb4_0900_bin) do too. Every str() column is plain UTF-8 +TEXT: strings go to the driver and come back as they are, readable in sqlite3 and in any other tool.

+

It differs from JavaScript's < and sort() in one range only: characters U+E000-U+FFFF (fullwidth forms such as +～, private use, U+FFFD) against characters above U+FFFF (emoji):

+
JS sort():    "测试😀" < "测试（一）"
+store order:  "测试（一）" < "测试😀"
+

The store exports compareText(a, b) for plugins that sort store text themselves; the memory backend uses it for +ordering, ranges, max and min. Page with the store's cursors and use prefix(); a hand-built bound such as +p + "\uffff" misses every p followed by an emoji.

+

UTF-8 cannot hold a lone surrogate, so every str() value written, and every query operand, goes through +toWellFormed(): a lone surrogate becomes U+FFFD, as TextEncoder does. A string cut in the middle of an emoji by +slice() is stored with U+FFFD in its place; cut text with [...s].slice() to keep it whole. A str() value may not +contain NUL (invalid/args); content belongs in json(...) columns, which escape both. There is no Unicode +normalization: "é" written as one code point and "e" plus a combining accent are two keys.

+ + + + + + + + + + + + + + + + + + + + + + + +
100k path-like keysUTF-16 BLOB (the first design)UTF-8 TEXT
insert, one transaction125 ms113 ms
read106 ms32 ms
bytes of an 18-character ASCII path3618
+

Running queries

+

A query is immutable data; the table handle runs it, and the context comes last:

+
const edits = self.table(Edits);
+const byPath = query(Edits).where((e) => prefix(e.path, "src/")).orderBy((e) => e.path);
+
+const page = await edits.page(byPath, 100, ctx, { after: cursor });   // { rows, next? }
+for await (const row of edits.rows(byPath, ctx)) { … }                 // all pages
+const e1 = await edits.get(["e1"], ctx);
+
+await self.tx(async (tx) => {
+  for await (const row of tx.rows(edits, byPath)) {                    // committed rows, stable during the tx
+    await tx.update(edits, [row.id], { dir: dirname(row.path) });
+  }
+  const again = await tx.get(edits, ["e1"]);                           // sees this transaction's own writes
+}, ctx);
+

Inside a transaction, tx.page and tx.rows return committed rows: a stable view, because only the owning object +writes its tables and its transactions run one at a time. They do not show the transaction's own writes; tx.get does. +The rule in one sentence: queries inside a transaction see the state before it; your own writes become visible after +the commit. After an updateWhere or deleteWhere on a table, tx.get on that table throws read_after_bulk +(W10): the store would need a second evaluator to know what a bulk operation did to one row.

+

Cursors and consistent reads

+

Every index ends with the primary key, so order is total and cursors are stable. A cursor is opaque, carries its +direction and its last key as logical values, is bound to its query's order, and is refused by another query or the +other direction.

+

Reads outside a transaction see committed state, one query at a time. A read that needs several queries, like a +document assembled from its base and later revisions, runs in one backend read transaction (S4). Two separately +awaited reads in object code are not a snapshot, and the guide says so.

+
+ +
VI

Documents

One documents engine in the store, used by Durable's adapter and by Chord.

+

Durable and Chord both persist documents the same way: a record per incarnation, revisions made of a base and later +deltas, versions, checkpointWhen, one live document per address. The store implements it once (S13):

+
type DocAddress = { namespace: string; owner: string; kind: string; key?: string };
+
+interface Documents {                                   // reads, committed state
+  find(address: DocAddress, at: At, context: Context): Promise<DocRecord | undefined>;
+  load(id: number, at: At, context: Context): Promise<StoredDoc | undefined>;   // base + deltas, applied once
+  list(query: DocListQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<DocRecord>>;
+}
+interface DocumentWrites {                              // inside the commit step
+  create(record: NewDocRecord, base: BaseContent): number;
+  change(id: number, content: DocContent): void;
+  retire(id: number): void;
+}
+

The owner string is opaque to the engine; each adapter fills it:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
namespaceownerkindkey
Durable session documentpi-durablesessionpi.settings
Durable conversation documentpi-durableconversation:7pi.provider
Durable task documentpi-durabletask:31pi.codemode
Chord persisted documentchordsession/notes.storenotesn-42
+

What the owner string means (a conversation's documents fork and rewind with it, a task's are retired with the task) +stays in the adapter. The engine keeps nothing in memory; Durable's Session and Chord's objects keep their Delta +trackers, cache, pin and evict exactly as Durable does today. A fork's document copies are reconstructed by +StoreStorage before it enters the commit step (S11), so the step never replays revisions while every owner waits; +the outbox only ever carries the resulting documents.

+
+ +
VII

Writing Many Rows

Bulk operations recorded as operations, and migrations that run only where the code is.

+

A full-row put per changed row makes the rename of a/ to b/ over ten thousand 4 KB rows cost about 40 MB in the +outbox. The store's writes are therefore five:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
WriteRecords
tx.put(table, row)the whole row
tx.update(table, key, set)the changed columns of one row
tx.delete(table, key)the key
tx.updateWhere(table, where, set)one operation, plus the affected row count
tx.deleteWhere(table, where)one operation, plus the affected row count
+
await self.tx(async (tx) => {
+  await tx.updateWhere(Edits, (e) => prefix(e.path, "a/"), { path: replacePrefix("a/", "b/") });
+}, ctx);
+
-- on SQLite: one statement; path is UTF-8 TEXT, "a/" is 2 code points
+UPDATE "file-history/edits" SET path = :to || substr(path, 3) WHERE path >= :lo AND path < :hi
+
outbox, seq 57 (about 150 bytes):
+{ op: "update", table: "file-history/edits", where: { prefix: ["path", "a/"] },
+  set: { path: { replacePrefix: ["a/", "b/"] } }, count: 10000 }
+

A replica runs the same operation, without plugin code, and compares the number of affected rows. A different number +means divergence: it stops applying that store and restores it from a snapshot. The set expressions touch only real +columns: value, col, add, sub, max, min, concat, replacePrefix, coalesce, when. They are the store's +own, specified once and checked on every backend by one conformance suite, which is why there are no SQL subqueries, +joins, case folding or floats: each of those behaves differently somewhere.

+

Migrations (S17)

+

Adding, renaming or dropping a column is a recorded table change (alter); a new column gets its default without a +bulk rewrite. A change of the shape inside a json(...) column (a field added, renamed, restructured) migrates on +access (W11): each json(...) column has its own version and a value-only migration, +json(MetaV2, { version: 2, migrate: (old, from) => … }), stored with its value; reading an older value migrates it, +writing the column stores the new shape and version. Nothing is rewritten in bulk and indexes are unaffected. The row's +v counts only eager steps (below); update, updateWhere and deleteWhere never set it. Old code refuses a newer +row v or a newer JSON column version.

+

A change to a key or an indexed column must be eager, because an index over a column filled only on access would +silently miss rows. It runs when the owning object starts on the driver, before the object goes live:

+
migrations: {
+  4: { alter: { add: { dir: { type: "text" } } },
+       rows: (row) => ({ dir: dirname(row.path) }) },                         // scan + tx.update, in chunks
+  5: { alter: { add: { size: { type: "int", default: 0 } } },
+       update: { where: (e) => isNull(e.size), set: { size: value(0) } } },     // one updateWhere
+},
+
Migration 4 on the driver

The table change

Commit 80 records alter: add column dir and marks table version 4 as migrating. Every replica adds the column +from the record.

+

The scan

The store reads rows with v < 4 in key order, 500 at a time, applies rows to each and writes the changed columns +plus v = 4 in the same write. One commit per chunk, so other owners' commits interleave and the commit step stays +short. (A full-row put also sets v; partial and bulk writes never do.)

+

Ready

The last chunk marks version 4 ready. The store creates the byDir index and records it, and FileHistoryStore +goes live.

+

After a crash in the middle

The next start finds migrating, scans again from the first key and skips rows with v >= 4. Each row is +transformed exactly once, even when the transform is not idempotent. A key change is never a rows step: it is a +recorded rebuild that every replica performs itself.

+
+

A replica without the plugin applies the column change and the column updates; a replica with the plugin's code treats +the table as not ready while it is migrating. Chord documents keep their own per-document migrate on access, and +Durable versions its records itself.

+
+ +
VIII

The Easy Parts

Blobs and derived data: why they need none of the store's ordering machinery.

+

Two kinds of session data stay outside the store's tables, and both are easy for the same reason: nothing about them +ever changes in place. A blob is final once written; derived data has no data of its own.

+

Blobs

+

Artifact versions, uploads and sent files are blobs: files in the session directory, named by the hash of their +content (blobs/sha256/9f/9f3a…), never changed. They are app layer, not store: any process may write one, because a +file named by its content cannot conflict with another writer. The order is the whole trick: the blob is written first, +then the row or document that names it.

+
artifact toolblobs/storewrite sha256:9f3a… (tmp, fsync, rename)seq 41: chart.versions[2] = { blob: "sha256:9f3a…" }seq 42: entry 812 "chart v3 saved" (Durable)
Figure 3. Saving artifact "chart" version 3. The blob exists before anything refers to it.
+
    +
  • Export copies blobs/ together with the store snapshot. Every blob a record up to the export's head names was +written before that record, so it is in the directory. Import copies them back and checks each hash.
    • +
  • Sync: before uploading a batch, the sync layer uploads every blob file that is present and not yet uploaded. No +reference tracking is needed: a blob is always written before the commit that names it, so a replica never sees a +hash whose file is missing.
    • +
  • Deleting: no blob is deleted before its session is deleted. A blob that nothing references any more (a crash +between the write and the commit, a retired document) only costs disk; a garbage collector would have to know every +reference in every table.
    • +
+

Derived data

+

A full-text index, a cache, a plugin's own search database: derived data can be recomputed from the store and the +blobs, so it is never exported or synced. Its owner keeps it current with a routine that compares it with the store. +The artifacts plugin's full-text index in its own SQLite file:

+
host.serve(ArtifactStore, (_, self) => {
+  const artifacts = self.state(Artifact);                         // keyed documents in the store
+  const blobs = self.get(SessionBlobs, {});
+  const index = self.open(async () => openIndex(join(dir, "derived/artifacts/index.sqlite")), (db) => db.close());
+
+  self.routines.loop("catch-up", async (ctx) => {                 // the index repairs itself
+    for await (const name of artifacts.keys()) {
+      const a = await artifacts.get(name);
+      if (a && index.value.versions(name) !== a.versions.length)
+        index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx));
+    }
+  });
+  …
+});
+
One routine, three situations

Normal save

save writes the blob, commits the document, then updates the index. The routine finds nothing to do.

+

Crash after the commit

The process dies between the commit and the index update. The document says version 3, the index still has version 2. +On the next start the routine sees the difference, reads the blob of version 3 and indexes it.

+

Import or sync

On the desktop derived/artifacts/ does not exist. The routine finds every artifact missing and indexes them all from +the documents and blobs that came with the session.

+
+

This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store keeps +the last seq it applied and reads only what changed after it.

+
+ +
IX

Moving a Session

Outbox, replicas (no compaction yet), takeover, export and import.

+

Every commit's outbox record is written in the same backend transaction as its data, so a crash keeps both or +neither. Sync is a layer above the store; it follows Armin's transport and fencing with the rules the review added.

+
    +
  • Uploading: the driver may upload batches in parallel, resubmits each until Radius acknowledges it, and trims a +batch from its outbox only after its acknowledgement (S5).
    • +
  • Applying: every batch carries from and to. A replica applies a batch only when its from equals the +replica's applied head, as a whole, in one backend transaction together with the new applied head. A later batch +waits for a missing one. A crash leaves all of a batch or none of it.
    • +
  • No compaction at first (W9): batches carry the ordered operations exactly as committed. "Keep the last write per +row" would lose rows built from put plus partial updates; a later compaction must merge those and treat bulk +operations as barriers.
    • +
  • After a power loss the driver's local head may be behind what Radius has; it first pulls back its own uploaded +batches, then continues (S3).
    • +
  • Blobs go up before the batch that may name them: every blob file present and not yet uploaded (chapter VIII).
    • +
+

Losing the driver role (S7)

+
laptopdriver, epoch 3Radiusdesktoptake over S1epoch 4commits 40, 41 locally (a routine, a late tool result)upload 40–41 (epoch 3)refused: epoch 4the copy becomes session S1′ (new id)S1 continues on the desktop; S1′ on the laptop; they never sync again
Figure 4. The laptop learns the desktop took over. Nothing is rolled back.
+

Rolling back is not possible: tools ran, files were written, natives and derived data followed the tail. The laptop's +copy becomes a separate session with a new id and keeps running; commits in flight land in it. The catalog shows both, +the split-off copy marked with its origin.

+

Export and import (S4)

+

An export holds the commit queue while it reads, so the archive is the state at one head: Durable's records, Chord's +documents and every plugin table, including tables of plugins that are not installed, plus the session directory's +blobs (chapter VIII). An import restores the archive into a fresh store; the server then mirrors the session into its catalog. +Derived data (full-text indexes, caches) is never exported; it is rebuilt where it is needed.

+
+ +
X

Rules for Plugin Authors

What the store does not do for you.

+

Nothing leaves a transaction before it commits (S2). Ids are handed out inside the transaction that creates them. +Reference what is committed: the file-history plugin references the tool-call entry, which is committed before the tool +runs, never the tool-result entry, which does not exist yet.

+

Work that finishes out of order is yours (S8). A routine that starts a slow job per change can let an old result +overwrite a newer one. The handler calls of one routine never overlap, so either chain through the routine or drop +stale results inside the destination's transaction:

+
self.routines.on(conversation.entries, async (entries, ops, ctx) => {
+  const run = await summarize.start({ upTo: entries.at(-1)!.id }, ctx);
+  await run.outcome(ctx);                                  // the next handler waits for this run
+});
+

Text compares by code point. Do not sort keys yourself to page through a table; use the store's cursors, or +compareText. Normalize text yourself if two spellings must be one key.

+

Bulk changes are operations. Prefer updateWhere and deleteWhere over loops of puts; a loop records every row.

+
+ +
∴

What's Next

What is decided, and the order in which it gets built.

+

The direction and all seventeen review items are decided (decision log, 2026-10-21). The store is new code; the parts +outside it are small changes in pi:

+
    +
  • pi, one change: Storage.commit(writes, adopt, context) with every storage calling adopt (S1), the +Session's handling of rejections after adoption, the narrowed mintId comment (S10), traceIn(options, fn, context) (context last), Durable's WatchHandle/WatchEnd moved into Chord with generic end reasons and explicit +retirement (W2), and Durable's state getters and the replicatedState family removed (W14).
    • +
  • The store in packages/chord/src/next: commit step and outbox, tables and the query language, the SQLite and +memory backends with the text encoding, the documents engine, bulk operations and migrations, export and import; +about 1,650 lines by Fable's estimate, plus the conformance suite.
    • +
  • StoreStorage, passing Durable's existing storage conformance suite.
    • +
  • The sync layer (upload, apply, pull-back, epochs, splitting a session) on top, when Radius sync is built.
    • +
+

Open, to settle while building: how a split-off session gets its new id (store metadata, directory, catalog), the +chunk size of row migrations, and a Postgres backend once a serverless target needs one (W14).

+
+ +
A

Decisions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Decision
The storeone transactional store per session under Durable (adapter), Chord documents and plugin tables
S1adoption inside the commit step; Storage.commit(writes, adopt, context)
S2nothing leaves a transaction before it commits; no mechanism
S3a commit is done on the backend's terminal success; ambiguous outcomes stop the store; pull-back after restart
S4export holds the commit queue; multi-query reads in one read transaction; no global snapshots
S5parallel uploads, resubmit until acknowledged, trim after acknowledgement; apply whole batches in order
S6(superseded by W9)
S7losing the driver role splits the session; no rollback
S8out-of-order work is an app concern; routine handlers never overlap
S9the head is persisted with every commit
S10id counters derived at open; mintId contract narrowed
S11fork copies reconstructed by StoreStorage before the commit step
S12the query language starts without joins, grouping and projections
S13one documents engine in the store, opaque owner
S14one transaction API, self.tx(fn, ctx)
S15Durable's existing storages are kept
S16, textreal tables; text as plain UTF-8 in code point order (replaced W8's UTF-16 BLOBs)
S17, W11five writes, bulk operations recorded with a row count; JSON columns migrate on access with their own version; eager steps by scan with row v
W1adopt and mark in the commit step, one synchronous flush right after (derives, wire, watches)
W6, W7workers connect before loading; store addresses carry the host kind, never an instance
W9, W10no compaction at first; no tx.get on a table after a bulk operation in the same transaction
W14no IndexedDB backend (SQLite in WebAssembly in the browser); no replicatedState
index definitionsrecorded in the outbox and applied by every replica; contents never travel
blobs, derived datablobs are app-layer files named by hash, written before what names them, uploaded before each batch, deleted only with the session; derived data is rebuilt, never shipped
+
+ +
B

Glossary

+

Adoption: an owner updating its in-memory state to a committed write. Blob: a file in the session directory +named by the hash of its content, never changed. Derived data: data recomputed from the store and blobs, never +shipped. Backend: what the store runs on (SQLite, +memory). Batch: a range of outbox records from from to to, uploaded and applied as a whole. +Commit step: the short, serialized part of a commit in which the store checks, writes, numbers and records it, and +owners adopt. Driver: the machine that writes a session. Line: the serialization of one owner's transactions +(per Chord object, per Durable Session). Outbox: the record of every commit, written in the same transaction. +Owner: the code that writes a piece of data. Replica: a copy of a session that applies the driver's batches. +Seq: the number of a commit, one sequence per store. Store: the transactional store of one session. +Table: a set of rows owned by one object, with a flat record of typed columns (scalars or json(...)).

+
+ +

One Store · first edition · 2026-10-21 · based on the redesign decision log and the store review

+
+ + diff --git a/packages/chord/docs/next/store/walkthrough.md b/packages/chord/docs/next/store/walkthrough.md new file mode 100644 index 000000000..c682fe3e3 --- /dev/null +++ b/packages/chord/docs/next/store/walkthrough.md @@ -0,0 +1,193 @@ +--- +title: One Store +kind: walkthrough +kicker: A walkthrough · Chord v2 and p3 · 2026-10-21 +subtitle: How one transactional store keeps a p3 session's transcript, documents and plugin tables consistent, taught one concept at a time +model: anthropic/claude-sonnet-5-5 +sources: + - packages/chord/docs/next/store/book.md + - packages/chord/docs/next/decisions.md + - apps/p2/docs/p3-reviews/store-review-summary.md + - apps/p2/docs/p3-reviews/store-bulk-migrations.md + - apps/p2/docs/p3-reviews/session-data-scenarios.md + - apps/p2/docs/p3-reviews/store-proposal.md + - packages/durable/src/storage/sqlite/storage.ts + - packages/durable/src/session/session.ts +truth: The decision log (redesign-decisions.md, entries "The store", "S1" to "S17", "Context last") wins over everything; the book (book.md) is written from it; store-proposal.md and session-data-scenarios.md are earlier drafts and the discussion, superseded where the log differs; the Durable sources show today's code, before the S1 change. +--- + +:::tutor +Audience: the owner of p3 and Chord, who decided this design in one long discussion and wants to understand it end to +end, or a developer joining the work who knows TypeScript, SQLite and pi-durable but not this store. Use the book's +running example throughout: Mario's session S1 on his laptop, the file-history plugin with its `edits` table, entry +812, the rename of `a/` to `b/` over 10,000 rows, continuing on the desktop. Read the matching book chapter before each +concept and check decisions in the log. Be concrete: code, short traces with seq numbers, small tables. Short messages; +one figure per concept at most. When the reader asks why something was not done another way, answer from the book's +"What We Tried and Dropped" chapter and the log. +::: + +:::intro +#### What this is about + +A p3 session must be self-contained: everything it owns lives in its own store and directory, so it can be exported, +imported into another p3, synced to Radius and continued on another device. Three kinds of owners write into a +session: Durable (the transcript), Chord objects (persisted documents) and plugins (their own tables). This walkthrough +explains the store they all share and why it is built that way. + +#### The example + +Mario works in session S1 on his laptop. A plugin called **file-history** records every file edit in its table +`edits`: path, the conversation entry that made it (say entry 812), the time. He renames the directory `a/` to `b/` +(10,000 rows change). In the evening he continues S1 on his desktop; another day he exports it and a colleague imports +it. Every concept follows this session through one mechanism. + +#### How it progresses + +Eleven concepts in four parts: the problem and what was dropped; how the store writes (commit queue, seq, adoption, +completion); how it reads and stores documents and tables (text order, the documents engine, bulk operations and +migrations; blobs and derived data); and how a session moves (sync, takeover, export) plus the rules left to plugin authors. +::: + +# The problem {#problem part="Why one store"} +Book "Before we begin" and chapter I. Must cover: P1 (a session is self-contained; the server only mirrors); three +owners (Durable, Chord objects, plugins); the trace where file-history's own `history.sqlite` and Durable's log sync +separately and entry 812 dangles or points to the wrong entry after takeover; what is missing is one order across +storages. A short trace or a steps figure helps. + +:::check +1. Both of S1's storages are consistent on their own. Why can the desktop still end up in a state the laptop never had? ++++ answers +1. Because nothing orders the changes of one storage relative to the other: the desktop can receive history.sqlite up to the edit but Durable's log only up to the commit before it (or the reverse), a combination that never existed on the laptop. +::: + +# What was dropped {#dropped} +Book chapter II; log entry "The store" (its list of rejected options) and S16. Must cover the main dropped options and +the reason for each: derived-only plugin data, opaque directories, export hooks, plugin record logs with apply, several +storages with a shared position counter, recorded SQLite statements, raw SQL reads, an IndexedDB-only API; inside the +store: generic JSON rows, store-maintained index entries, code point order with replaced surrogates, text equality only. +Point out what the final design kept from them (one order, recorded operations, a small query language). + +:::check +1. Why was "record the plugin's SQL statements and replay them on the replica" dropped, while bulk operations are now recorded as operations and replayed? ++++ answers +1. Arbitrary SQL can produce different results on a replica (random(), datetime('now'), unordered INSERT ... SELECT, other SQLite versions); the store's bulk operations use only the store's own expressions, specified once and identical on every backend, plus a row count check. +::: + +# The store and its tables {#store part="Writing"} +Book chapter III; log "The store", S14, S16. Must cover: layers (owners, store with documents engine, tables and commit +step, backends); a table belongs to one object (address host, object type, table name); `defineTable` with typed key +and indexed columns (`str` or `int`) in a flat `columns` record of real columns, nesting only inside `json()` columns (typed but not queryable); one transaction API +`self.tx(fn, ctx)`; index definitions are recorded in the outbox (contents never travel); Durable keeps its storages and gains `StoreStorage`; context last. + +:::check +1. In `edits`, which fields can a query filter on, and where does the rest of the row live? ++++ answers +1. Every scalar column (str, int, bool, num) is a real column and can be filtered; keys and indexes need str or int; json(...) columns hold JSON text, typed when read but not queryable or indexable. +::: + +# Lines, the commit queue and the seq {#commit} +Book chapter IV; log S1, S3, S9, S10. Must cover: lines (per Chord object, per Durable Session) serialize whole +transactions, the commit queue serializes only the short final step; one seq per store, Durable sees gaps and only +compares seqs; adoption inside the step with `Storage.commit(writes, adopt, context)`; why (the trace where Durable +adopts 41 before file-history adopts 40); listeners still run afterwards and read a complete prefix; a commit is done +only on the backend's terminal success; ambiguous outcomes stop the store; ids derived at open. The book's sequence +figure of two objects and a Durable turn helps. + +:::check +1. Object A commits seq 40 and Durable commits 41. Without S1, how could a routine see 41 but not 40? +2. Why is it harmless that Durable's seqs now have gaps? ++++ answers +1. Each owner adopted in its own continuation after the disk commit; Durable's continuation ran first, so a routine triggered by 41 read A's in-memory state still at 39. With S1 both adoptions run inside the commit step, in seq order. +2. Durable only compares seqs (created_at <= at, seq > base); it never computes seq + 1 to find a neighbouring commit. +::: + +# Reading and text order {#reading part="Reading and storing"} +Book chapter V; log S4, S12, S16. Must cover: the small query language (where operators, orderBy, limit, cursor; no +joins, grouping, projections at first); JavaScript and SQLite order strings differently (the ～ vs 😀 example); the +decision: JavaScript order everywhere, SQLite stores key and indexed text as UTF-16 BLOBs (big-endian), lossless, +measured costs; no normalization; cursors carry direction and logical values; multi-query reads in one read +transaction, no global snapshot. + +:::check +1. A plugin pages through `edits` by path with a cursor. Why would it break if SQLite stored paths as ordinary TEXT while the memory backend compared JS strings? +2. What does the UTF-16 BLOB encoding cost, according to the measurements? ++++ answers +1. SQLite would order by UTF-8 bytes and JS by UTF-16 code units; they disagree for characters between U+E000 and U+FFFF versus emoji, so a cursor taken from one order skips or repeats rows in the other (and tests on the memory backend would not show what SQLite does). +2. Mostly JS encoding and decoding plus twice the bytes for ASCII: about +40% on bulk inserts, +62% file size in the test; SQLite's own range scans are about 4% slower when values are not decoded from the BLOBs. +::: + +# The documents engine {#documents} +Book chapter VI; log S11, S13. Must cover: Durable and Chord persist documents the same way, so the store has one +engine; address (namespace, owner, kind, key) with an opaque owner string filled by each adapter (Durable: session, +conversation:7, task:31; Chord: host/object type); find, load, list outside the step, create, change, retire inside it; +load reconstructs base plus deltas once, the engine keeps nothing in memory; Durable's Session caches and evicts as +today; fork copies are reconstructed by StoreStorage before the commit step. + +:::check +1. Who decides that a conversation's documents fork and rewind with it: the documents engine or Durable's adapter? ++++ answers +1. Durable's adapter (StoreStorage): the engine only sees an opaque owner string such as "conversation:7"; the meaning of that owner stays with Durable. +::: + +# Bulk operations and migrations {#bulk} +Book chapter VII; log S17; store-bulk-migrations.md. Must cover: the five writes (put, update, delete, updateWhere, +deleteWhere); the rename of a/ to b/ as one recorded operation with a row count (about 150 bytes instead of about 40 +MB), compiled to one SQL UPDATE on the UTF-16 BLOB column; the replica runs the same operation and compares the count; +the set expressions and why they are limited (identical behaviour on every backend; no JSON operands yet); migrations: +changes inside json(...) columns on access via migrate(row, from), column changes as recorded alters, key and indexed columns eagerly by recorded table changes plus an update +step or a rows step (scan + tx.update in chunks, migrating then ready, resume after a crash). The book's migration steps +figure helps. + +:::check +1. Version 4 of `edits` adds an indexed column `dir` computed with dirname(path). Why can this not migrate on access, and how does it run instead? +2. A replica applies the rename and affects 9,998 rows instead of 10,000. What happens? ++++ answers +1. An index over a column filled only on access would silently miss unmigrated rows; so it runs eagerly on the driver before the object goes live: a recorded alter adds the column, then a rows step scans in key order and writes dir with tx.update, chunk by chunk, until the table is marked ready. +2. The counts differ, so the replica stops applying that store and restores it from a snapshot: divergence is detected, not silent. +::: + +# Blobs and derived data {#easy} +Book chapter VIII "The Easy Parts"; log "Blobs and derived data". Must cover: blobs are app-layer files named by the +hash of their content, written before the row or document that names them; export copies them, import checks hashes; +sync uploads every blob present and not yet uploaded before each batch (no reference tracking); no deletion before the +session is deleted; derived data is never shipped and is repaired by a catch-up routine (the artifacts full-text index +example: normal save, crash after commit, import). + +:::check +1. Why does the sync layer not need to know which rows reference which blobs? +2. The process dies between the artifact document's commit and the index update. What fixes the index? ++++ answers +1. A blob is always written before the commit that names it, so uploading every blob present before each batch guarantees that any hash a batch names has its file on Radius already. +2. The catch-up routine on the next start: it compares the index with the documents, finds version 3 missing and indexes it from the blob. +::: + +# Sync and takeover {#sync part="Moving a session"} +Book chapter IX; log S3, S5, S6, S7. Must cover: the outbox record is written in the same backend transaction; +parallel uploads, resubmit until acknowledged, trim after acknowledgement; a replica applies whole batches in order +(from equals its applied head), later batches wait; compaction is safe because batches apply whole; pull-back after a +power loss; losing the driver role splits the session (new id) without rollback, and why rollback is impossible. + +:::check +1. The laptop committed 40 and 41 locally after the desktop took over. What happens to those commits? ++++ answers +1. Nothing is rolled back: when Radius refuses the laptop's upload for an old epoch, the laptop's copy becomes a separate session with a new id, keeps 40 and 41, and never syncs with S1 again; S1 continues on the desktop. +::: + +# Export, import and the rules left to plugins {#export} +Book chapters IX ("Export and import") and X; log S2, S4, S8. Must cover: export holds the commit queue so the +archive is one head, includes tables of plugins that are not installed, never derived data; import restores into a +fresh store and the server mirrors the catalog; rules: nothing leaves a transaction before it commits (reference the +tool-call entry, not the tool-result entry); work that finishes out of order is the app's concern (routine handlers +never overlap: chain or drop stale results); text compares like JS strings; prefer updateWhere over loops of puts. + +:::check +1. The file-history plugin wants to store which entry made an edit. Which entry can it safely reference, and why? ++++ answers +1. The assistant entry with the tool call: it is committed before the tool runs. The tool-result entry does not exist yet while the tool runs, and its id is handed out only inside the harness's later transaction. +::: + +# What comes next {#next part="Where it stands"} +Book "What's Next"; log entries. Must cover: everything decided on 2026-10-21; the pi change (adopt argument, Session +rejection handling, mintId wording, traceIn context last); the store in packages/chord/src/next (about 1,650 lines +estimated); StoreStorage passing Durable's conformance suite; the sync layer later; open points (new id of a split +session, chunk size, whether IndexedDB is needed while sqlite-wasm covers the browser). diff --git a/packages/chord/docs/spec.md b/packages/chord/docs/spec.md index ca9c93b38..4c261cb54 100644 --- a/packages/chord/docs/spec.md +++ b/packages/chord/docs/spec.md @@ -3,7 +3,7 @@ This document specifies the runtime in `src`. Where it and the code disagree, the disagreement is a bug. [Known limitations and decisions](#12-known-limitations-and-decisions) lists what the runtime does not do yet and the choices the sections below leave open. Hard constraints: -- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins plus `typebox` for schemas. No other Pi packages. +- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins, plus `@earendil-works/pi-telemetry` for telemetry and `@earendil-works/pi-schema` for schemas. No other Pi packages. - Built-in plugins are trusted, loaded from source in-process. The plugin loader uses dynamic `import()` with a cache-busting query (owner-approved exception, `TODO(loader)`: replace with an isolating, evicting loader; old module copies leak; TUI plugins need isolation). - Hot reload is required. Host restart only for Chord/runtime code changes or host-level crash recovery. @@ -29,23 +29,23 @@ Contracts are runtime values for every type (shipped bundles have no TypeScript; ```ts export const Chat = remote("pi.chat", { version: 1, // contract version: a plain number, bumped only on breaking changes - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // where its objects live: `session:` + input.session variants: { viewer: { state: { view: ViewSchema } }, participant: { state: { view: ViewSchema }, - methods: { send: method(SendSchema, Type.Object({ submissionId: Type.Number() })) }, + methods: { send: method(SendSchema, obj({ submissionId: num() })) }, }, }, }); export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(Hit)), - onChange: method(Type.Function([Change], Type.Void()), Unsubscribe), // non-JSON allowed for locals + search: method(obj({ query: str() }), arr(Hit)), + onChange: method({ type: "function", parameters: [Change], returnType: { type: "void" } }, Unsubscribe), // non-JSON allowed for locals harness: opaque("pi.durable-harness@1"), // native object, by name+version }, }); @@ -81,7 +81,7 @@ Factories are synchronous; returning a promise throws. A declaration has a name host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }, // schema required, initial value { version: 2, migrate: (old, from) => upgrade(old, from) }); // per cell; default version 1 - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); // reference; identity = canonical address (+ at) const watch = self.resource("watch", { version: 1, deps: [catalog], exclusive: false, @@ -278,8 +278,8 @@ The tests cover at least: lifetime/references/linger/forced disposal/gone; activ - **Joining during a reload.** Participants are the child hosts in step when a reload starts. A child whose hello arrives while it runs is decided when it ends: in step, or reconciled to the committed code version; a child already behind is fenced at once, so the calls its roots make meanwhile are admitted in the parent's code version. A reconciliation that is running when a reload starts syncs its child only after that reload ended, against the code version it committed. A reconciliation announces the parent's retired code versions before its `prepare`. - **Two-version rule.** An object of a changed type created after a commit has only new code; a previous-code-version root calling it gets `superseded`. - **Validation.** Remote calls are validated at reference dispatch, in-host too: a direct binding against the provider's schema, a binding through an adapter against the consumer's. A provider validates each snapshot and reset of a remote state member once per (re)subscription, against the consumer's contract through an `accepts` adapter and the provider's otherwise; ops are validated only in dev mode. At reload, the new provider's description must cover the old provider's when the version is unchanged, in addition to the check against live consumers and facets' `uses`. -- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; with TypeBox, use `Type.Optional(Type.X({ default }))` to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. -- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. +- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; use `opt(str({ default }))` (pi-schema) to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. +- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. The host `resolve` returns is admitted before that second check, so a refused, revoked or released relay never leaves a spawned host's link unread. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. - **Cells and readables.** A value returned from `change` replaces the value and is always validated; `replace` transfers ownership without copying; by default (`{ diff }` overrides) it publishes a diff when the new value reuses a container of the old one at the same key, else `r`. `subscribe(fn)` delivers values, starting with the current one; `events(fn)` delivers `snap`, then `ops`, or `reset`. Reference state members are stable readables that throw `not_ready` while loading and `not_in_variant` outside the variant, and reset when their source moves; a listener added before they are readable gets its snapshot then. Remote state members stay hydrated while the reference is held. - **References and calls.** A reference's identity is its address; a second `self.get` of the same address in one run returns the same reference. A reference call without ctx, or with a task ctx, starts a fresh root in the current code version; a ctx whose call or root already settled is refused (`aborted/expired`), one from another host `invalid/foreign_ctx`. Calling a reference inside a factory throws `not_ready`. - **Resources and disposal.** New and replaced exclusive resources open at the gate, and so do non-exclusive resources of the same object that depend on them. A new object for an address that is still being disposed waits for that disposal. A resource open that completes after its signal aborted closes its value at once. diff --git a/packages/chord/docs/tutorial.md b/packages/chord/docs/tutorial.md index fe16dce4c..65a6334ed 100644 --- a/packages/chord/docs/tutorial.md +++ b/packages/chord/docs/tutorial.md @@ -45,11 +45,12 @@ shuts down its objects and connections. Before callers can use an object, they need to agree on its name and members. That agreement is a **contract**: a runtime value that also gives TypeScript its -types. A **schema** describes a value's shape. We use TypeBox's `Type` to write -schemas. `method(args, result)` declares a method by its argument and result -schemas: `method()` takes no arguments and returns nothing, and -`method(undefined, Type.String())`, used below, takes no arguments and returns a -string. +types. A **schema** describes a value's shape: plain JSON Schema, which we write +with the builders of `@earendil-works/pi-schema` (`obj`, `str`, `num`, ...; +`{ open: true }` lets an object carry fields it does not name). +`method(args, result)` declares a method by its argument and result schemas: +`method()` takes no arguments and returns nothing, and +`method(undefined, str())`, used below, takes no arguments and returns a string. Start with a label for a named list. `local` means only code inside this host can reach it. The input `{ list: "family" }` identifies which list we want; @@ -59,8 +60,8 @@ it is not an argument to `label`. ```ts const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); ``` @@ -116,17 +117,17 @@ name, because it holds nothing a reload must keep. ```ts -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -230,13 +231,13 @@ to a declared resource, without reading the resource during the factory. ```ts const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -338,13 +339,13 @@ puts every `Todos` object on the host named `server`. Section 6 uses that. ```ts const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -499,8 +500,8 @@ test; they are not stored application state. ```ts const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -509,7 +510,7 @@ const seeded = Promise.withResolvers(); const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -570,7 +571,7 @@ default lifetime, `"referenced"`, are still created only when referenced. ```ts const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); -const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); +const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -745,7 +746,7 @@ const { label: _label, ...members } = TodoStore.def.members; const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { @@ -890,7 +891,7 @@ tutorial's source-matching check, not by the application. ```ts import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, diff --git a/packages/chord/package.json b/packages/chord/package.json index 57dbb5406..6e8e12a4d 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "description": "Typed host runtime for plugins, contracts, replicated state, and RPC over JSON links", "type": "module", "main": "./dist/index.js", @@ -36,6 +36,11 @@ "types": "./dist/telemetry-sqlite/index.d.ts", "import": "./dist/telemetry-sqlite/index.js" }, + "./next": { + "source": "./src/next/index.ts", + "types": "./dist/next/index.d.ts", + "import": "./dist/next/index.js" + }, "./sqlite": { "source": "./src/sqlite/index.ts", "types": "./dist/sqlite/index.d.ts", @@ -62,7 +67,7 @@ "src/delta/README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -85,12 +90,23 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" + }, + "peerDependencies": { + "esbuild": "0.28.2", + "ws": "8.21.0" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + }, + "ws": { + "optional": true + } }, "devDependencies": { "@types/ws": "8.18.2", - "shx": "0.3.4", "vitest": "4.1.12", "ws": "8.21.0" } diff --git a/packages/chord/src/cell.ts b/packages/chord/src/cell.ts index 65d391128..d520258e3 100644 --- a/packages/chord/src/cell.ts +++ b/packages/chord/src/cell.ts @@ -1,4 +1,4 @@ -import type { TSchema } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; import { type Draft, diffRevisions, type JsonValue, type Op, type Tracker, track } from "./delta/index.ts"; import { ChordError } from "./errors.ts"; import { enqueue, holdPublications } from "./publish.ts"; @@ -87,7 +87,7 @@ export abstract class Source implements Readable { readonly seq = ++sequence; readonly dependents = new Set>(); /** Schemas of remote state members currently exposing this source (checked per write in dev mode). */ - schemas: readonly TSchema[] = []; + schemas: readonly Schema[] = []; protected readonly options: SourceOptions; protected readonly fanout: Fanout; @@ -114,7 +114,7 @@ export abstract class Source implements Readable { for (const schema of this.schemas) assertValid("state", `state ${this.key}`, schema, value); } - protected ownSchemas(): readonly TSchema[] { + protected ownSchemas(): readonly Schema[] { return []; } } @@ -149,7 +149,7 @@ function unbox(ops: readonly Op[], value: unknown): Op[] { /** A state cell holding any JSON value, validated against its schema at creation and (dev mode) on writes. */ export class Cell extends Source { /** Replaced at a reload commit that changes the schema (state-version bump). */ - schema: TSchema; + schema: Schema; #tracker: Tracker>; readonly #guard: WriteGuard; /** False until `initialize` for a cell declared with `init`: reads and writes throw `not_ready`. */ @@ -161,7 +161,7 @@ export class Cell extends Source { */ constructor( key: string, - schema: TSchema, + schema: Schema, initial: { readonly value: T } | undefined, guard: WriteGuard, options: SourceOptions, @@ -182,7 +182,7 @@ export class Cell extends Source { this.#initialized = true; } - protected override ownSchemas(): readonly TSchema[] { + protected override ownSchemas(): readonly Schema[] { return [this.schema]; } @@ -238,7 +238,7 @@ export class Cell extends Source { * Installs a migrated value and its schema (reload commit). The caller validated the value and the derivations * over it against every schema that will apply, so nothing is checked here. Subscribers receive a reset. */ - reset(value: T, schema: TSchema): void { + reset(value: T, schema: Schema): void { this.schema = schema; this.#publish(this.#tracker.prepareReplace({ v: value }), "reset", false); } diff --git a/packages/chord/src/contract.ts b/packages/chord/src/contract.ts index c2ebd6b96..1a6d64031 100644 --- a/packages/chord/src/contract.ts +++ b/packages/chord/src/contract.ts @@ -1,15 +1,14 @@ -import { type Static, type TSchema, Type } from "typebox"; -import { Value } from "typebox/value"; +import { int, lit, map, obj, opt, type Schema, type Static, str, union } from "@earendil-works/pi-schema"; import { ChordError, invalid } from "./errors.ts"; -import { canonicalJson, fits, hash64 } from "./schema.ts"; +import { canonicalJson, fits, hash64, isValid } from "./schema.ts"; import type { Ctx, Exposed, Readable, RefBase, StateSource } from "./types.ts"; const MEMBER = Symbol("chord.member"); /** A method's arguments and result; `undefined`: it takes no arguments, or returns nothing (void). */ export interface MethodSpec< - A extends TSchema | undefined = TSchema | undefined, - R extends TSchema | undefined = TSchema | undefined, + A extends Schema | undefined = Schema | undefined, + R extends Schema | undefined = Schema | undefined, > { readonly [MEMBER]: "method"; readonly args: A; @@ -29,9 +28,9 @@ export interface OpaqueSpec { * for a method without arguments that returns something. */ export function method(): MethodSpec; -export function method(args: A): MethodSpec; -export function method(args: A, result: R): MethodSpec; -export function method(args?: TSchema, result?: TSchema): MethodSpec { +export function method(args: A): MethodSpec; +export function method(args: A, result: R): MethodSpec; +export function method(args?: Schema, result?: Schema): MethodSpec { return { [MEMBER]: "method", args, result }; } @@ -42,7 +41,7 @@ export const opaque = (id: string): OpaqueSpec => { }; export interface VariantDef { - readonly state?: Readonly>; + readonly state?: Readonly>; readonly methods?: Readonly>; } @@ -102,63 +101,59 @@ export const placementText = (placement: Placement): string => export interface RemoteDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; /** Where its objects live: `hostNamed(…)` or `hostFamily(…)`. Part of the contract (and its fingerprint). */ readonly host: Placement; readonly variants: Readonly>; } /** A local member: a method, an opaque native object, or a state member described by its schema. */ -export type LocalMember = MethodSpec | OpaqueSpec | TSchema; +export type LocalMember = MethodSpec | OpaqueSpec | Schema; export interface LocalDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly members?: Readonly>; } // A description travels as a reference's contract token (wire.ts), so it is declared once, as the schema that validates it. -// `TSchema` is an empty interface: any object is one, which is all a peer's schema is checked for here. -const SchemaObject = Type.Unsafe(Type.Object({})); +// Any object is a schema, which is all a peer's schema is checked for here. +const SchemaObject = obj({}, { open: true }) as Schema; -const MemberDescriptionSchema = Type.Union([ +const MemberDescriptionSchema = union([ // Absent `args`: the method takes none. Absent `result`: it returns nothing. - Type.Object({ - kind: Type.Literal("method"), - args: Type.Optional(SchemaObject), - result: Type.Optional(SchemaObject), - }), - Type.Object({ kind: Type.Literal("state"), schema: SchemaObject }), - Type.Object({ kind: Type.Literal("opaque"), id: Type.String() }), + obj({ kind: lit("method"), args: opt(SchemaObject), result: opt(SchemaObject) }, { open: true }), + obj({ kind: lit("state"), schema: SchemaObject }, { open: true }), + obj({ kind: lit("opaque"), id: str() }, { open: true }), ]); export type MemberDescription = Static; /** Local contracts have one unnamed variant. */ export const LOCAL_VARIANT = ""; -const PlacementSchema = Type.Union([ - Type.Object({ name: Type.String({ minLength: 1 }) }, { additionalProperties: false }), - Type.Object( - { prefix: Type.String({ minLength: 1 }), key: Type.String({ minLength: 1 }) }, - { additionalProperties: false }, - ), +const PlacementSchema = union([ + obj({ name: str({ minLength: 1 }) }), + obj({ prefix: str({ minLength: 1 }), key: str({ minLength: 1 }) }), ]); -export const DescriptionSchema = Type.Object({ - name: Type.String({ minLength: 1 }), - scope: Type.Union([Type.Literal("remote"), Type.Literal("local")]), - version: Type.Integer(), - input: SchemaObject, - /** Remote types only (required there): where their objects live. */ - host: Type.Optional(PlacementSchema), - variants: Type.Record(Type.String(), Type.Record(Type.String(), MemberDescriptionSchema)), -}); +export const DescriptionSchema = obj( + { + name: str({ minLength: 1 }), + scope: union([lit("remote"), lit("local")]), + version: int(), + input: SchemaObject, + /** Remote types only (required there): where their objects live. */ + host: opt(PlacementSchema), + variants: map(map(MemberDescriptionSchema)), + }, + { open: true }, +); export type Description = Static; interface ContractBase { readonly name: string; readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly description: Description; readonly fingerprint: string; } @@ -204,7 +199,7 @@ function describeMember(name: string, member: unknown, scope: "remote" | "local" if (scope === "remote") throw new TypeError(`remote contracts cannot expose opaque members: ${name}`); return { kind: "opaque", id: (member as OpaqueSpec).id }; } - return { kind: "state", schema: member as TSchema }; + return { kind: "state", schema: member as Schema }; } function finish(contract: Omit): C { @@ -222,7 +217,7 @@ export function remote( ): RemoteContract { checkHeader(name, def.version); const placement = def.host as Placement | undefined; - if (placement === undefined || !Value.Check(PlacementSchema, placement)) + if (placement === undefined || !isValid(PlacementSchema, placement)) throw new TypeError(`remote contract ${name} needs \`host\`: hostNamed(…) or hostFamily(…)`); // A JSON copy: the description travels, and a placement object may carry nothing else. const host: Placement = @@ -353,23 +348,23 @@ export function bind(provider: AnyContract, accepts: Accepts | undefined, consum // ─── Static types ──────────────────────────────────────────────────────────── type Empty = Record; -type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; -type StaticOf = S extends TSchema ? Static : never; +type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; +type StaticOf = S extends Schema ? Static : never; type MethodsOf = V extends { readonly methods: infer M extends Readonly> } ? M : Empty; /** What a handler returns: the result's value, or nothing for a void method. */ -type Returned = [R] extends [TSchema] ? Static | Promise> : void | Promise; +type Returned = [R] extends [Schema] ? Static | Promise> : void | Promise; /** What a call resolves to. */ -type Resolved = [R] extends [TSchema] ? Static : undefined; +type Resolved = [R] extends [Schema] ? Static : undefined; /** A method's handler; `P` is the app's principal type (`ctx.principal`). */ export type MethodImplementation = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx: Ctx

) => Returned : (ctx: Ctx

) => Returned : never; export type Call = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx?: Ctx) => Promise> : (ctx?: Ctx) => Promise> : never; @@ -402,21 +397,21 @@ type LocalImplementationMember = M extends MethodSpec ? MethodImplementation : M extends OpaqueSpec ? T | Exposed - : M extends TSchema + : M extends Schema ? StateSource> : never; type LocalRefMember = M extends MethodSpec ? Call : M extends OpaqueSpec ? T | undefined - : M extends TSchema + : M extends Schema ? Readable> : never; type LocalMembers = D extends { readonly members: infer M extends Readonly> } ? M : Empty; -export type InputOf = C extends { readonly def: { readonly input: infer I extends TSchema } } ? Static : never; +export type InputOf = C extends { readonly def: { readonly input: infer I extends Schema } } ? Static : never; /** What a factory returns for contract C; its methods' `ctx.principal` is `P`. */ export type ImplementationOf = C extends RemoteContract diff --git a/packages/chord/src/delta/README.md b/packages/chord/src/delta/README.md index ecc6cefd6..8dbf06f75 100644 --- a/packages/chord/src/delta/README.md +++ b/packages/chord/src/delta/README.md @@ -154,6 +154,8 @@ Growing `length` inserts `null`; shrinking it removes elements. `tracker.value` is the latest adopted revision. `beginChange()` opens an overlay draft over it and never modifies it. A draft may stay open across `await`. +A draft dropped without `prepare()` or `abort()` stays referenced by its +tracker until the tracker's next `beginChange()` or `adopt()`. `prepare()` materializes the candidate and ops and does not change authority. `adopt()` checks the preparation and swaps the root pointer. @@ -210,9 +212,47 @@ and restores `Op` tuples. Use one encoder/decoder pair per ordered state stream. Path IDs span batches and an `r` resets both dictionaries. After a decode or apply error, discard the decoder and replica and recover from a later `r`. +## Growing strings: use `appendText` + +The footgun: appending to a string through a draft costs the string's whole length on every change, not the length of +what you append. + +```ts +const change = tracker.beginChange(); +change.state.message.text += chunk; // looks like O(chunk) +const prepared = change.prepare(); // emits ["a", ["message", "text"], chunk], but costs O(text.length) +``` + +The draft's proxy only sees the finished new string, never the chunk, so `prepare()` recovers the append by checking +that the new string starts with the old one. That check reads the whole string, and V8 first copies a string built by +`+=` (a rope) into one flat piece. There is no fast path: measured on Node 24 and 26, 20,000 changes appending 40 +characters each, up to 800 KB, take about 2.4 s with the check and about 1 ms without it. A stream of N chunks into one +string therefore costs O(N x length) in total: quadratic. + +The fix: hand Delta the chunk. + +```ts +import { appendText } from "@earendil-works/chord/delta"; + +appendText(change.state.message, "text", chunk); // records ["a", path, chunk]: O(chunk), no compare, no copy +``` + +- `appendText(parent, key, text)` takes the parent draft and a key whose type is exactly `string` (not optional, not a + string literal type), so the compiler refuses `appendText(block, "type", …)`. +- Several appends to one key in one change become one op. A plain assignment or deletion of the same key in that change + falls back to the compare, as does an append after a plain write. +- The appended text is copied into a string of its own, so neither the op nor the new value keeps a larger string + alive. +- Use it wherever a string grows by appends across many changes (streamed answers, tool output, logs). A one-off + assignment is fine with `=`. +- Derives cannot use it: `diffRevisions()` compares two revisions it did not write, so a derive producing a growing + string pays O(length) per delivery. + ## Limits and footguns - No freezing. Mutating any immutable value corrupts state silently. +- Appending to a string through a draft (`draft.text += chunk`) costs O(string length) per change; use `appendText` + (section "Growing strings"). - In-process sharing: local handles, cell subscribers, and `replicatedState` subscribers receive the provider's containers. A consumer mutation corrupts authority and makes remote replicas diverge. - A change inside a large flat array copies that array's pointer storage for the new revision. - Placement validation costs time and memory on bulk inserts. In a benchmark unshifting 100k three-field objects, it added about 19 ms and 20 MiB of transient heap. diff --git a/packages/chord/src/delta/apply-immutable-trusted.ts b/packages/chord/src/delta/apply-immutable-trusted.ts index 7944bfae1..cde0300e7 100644 --- a/packages/chord/src/delta/apply-immutable-trusted.ts +++ b/packages/chord/src/delta/apply-immutable-trusted.ts @@ -86,12 +86,9 @@ function copyPath(root: JsonValue, path: Path, length: number, owned: WeakSet; - for (const key of Object.keys(value)) defineData(result, key, value[key]!); - return result; + // Spread and `Object.assign()` onto a null-prototype object define own data properties, `__proto__` included, as + // `defineData()` per key does, at a fraction of its cost. + return Object.getPrototypeOf(value) === null ? Object.assign(Object.create(null), value) : { ...value }; } function read(target: JsonContainer, key: Seg): JsonValue { diff --git a/packages/chord/src/delta/index.ts b/packages/chord/src/delta/index.ts index f2ec7ec53..1af25be29 100644 --- a/packages/chord/src/delta/index.ts +++ b/packages/chord/src/delta/index.ts @@ -490,19 +490,9 @@ function copyContainers(root: JsonValue, path: Path, owned: WeakSet): Js const copy = (value: JsonValue): JsonValue[] | Record => { if (Array.isArray(value)) return value.slice(); if (!isObj(value)) throw new PathError(path); - const result = Object.create(Object.getPrototypeOf(value) === null ? null : Object.prototype) as Record< - string, - JsonValue - >; - for (const key of Object.keys(value)) { - Object.defineProperty(result, key, { - value: (value as Record)[key], - writable: true, - enumerable: true, - configurable: true, - }); - } - return result; + // Spread and `Object.assign()` onto a null-prototype object define own data properties, `__proto__` included, + // as `Object.defineProperty()` per key does, at a fraction of its cost. + return Object.getPrototypeOf(value) === null ? Object.assign(Object.create(null), value) : { ...value }; }; if (!isObj(root)) throw new PathError(path); let copiedRoot: JsonValue[] | Record; diff --git a/packages/chord/src/delta/tracker.ts b/packages/chord/src/delta/tracker.ts index db5eb6b66..731cc49a8 100644 --- a/packages/chord/src/delta/tracker.ts +++ b/packages/chord/src/delta/tracker.ts @@ -229,9 +229,20 @@ class ChangeImpl implements Change { } } +/** + * Open drafts are tracked so that adopting one change makes its open competitors stale at once. Prepared contexts need + * no tracking: `prepare()` releases their overlay, and `adopt()` rejects them by revision. + * + * The most recent open context is held strongly in `#open`: with one transaction at a time per tracker, the common case, + * that is the only one, and it costs no `WeakRef` (each `WeakRef` target survives the scavenges of the current job, and + * V8 then pretenures the context allocation site). A second `beginChange()` while one is open moves the older context + * into `#competitors` through a `WeakRef`, so a draft the caller drops can be collected. A dropped draft that was never + * settled stays held by `#open` until the tracker's next `beginChange()` or `adopt()`: at most one per tracker. + */ class TrackerImpl implements Tracker { readonly #owner = {}; - readonly #contexts = new Set>(); + #open: OverlayContext | undefined; + #competitors: Set> | undefined; readonly #selfRef: WeakRef>; #value: T; #revision = 0; @@ -252,14 +263,15 @@ class TrackerImpl implements Tracker { beginChange(): Change { const context = createContext(this.#selfRef, this.#owner, this.#revision, this.#value, false, this.#value); - this.#register(context); + if (this.#open !== undefined) this.#demote(this.#open); + this.#open = context; return new ChangeImpl(context); } prepareReplace(value: T): Prepared { + // Not tracked: materialization below releases the context before this returns, as `prepare()` does. const context = createContext(this.#selfRef, this.#owner, this.#revision, value, true, this.#value); context.status.value = "prepared"; - this.#register(context); try { return materializePrepared(context); } catch (error) { @@ -295,39 +307,37 @@ class TrackerImpl implements Tracker { } releaseContext(context: OverlayContext): void { - if (context.registryRef !== undefined) this.#contexts.delete(context.registryRef); + if (this.#open === context) this.#open = undefined; + else if (context.registryRef !== undefined) this.#competitors?.delete(context.registryRef); context.registryRef = undefined; } - #register(context: OverlayContext): void { + #demote(context: OverlayContext): void { const reference = new WeakRef(context); context.registryRef = reference; - this.#contexts.add(reference); + this.#competitors ??= new Set(); + this.#competitors.add(reference); this.#pruneBudget -= 1; if (this.#pruneBudget === 0) { - this.#prune(); - this.#pruneBudget = Math.max(256, this.#contexts.size); - } - } - - #prune(): void { - for (const reference of this.#contexts) { - if (reference.deref() === undefined) this.#contexts.delete(reference); + for (const entry of this.#competitors) { + if (entry.deref() === undefined) this.#competitors.delete(entry); + } + this.#pruneBudget = Math.max(256, this.#competitors.size); } } #invalidate(winner: OverlayContext): void { - for (const reference of this.#contexts) { - const context = reference.deref(); - if (context === undefined || context === winner) continue; - if (context.status.value === "open" || context.status.value === "prepared") context.status.value = "stale"; - // Adoption must not walk every node touched by a competing draft. Drop the - // context's strong overlay references in O(1); externally retained proxies - // still see the stale status and are reclaimed with their holders. - releaseOverlayReferences(context); - } - this.#contexts.clear(); + const open = this.#open; + const competitors = this.#competitors; + this.#open = undefined; + this.#competitors = undefined; this.#pruneBudget = 256; + if (open !== undefined) staleCompetitor(open, winner); + if (competitors === undefined) return; + for (const reference of competitors) { + const context = reference.deref(); + if (context !== undefined) staleCompetitor(context, winner); + } } replacementBase(context: OverlayContext): T | undefined { @@ -335,6 +345,16 @@ class TrackerImpl implements Tracker { } } +function staleCompetitor(context: OverlayContext, winner: OverlayContext): void { + if (context === winner) return; + if (context.status.value === "open" || context.status.value === "prepared") context.status.value = "stale"; + // Adoption must not walk every node touched by a competing draft. Drop the + // context's strong overlay references in O(1); externally retained proxies + // still see the stale status and are reclaimed with their holders. + context.registryRef = undefined; + releaseOverlayReferences(context); +} + /** Take immutable ownership of an alias-free strict-JSON root in O(1). */ export function track(initial: T): Tracker { return new TrackerImpl(initial); @@ -408,10 +428,10 @@ function createContext( status: { value: "open" }, root: undefined, bases: [], - stored: [], + stored: undefined, dirty: [], nodes: [], - rawNodes: new WeakMap(), + rawNodes: undefined, ops: undefined, replacement, replacementNoop: false, @@ -483,7 +503,7 @@ function createNode( parentSource: InsertSource | undefined = undefined, parentPlacement = false, ): OverlayNode { - const existing = context.rawNodes!.get(base); + const existing = context.rawNodes?.get(base); if (existing !== undefined) return existing; const target: object = Array.isArray(base) ? [] : {}; const baseIndex = context.bases!.length; @@ -502,7 +522,12 @@ function createNode( if (parentSource !== undefined) node.parentSource = parentSource; (target as Record)[NODE] = node; node.proxy = new Proxy(target, Array.isArray(base) ? sharedArrayHandler : sharedObjectHandler); - context.rawNodes!.set(base, node); + // The root is never reached again through a child value (tracked values are alias-free), so a change that touches + // only the root needs no map. + if (parent !== undefined) { + context.rawNodes ??= new WeakMap(); + context.rawNodes.set(base, node); + } context.nodes.push(node); return node; } @@ -517,8 +542,10 @@ function nodeParent(node: OverlayNode): OverlayNode | undefined { function storeValue(context: OverlayContext, value: Stored): StoredRef { if (!isContainer(value)) return value; - const index = context.stored!.length; - context.stored!.push(value); + // Created on the first container write: primitive writes never store. + context.stored ??= []; + const index = context.stored.length; + context.stored.push(value); return { index }; } @@ -1121,12 +1148,12 @@ function insertPiece(node: OverlayNode, items: Stored[], start = 0): InsertPiece function insertPlacementPiece(node: OverlayNode, values: unknown[], start = 0): InsertPiece[] { if (values.length === start) return []; - const storedLength = node.context.stored!.length; + const storedLength = node.context.stored?.length ?? 0; try { for (let index = start; index < values.length; index++) values[index] = storeValue(node.context, clonePlacement(values[index])); } catch (error) { - node.context.stored!.length = storedLength; + if (node.context.stored !== undefined) node.context.stored.length = storedLength; throw error; } return [ @@ -1611,7 +1638,7 @@ function cloneNode(node: OverlayNode): Container { function cloneStored(value: Stored, context: OverlayContext): Stored { if (!isContainer(value)) return value; - const node = context.rawNodes!.get(value); + const node = context.rawNodes?.get(value); return node === undefined || (!node.dirty && !node.subtreeDirty) ? value : cloneNode(node); } @@ -1636,7 +1663,7 @@ function clonePlacementNode(node: OverlayNode): Container { function clonePlacementStored(value: Stored, context: OverlayContext): Stored { if (!isContainer(value)) return copyJson(value); - const node = context.rawNodes!.get(value); + const node = context.rawNodes?.get(value); return node === undefined ? copyJson(value) : clonePlacementNode(node); } diff --git a/packages/chord/src/host.ts b/packages/chord/src/host.ts index 9f96a4909..fe94fd9be 100644 --- a/packages/chord/src/host.ts +++ b/packages/chord/src/host.ts @@ -1136,7 +1136,7 @@ export class HostImpl implements Host, HostCore { return best?.[1]; } - /** Authorize, resolve (maybe spawn), authorize again; returns the synced connection to the host. */ + /** Authorize, resolve (maybe spawn) and admit the host, authorize again; returns the synced connection to the host. */ async #resolve( route: Route, hostName: string, @@ -1148,15 +1148,17 @@ export class HostImpl implements Host, HostCore { if (!(await route.authorize(principal, hostName))) throw refuse(); if (!still()) throw revoked(); const link = await route.resolve(hostName); - if (!still()) throw revoked(); - if (!(await route.authorize(principal, hostName))) throw refuse(); - if (!still()) throw revoked(); + // Admitted before the checks: a host the route spawned for a request that is gone or refused meanwhile still runs, + // and its own references to this host would otherwise wait on a link nobody reads until they time out. let connection = [...this.connections].find((candidate) => candidate.link === link); connection ??= this.#addConnection(link, { role: "downstream", principal: route.principal(hostName), peer: { kind: "host", name: hostName }, }); + if (!still()) throw revoked(); + if (!(await route.authorize(principal, hostName))) throw refuse(); + if (!still()) throw revoked(); await connection.whenSynced(); if (!still()) throw revoked(); return connection; diff --git a/packages/chord/src/index.ts b/packages/chord/src/index.ts index 8b8bd47d6..34b3613ea 100644 --- a/packages/chord/src/index.ts +++ b/packages/chord/src/index.ts @@ -81,9 +81,11 @@ export type { Ctx, GetOptions, Readable, RefStatus, Shape, StateEvent } from "./ export { NativeWatch, type NativeWatchable, + type NativeWatchCall, type NativeWatchHandle, Watch, type Watchable, + type WatchCall, type WatchEnd, type WatchHandle, } from "./watch.ts"; diff --git a/packages/chord/src/next-node/compression.ts b/packages/chord/src/next-node/compression.ts new file mode 100644 index 000000000..d1b9e7006 --- /dev/null +++ b/packages/chord/src/next-node/compression.ts @@ -0,0 +1,24 @@ +// Compression for Chord v2 hosts on Node (§22, slice 4b): the synchronous compressor `createHost({ compression })` +// takes, over Node's zlib. Node-only, so it lives outside `src/next`, which imports no Node module. + +import { constants as buffer } from "node:buffer"; +import { deflateRawSync, inflateRawSync } from "node:zlib"; +import type { Compression } from "../next/link/index.ts"; + +/** + * Raw deflate at level 1 (`deflateRawSync`), and inflate bounded by `maxOutputLength` (`inflateRawSync`): past the + * limit `too_large`, data that is no raw deflate stream `corrupt`. + */ +export const nodeCompression: Compression = { + deflate: (bytes) => deflateRawSync(bytes, { level: 1 }), + inflate: (bytes, limit) => { + try { + // zlib refuses a bound of 0: inflate at most one byte, and one byte is past a limit of 0. + const inflated = inflateRawSync(bytes, { maxOutputLength: Math.max(1, Math.min(limit, buffer.MAX_LENGTH)) }); + return inflated.byteLength > limit ? { kind: "too_large" } : { kind: "bytes", bytes: inflated }; + } catch (error) { + if ((error as { readonly code?: unknown }).code === "ERR_BUFFER_TOO_LARGE") return { kind: "too_large" }; + return { kind: "corrupt", error }; + } + }, +}; diff --git a/packages/chord/src/next-node/http/README.md b/packages/chord/src/next-node/http/README.md new file mode 100644 index 000000000..266ba00d0 --- /dev/null +++ b/packages/chord/src/next-node/http/README.md @@ -0,0 +1,166 @@ +# http + +The HTTP component (slice 10 §2, design §23; decisions "The HTTP core is a fetch handler", "The HTTP server is a native +object", "The HTTP component's security baseline", "Slice 10 fix round, allowed hosts only through contributions", +"Settings are arguments, extensions are registries", "A CSP nonce per response"): the server's HTTP and WebSocket +entry. Plugins contribute routes, wraps, sockets, allowed hosts, framers, relays and +CSP sources to the registry `Http`; routes return response descriptions. The core is one Web-standard handler, +`fetch(Request) → Response`; Chord's own Node server (the listener) is the easy path, an app with a server of its own +mounts the handler. Node-only; `ws` is an optional peer dependency of `@earendil-works/chord`, imported only here. +Exported as `@earendil-works/chord/next/http` once v2 has its entry point. + +## For apps (`index.ts`) + +```ts +// the app's settings are arguments, fixed for the component's life (changing one is a restart) +const http = createHttp(server, { + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own listener; port 0 picks one; none: no listener + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, // or a value + closeTimeout: 30_000, + onError: (error) => log(error), // HttpError +}, ctx); +// the app's core facet: who someone is (a registry: plugins may add authenticators too) +const app = defineFacet((_host, self) => { + self.contribute(Authenticators, (request, ctx) => principalFromCookie(readCookie(request, "session", secret))); +}); +const core = [{ name: "chord.http", facet: http.facet }, { name: "@app/core", facet: app }]; +await plugins.load(server, { kind: "server", core, modules }, ctx); +const { port } = await http.listening(ctx); +… +await http.close(ctx); // first: drains running requests (host.close would seal their scopes at once) +await server.close(ctx); +``` + +```ts +// a plugin's server facet +self.contribute(Http, { + routes: { + "GET /notes/:id": async (request, ctx) => ({ kind: "json", value: … }), + "GET /": { page: true, handler: async (request) => ({ kind: "html", body: `` }; + }, + }, + framers: ["https://shell.example.com"], + csp: { "img-src": ["https://cdn.example.com"] }, + }), + }); + const page = await send(server.port, "GET", "/page"); + expect(page.headers).toMatchObject({ + "cache-control": "no-store", + "x-content-type-options": "nosniff", + "referrer-policy": "no-referrer", + }); + expect(page.headers["x-frame-options"]).toBeUndefined(); + expect(page.headers["content-security-policy"]).toBe( + `default-src 'none'; script-src 'self' 'nonce-${nonce}'; connect-src 'self' ws://127.0.0.1:${server.port}; base-uri 'none'; form-action 'self'; frame-ancestors https://shell.example.com; img-src https://cdn.example.com`, + ); + // Fable's recheck: a nonce read from one page must not run a script injected into another. + const first = nonce; + await send(server.port, "GET", "/page"); + expect(nonce).not.toBe(first); + const missing = await send(server.port, "GET", "/nope"); + expect([missing.status, missing.headers["x-content-type-options"]]).toEqual([404, "nosniff"]); + expect(missing.headers["content-security-policy"]).toMatch(/'nonce-[A-Za-z0-9+/]{22}=='/); + expect(missing.headers["content-security-policy"]).not.toContain(nonce); + await server.close(); + const plain = await served({}); + expect((await send(plain.port, "GET", "/")).headers["x-frame-options"]).toBe("DENY"); + await plain.close(); + }); + + it("lets a relay's verified request through with its framer", async () => { + const server = await served({ + "@app/who": whoami, + "@app/relay": contributing({ + relays: [ + { + name: "radius", + verify: (request) => + request.headers.get("x-relay") === "secret" + ? { origin: "https://me.relay.example", framer: "https://relay.example" } + : undefined, + }, + ], + }), + }); + const forwarded = { "x-forwarded-for": "203.0.113.9" }; + expect((await send(server.port, "GET", "/whoami", { headers: forwarded })).status).toBe(403); + const relayed = await send(server.port, "GET", "/whoami", { headers: { ...forwarded, "x-relay": "secret" } }); + expect(relayed.status).toBe(200); + expect(relayed.headers["content-security-policy"]).toContain("frame-ancestors https://relay.example"); + await server.close(); + }); +}); + +describe("routes and wraps failing", () => { + it("answers a throwing route 500 without its message, a throwing wrap 500 naming its plugin, an HttpFailure its status", async () => { + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /throws": async () => { + throw new Error("secret"); + }, + "GET /failure": async () => { + throw new HttpFailure(422, "no"); + }, + }, + }), + "@app/b": contributing({ + routes: { "GET /wrapped": async () => ({ kind: "json", value: 1 }) }, + wraps: [ + { + name: "broken", + phase: "headers", + wrap: async () => { + throw new Error("wrap"); + }, + }, + ], + }), + }); + const thrown = await send(server.port, "GET", "/throws"); + expect([thrown.status, thrown.body.toString()]).toEqual([500, "Internal Server Error"]); + expect((await send(server.port, "GET", "/failure")).status).toBe(422); + expect((await send(server.port, "GET", "/wrapped")).status).toBe(500); + expect(server.errors).toMatchObject([ + { kind: "route_failed", pattern: "GET /throws", plugin: "@app/a", error: { message: "secret" } }, + { kind: "wrap_failed", name: "broken", plugin: "@app/b", error: { message: "wrap" } }, + ]); + await server.close(); + }); + + it("runs wraps outermost first around the route, in its request's scope", async () => { + const order: string[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": async (_request, ctx) => { + order.push(`route as ${ctx.principal?.kind ?? "anonymous"}`); + return { kind: "json", value: 1 }; + }, + }, + wraps: [ + { + name: "inner", + phase: "transform", + wrap: async (request, next) => { + order.push("transform"); + return next(request); + }, + }, + { + name: "outer", + phase: "observe", + wrap: async (request, next) => { + order.push("observe"); + const response = await next(request); + return { ...response, headers: { "X-Observed": "1" } }; + }, + }, + ], + }), + }); + const answer = await send(server.port, "GET", "/r", { headers: { "x-principal": "owner" } }); + expect([order, answer.headers["x-observed"]]).toEqual([["observe", "transform", "route as owner"], "1"]); + await server.close(); + }); +}); + +describe("reloads and close", () => { + it("a reload swapping a route leaves a running request on the old router; the next uses the new one", async () => { + const gate = Promise.withResolvers(); + const old = contributing({ + routes: { + "GET /v": async () => { + await gate.promise; + return { kind: "html", body: "old" }; + }, + }, + }); + const server = await served({ "@app/v": old }); + const running = send(server.port, "GET", "/v"); + await new Promise((resolve) => setTimeout(resolve, 50)); + const next = contributing({ routes: { "GET /v": async () => ({ kind: "html", body: "new" }) } }); + await server.host.reload(server.facets({ "@app/v": next }), context); + expect((await send(server.port, "GET", "/v")).body.toString()).toBe("new"); + gate.resolve(); + expect((await running).body.toString()).toBe("old"); + await server.close(); + }); + + it("close refuses new requests (503), waits for running ones up to closeTimeout, then seals them (503)", async () => { + let sealed: AbortSignal | undefined; + const server = await served( + { + "@app/slow": contributing({ + routes: { + "GET /quick": async () => { + await new Promise((resolve) => setTimeout(resolve, 30)); + return { kind: "html", body: "done" }; + }, + "GET /stuck": async (_request, ctx: Ctx) => { + sealed = ctx.abortSignal; + return new Promise(() => {}); + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 100 } }, + ); + const quick = send(server.port, "GET", "/quick"); + const stuck = send(server.port, "GET", "/stuck"); + await new Promise((resolve) => setTimeout(resolve, 10)); + const closing = server.http.close(context); + expect((await server.http.fetch(loopback("/quick"), LOOPBACK)).status).toBe(503); + expect((await quick).body.toString()).toBe("done"); + await closing; + expect(sealed?.aborted).toBe(true); + await server.host.close(context); + expect((await stuck).status).toBe(503); + await expect(send(server.port, "GET", "/quick")).rejects.toThrow(); + }); + + // Astra's build review: a request authenticating when close began ran its route afterwards. + it("never runs a route for a request still authenticating when close sealed it", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let calls = 0; + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": async () => { + calls++; + return { kind: "json", value: 1 }; + }, + }, + }), + }, + { + config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 20 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + gate.resolve(); + expect([(await answer).status, calls]).toEqual([503, 0]); + await server.close(); + }); +}); + +describe("mounting", () => { + it("without a connection a request is never local", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined } }); + expect((await server.http.fetch(loopback("/whoami"))).status).toBe(403); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Sol's second recheck C: after cancellation the chain still started the next authenticator. + it("starts no further authenticator once close sealed the request", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let later = 0; + const server = await served( + { + "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }), + "@app/z-later": defineFacet((_host, self) => + self.contribute(Authenticators, () => { + later++; + return { kind: "owner" }; + }), + ), + }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return undefined; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + expect((await answer).status).toBe(503); + gate.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(later).toBe(0); + await server.close(); + }); + + it("asks no authenticator for a request aborted before it arrived", async () => { + let calls = 0; + const server = await served( + { "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }) }, + { + config: { listen: undefined }, + authenticate: () => { + calls++; + return undefined; + }, + }, + ); + await server.http.fetch(loopback("/", { signal: AbortSignal.abort() }), LOOPBACK); + expect(calls).toBe(0); + await server.close(); + }); + + // Sol's third recheck: a request aborted while its route returned the `bytes` description got a stream born sealed, + // which never errored; the request never finished. + it.each(["inside the route", "after the route returned"] as const)( + "ends a request whose bytes Response was born sealed (aborted %s)", + async (when) => { + const abort = new AbortController(); + const entered = Promise.withResolvers(); + let reads = 0; + let bornSealed: boolean | undefined; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + entered.resolve(); + if (when === "inside the route") abort.abort(new Error("client gone")); + bornSealed = ctx.abortSignal?.aborted; + return { + kind: "bytes", + mime: "application/octet-stream", + read: async () => { + reads++; + return new Uint8Array([1]); + }, + }; + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const answer = server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + await entered.promise; + if (when === "after the route returned") abort.abort(new Error("client gone")); + await answer; + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect([Date.now() - started < 500, reads, bornSealed]).toEqual([true, 0, when === "inside the route"]); + await server.close(); + }, + ); + + // Sol's second recheck B: an aborted `Response` nobody touched again kept its request: close waited its bound. + it("ends a request whose bytes Response was aborted and dropped without another read", async () => { + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async () => new Uint8Array([1]), + }), + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + await server.close(); + }); + + // Astra's recheck R4, Sol's N1: a close before the first resolution started a stopped watch; its throw ended the process. + it("a close before the publication leaves nothing running: the waiting request answers 503", async () => { + const server = await served({}, { load: false, config: { listen: undefined } }); + const waiting = server.http.fetch(loopback("/x"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + await server.load(); + expect((await waiting).status).toBe(503); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + it("a close while the first request resolves the registries starts no follow", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined, closeTimeout: 10 } }); + const pending = server.http.fetch(loopback("/whoami"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.timeout(20), context)); + expect([200, 503]).toContain((await pending).status); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + // Sol's recheck N9. + it("a close whose context is aborted already does not wait", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { + "@app/wait": contributing({ + routes: { + "GET /wait": async () => { + entered.resolve(); + await gate.promise; + return { kind: "html", body: "done" }; + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1000 } }, + ); + const answer = server.http.fetch(loopback("/wait"), LOOPBACK); + await entered.promise; + const started = Date.now(); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + expect(Date.now() - started).toBeLessThan(500); + gate.resolve(); + await answer; + await server.close(); + }); + + // Sol's recheck N5: an authenticator ignoring its signal kept fetch pending after close returned. + it("close settles a request whose authenticator ignores its signal (503)", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }) }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + const settled = await Promise.race([answer.then((response) => response.status), sleepThen(30, "pending")]); + expect(settled).toBe(503); + gate.resolve(); + await server.close(); + }); + + // Astra's recheck R2, Sol's N3: a held `Response` went on calling `read` after its scope was sealed. + it.each(["the request aborted", "the component closed"] as const)("reads no bytes after %s", async (kind) => { + let reads = 0; + let readAfterSeal = false; + const abort = new AbortController(); + const server = await served( + { + "@app/bytes": contributing({ + routes: { + "GET /stream": async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async (_offset, _length, ctx) => { + reads++; + readAfterSeal ||= ctx.abortSignal?.aborted ?? false; + return new Uint8Array([reads]); + }, + }), + }, + }), + }, + { config: { listen: undefined, closeTimeout: 10 } }, + ); + const response = await server.http.fetch(loopback("/stream", { signal: abort.signal }), LOOPBACK); + const reader = response.body!.getReader(); + await reader.read(); + if (kind === "the request aborted") abort.abort(new Error("client gone")); + else await server.http.close(context); + await new Promise((resolve) => setImmediate(resolve)); + await expect(reader.read()).rejects.toThrow(); + expect({ reads, readAfterSeal }).toEqual({ reads: 1, readAfterSeal: false }); + await server.close(); + }); + + // Sol's recheck N4. + it("a body that errors after valid JSON fails the request (400); the route's side effect never runs", async () => { + let writes = 0; + const server = await served( + { + "@app/body": contributing({ + routes: { + "POST /body": async (request) => { + const value = await request.json(); + writes++; + return { kind: "json", value }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + if (pulls++ === 0) controller.enqueue(new TextEncoder().encode("{}")); + else controller.error(new Error("incoming body checksum failed")); + }, + }); + const init = { method: "POST", body, duplex: "half" as const }; + const response = await server.http.fetch(loopback("/body", init), LOOPBACK); + expect([response.status, writes]).toEqual([400, 0]); + await server.close(); + }); +}); + +describe("answers", () => { + // Sol's recheck N8: `Response` threw on a body for these statuses. + it.each([204, 205, 304])("answers %s without a body", async (status) => { + const server = await served( + { "@app/s": contributing({ routes: { "GET /": async () => ({ kind: "json", value: null, status }) } }) }, + { config: { listen: undefined } }, + ); + const response = await server.http.fetch(loopback("/"), LOOPBACK); + expect([response.status, response.body, response.headers.has("content-length")]).toEqual([status, null, false]); + await server.close(); + }); + + // Fable's recheck N3: the kernel's `forbidden` reached the client as a reported 500. + it("answers a forbidden escaping a route 403, unreported", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": async () => { + throw new ChordError({ code: "forbidden", message: "no" }); + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + expect((await server.http.fetch(loopback("/"), LOOPBACK)).status).toBe(403); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Opus's recheck R2: one value per header name allowed one cookie per response. + it("sends several Set-Cookie headers", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { "GET /": async () => ({ kind: "json", value: 1, headers: { "Set-Cookie": ["a=1", "b=2"] } }) }, + }), + }); + expect((await send(server.port, "GET", "/")).headers["set-cookie"]).toEqual(["a=1", "b=2"]); + const fetched = await server.http.fetch( + new Request(`http://127.0.0.1:${server.port}/`, { headers: { host: `127.0.0.1:${server.port}` } }), + { localAddress: "127.0.0.1", localPort: server.port }, + ); + expect(fetched.headers.getSetCookie()).toEqual(["a=1", "b=2"]); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's second recheck B, her GC probe: 8 of 8 such requests kept their scopes. + it("collects the scopes of aborted bytes Responses dropped without another read", async () => { + const refs: WeakRef[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + return { kind: "bytes", mime: "application/octet-stream", read: async () => new Uint8Array([1]) }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + for (let i = 0; i < 8; i++) { + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + } + await expectCollected(...refs); + await server.close(); + }); + + it("collects the scopes of bytes Responses born sealed (aborted while the route returned)", async () => { + const refs: WeakRef[] = []; + const aborts: AbortController[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + aborts.at(-1)?.abort(new Error("client gone")); + return { kind: "bytes", mime: "application/octet-stream", read: async () => new Uint8Array([1]) }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + // In a function of its own: the test's frame keeps no last request. + const fetchAborted = async (): Promise => { + const abort = new AbortController(); + aborts.push(abort); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + }; + for (let i = 0; i < 8; i++) await fetchAborted(); + aborts.length = 0; + await expectCollected(...refs); + await server.close(); + }); + + it("collects requests' scopes and requests once answered", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": async (request, ctx) => { + refs.push(new WeakRef(ctx), new WeakRef(request)); + return { kind: "json", value: 1 }; + }, + }, + }), + }); + for (let i = 0; i < 5; i++) await send(server.port, "GET", "/r"); + await expectCollected(...refs); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/node.test.ts b/packages/chord/test/next-node/http/node.test.ts new file mode 100644 index 000000000..8349bd751 --- /dev/null +++ b/packages/chord/test/next-node/http/node.test.ts @@ -0,0 +1,215 @@ +// Tests of Chord's own Node server (the listener's native): request targets (origin-form only; a malformed or +// double-slash path never crashes or reroutes), raw `Host` headers, Web requests made only when read, the request's scope lasting until its response is +// flushed (and `close` waiting for it), and a client's disconnect aborting the scope. + +import { once } from "node:events"; +import { createServer, request as httpRequest, type IncomingMessage } from "node:http"; +import { connect } from "node:net"; +import { describe, expect, it } from "vitest"; +import { nodeRequest } from "../../../src/next-node/http/index.ts"; +import { context, contributing, served } from "./support.ts"; + +/** What `port` answers a raw request `text` with: its status line and body. */ +async function raw(port: number, text: string): Promise<{ readonly status: string; readonly body: string }> { + const socket = connect(port, "127.0.0.1"); + let out = ""; + socket.on("data", (chunk) => { + out += chunk.toString("latin1"); + }); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write(text); + await once(socket, "close"); + const [head = "", body = ""] = out.split("\r\n\r\n"); + return { status: head.split("\r\n")[0] ?? "", body }; +} + +/** A route answering its path and params, under `GET /who` and `GET /n/:name`. */ +const echo = contributing({ + routes: { + "GET /who": async (request) => ({ kind: "json", value: request.path }), + "GET /n/:name": async (request) => ({ kind: "json", value: request.params }), + }, +}); + +describe("targets", () => { + // Astra's build review, Fable's security audit C1: `GET //[` threw out of the request handler and ended the process. + it("answers a malformed or double-slash path without crashing or rerouting it (Opus's audit H5)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const get = (target: string) => + raw(server.port, `GET ${target} HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`); + expect((await get("//[")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("//evil.example/who")).status).toBe("HTTP/1.1 404 Not Found"); + expect(await get("/a/../who")).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + // The URL parser resolves `%2e%2e` as a dot segment: `/n/%2e%2e` is `/`. + expect((await get("/n/%2e%2e")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("/n/a%5Cb")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/n/..%2F..%2Fsecret")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/who")).status).toBe("HTTP/1.1 200 OK"); + await server.close(); + }); + + it("refuses an absolute-form target, two Host headers and none (403)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const answers = await Promise.all([ + raw(server.port, `GET http://evil.example/who HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`), + raw(server.port, `GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\nConnection: close\r\n\r\n`), + raw(server.port, "GET /who HTTP/1.0\r\n\r\n"), + ]); + expect(answers.map(({ status }) => status)).toEqual([ + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + ]); + await server.close(); + }); + + it("refuses a Host with userinfo (403) and a method a Web Request cannot carry (501)", async () => { + const server = await served({ "@app/echo": echo }); + const answers = await Promise.all([ + raw(server.port, `GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + raw(server.port, `TRACE /who HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + ]); + // `TRACK` never reaches Chord: Node's parser refuses it (400); `CONNECT` goes to Node's `connect` event. + expect(answers.map(({ status }) => status)).toEqual(["HTTP/1.1 403 Forbidden", "HTTP/1.1 501 Not Implemented"]); + await server.close(); + }); +}); + +describe("mounting on an app's own Node server", () => { + // Sol's recheck N6: a hand-made `Request` threw unhandled on `Host: [` or an absolute-form target (the process + // ended) and kept only the first of two `Host`s. + it("converts with the listener's checks, never throwing: bad targets, userinfo, TRACE refused; duplicate Hosts kept", async () => { + const server = await served({ "@app/echo": echo }, { config: { listen: undefined } }); + const own = createServer(async (message, res) => { + const converted = nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await server.http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); + }); + await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); + const address = own.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const host = `Host: 127.0.0.1:${port}`; + const get = (text: string) => raw(port, `${text}Connection: close\r\n\r\n`); + expect(await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + expect((await get("GET /who HTTP/1.1\r\nHost: [\r\n")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get(`GET http://evil.example/who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 403 Forbidden"); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + // Sol's second recheck A: both threw out of the conversion; the app's callback ended the process. + expect((await get(`TRACE /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 501 Not Implemented"); + expect((await get(`GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${port}\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 200 OK"); + await new Promise((resolve) => own.close(resolve)); + await server.close(); + }); +}); + +describe("Web requests", () => { + it("makes a Web Request only when something reads the headers or the body (here: no authenticator does)", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /plain": async (request) => ({ kind: "json", value: request.path }), + "GET /headers": async (request) => ({ kind: "json", value: request.headers.get("x-thing") }), + "POST /body": async (request) => ({ kind: "json", value: await request.json() }), + }, + }), + }, + { authenticate: () => undefined }, + ); + const original = globalThis.Request; + let made = 0; + globalThis.Request = class extends original { + constructor(...args: ConstructorParameters) { + super(...args); + made++; + } + }; + try { + const host = `Host: 127.0.0.1:${server.port}\r\nConnection: close`; + await raw(server.port, `GET /plain HTTP/1.1\r\n${host}\r\n\r\n`); + expect(made).toBe(0); + const read = await raw(server.port, `GET /headers HTTP/1.1\r\n${host}\r\nX-Thing: t\r\n\r\n`); + const posted = await raw(server.port, `POST /body HTTP/1.1\r\n${host}\r\nContent-Length: 3\r\n\r\n"a"`); + expect([read.body, posted.body, made]).toEqual(['"t"', '"a"', 2]); + } finally { + globalThis.Request = original; + } + await server.close(); + }); +}); + +describe("the request's scope", () => { + // Sol's build review: the scope sealed, and `close` stopped waiting, before a large response was flushed. + it("lasts until the response is flushed; close waits for it", async () => { + const size = 32 * 1024 * 1024; + let signal: AbortSignal | undefined; + const server = await served( + { + "@app/large": contributing({ + routes: { + "GET /large": async (_request, ctx) => { + signal = ctx.abortSignal; + return { kind: "html", body: "x".repeat(size) }; + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 2000 } }, + ); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/large", agent: false }); + const response = await new Promise((resolve) => { + request.on("response", (arrived: IncomingMessage) => { + arrived.pause(); + resolve(arrived); + }); + request.end(); + }); + let length = 0; + response.on("data", (chunk: Buffer) => { + length += chunk.length; + }); + response.pause(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const sealedBeforeFlush = signal?.aborted; + const closing = server.http.close(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + response.resume(); + await once(response, "end"); + await closing; + expect({ sealedBeforeFlush, length }).toEqual({ sealedBeforeFlush: false, length: size }); + await server.close(); + }); + + it("is aborted when the client disconnects", async () => { + const entered = Promise.withResolvers(); + const server = await served({ + "@app/slow": contributing({ + routes: { + "GET /slow": async (_request, ctx) => { + entered.resolve(ctx.abortSignal!); + return new Promise(() => {}); + }, + }, + }), + }); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/slow", agent: false }); + request.on("error", () => {}); + request.end(); + const signal = await entered.promise; + const aborted = new Promise((resolve) => signal.addEventListener("abort", resolve, { once: true })); + request.destroy(); + await aborted; + expect(signal.aborted).toBe(true); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/request.test.ts b/packages/chord/test/next-node/http/request.test.ts new file mode 100644 index 000000000..a48a2429e --- /dev/null +++ b/packages/chord/test/next-node/http/request.test.ts @@ -0,0 +1,119 @@ +// Tests of requests (slice 10 §2, D4): the summary a route gets, `json` and `body` with their limits and statuses +// (400, 413, an `HttpFailure` escaping a route answering its status), and a body nobody read (drained or closed). + +import { Agent, request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { HttpRequest } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +describe("the request a route gets", () => { + it("has its method, path, params, query, headers, origin, provenance and the response's nonce", async () => { + let seen: HttpRequest | undefined; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /notes/:id": async (request) => { + seen = request; + return { kind: "json", value: null }; + }, + }, + }), + }); + await send(server.port, "GET", "/notes/a%20b?x=1&x=2", { headers: { "X-Thing": "t" } }); + expect(seen).toMatchObject({ + method: "GET", + path: "/notes/a%20b", + params: { id: "a b" }, + origin: `http://127.0.0.1:${server.port}`, + provenance: { kind: "local" }, + }); + expect([seen?.query.getAll("x"), seen?.headers.get("x-thing")]).toEqual([["1", "2"], "t"]); + expect(seen?.nonce).toMatch(/^[A-Za-z0-9+/]{22}==$/); + await server.close(); + }); +}); + +describe("bodies", () => { + /** A server whose `POST /json` answers the JSON it read (limit 16), `POST /body` the bytes it read (limit 16). */ + const reading = () => + served({ + "@app/a": contributing({ + routes: { + "POST /json": async (request) => ({ kind: "json", value: await request.json(16) }), + "POST /body": async (request) => { + let bytes = 0; + for await (const chunk of request.body(16)) bytes += chunk.byteLength; + return { kind: "json", value: bytes }; + }, + }, + }), + }); + + it("parses JSON within its limit; malformed is 400, larger is 413 (declared or streamed)", async () => { + const server = await reading(); + const { port } = server; + expect((await send(port, "POST", "/json", { body: '{"a":1}' })).body.toString()).toBe('{"a":1}'); + expect((await send(port, "POST", "/json", { body: "{nope" })).status).toBe(400); + expect((await send(port, "POST", "/json", { body: JSON.stringify({ long: "x".repeat(40) }) })).status).toBe(413); + const chunked = await send(port, "POST", "/body", { + headers: { "transfer-encoding": "chunked" }, + body: "y".repeat(40), + }); + expect(chunked.status).toBe(413); + expect((await send(port, "POST", "/body", { body: "y".repeat(10) })).body.toString()).toBe("10"); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Sol's and Astra's build reviews: an unfinished chunked upload past the limit got no answer. + it("answers 413 before a chunked upload past the limit has ended", async () => { + const server = await reading(); + const request = httpRequest({ + host: "127.0.0.1", + port: server.port, + path: "/json", + method: "POST", + headers: { "transfer-encoding": "chunked" }, + agent: false, + }); + request.on("error", () => {}); + const status = new Promise((resolve) => { + request.on("response", (response) => { + response.resume(); + resolve(response.statusCode ?? 0); + }); + }); + request.write(`"${"x".repeat(40)}`); + expect(await status).toBe(413); + request.destroy(); + await server.close(); + }); + + it("drains a small body nobody read and keeps the connection; a large one closes it", async () => { + const server = await served({ + "@app/a": contributing({ routes: { "POST /ignore": async () => ({ kind: "json", value: 1 }) } }), + }); + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + const post = (body: string) => + new Promise<{ connection: string | undefined; socket: unknown }>((resolve, reject) => { + const request = httpRequest( + { host: "127.0.0.1", port: server.port, method: "POST", path: "/ignore", agent }, + (response) => { + response.resume(); + response.on("end", () => + resolve({ connection: response.headers.connection, socket: request.socket }), + ); + }, + ); + request.on("error", reject); + request.end(body); + }); + const first = await post("x".repeat(1000)); + const second = await post("x".repeat(1000)); + expect(first.connection).toBe("keep-alive"); + expect(second.socket).toBe(first.socket); + expect((await post("x".repeat(100_000))).connection).toBe("close"); + agent.destroy(); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/response.test.ts b/packages/chord/test/next-node/http/response.test.ts new file mode 100644 index 000000000..3c953ba85 --- /dev/null +++ b/packages/chord/test/next-node/http/response.test.ts @@ -0,0 +1,245 @@ +// Tests of responses (slice 10 §2, D5, R6; security baseline (4), (6)): every kind written by a running server, the +// response's headers over the default ones (an invalid one reported, 500), ranges (206, 416, multi-range), `HEAD`, +// attachments and inline bodies, safe download names, bodies without `size`, a short or throwing read, a disconnect +// stopping the reads. + +import { request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { Ctx } from "../../../src/next/host/index.ts"; +import type { HttpResponse } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +/** A server answering `GET /r` with `response`. */ +async function answering(response: HttpResponse) { + return served({ "@app/r": contributing({ routes: { "GET /r": async () => response } }) }); +} + +/** A `bytes` response over `data` that records its reads. */ +function bytesOf(data: Buffer, extra: Partial> = {}) { + const reads: [number, number][] = []; + const response: HttpResponse = { + kind: "bytes", + mime: "application/octet-stream", + size: data.length, + read: async (offset, length) => { + reads.push([offset, length]); + return data.subarray(offset, offset + length); + }, + ...extra, + }; + return { response, reads }; +} + +describe("descriptions", () => { + it("writes json, html, redirect and error, the default headers first, the response's replacing same-named ones", async () => { + const cases: [HttpResponse, number, string, string | undefined][] = [ + [{ kind: "json", value: { a: 1 } }, 200, '{"a":1}', "application/json; charset=utf-8"], + [ + { kind: "html", body: "

x

", status: 201, headers: { "content-security-policy": "own" } }, + 201, + "

x

", + "text/html; charset=utf-8", + ], + [{ kind: "redirect", to: "/next", status: 303 }, 303, "", undefined], + [{ kind: "error", status: 418 }, 418, "I'm a Teapot", "text/plain; charset=utf-8"], + [{ kind: "error", status: 409, message: "taken" }, 409, "taken", "text/plain; charset=utf-8"], + ]; + for (const [response, status, body, type] of cases) { + const server = await answering(response); + const answer = await send(server.port, "GET", "/r"); + expect([answer.status, answer.body.toString(), answer.headers["content-type"]]).toEqual([status, body, type]); + expect([answer.headers["cache-control"], answer.headers["x-content-type-options"]]).toEqual([ + "no-store", + "nosniff", + ]); + expect(answer.headers["content-security-policy"]).toMatch( + response.headers === undefined ? /^default-src 'none'/ : /^own$/, + ); + if (response.kind === "redirect") expect(answer.headers.location).toBe("/next"); + await server.close(); + } + }); + + it("answers 500 for a header with a CR or LF and a value that is no JSON, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /header": async () => ({ kind: "json", value: 1, headers: { "X-Split": "a\r\nSet-Cookie: x=1" } }), + "GET /bigint": async () => ({ kind: "json", value: { n: 1n } as never }), + }, + }), + }); + for (const path of ["/header", "/bigint"]) { + const answer = await send(server.port, "GET", path); + expect([answer.status, answer.headers["set-cookie"], answer.headers["x-content-type-options"]]).toEqual([ + 500, + undefined, + "nosniff", + ]); + } + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /header" }, + { kind: "response_failed", pattern: "GET /bigint" }, + ]); + await server.close(); + }); +}); + +describe("bytes", () => { + const data = Buffer.from("0123456789".repeat(60_000)); + + it("sends Content-Length, reads 256 KiB pieces, answers a single range 206 and an unsatisfiable one 416", async () => { + const { response, reads } = bytesOf(data, { ranges: true }); + const server = await answering(response); + const whole = await send(server.port, "GET", "/r"); + expect([whole.status, whole.headers["content-length"], whole.headers["accept-ranges"]]).toEqual([ + 200, + "600000", + "bytes", + ]); + expect(whole.body.equals(data)).toBe(true); + expect(reads).toEqual([ + [0, 262144], + [262144, 262144], + [524288, 75712], + ]); + const range = await send(server.port, "GET", "/r", { headers: { range: "bytes=10-19" } }); + expect([range.status, range.body.toString(), range.headers["content-range"]]).toEqual([ + 206, + "0123456789", + "bytes 10-19/600000", + ]); + const suffix = await send(server.port, "GET", "/r", { headers: { range: "bytes=-3" } }); + expect([suffix.status, suffix.body.toString()]).toEqual([206, "789"]); + const past = await send(server.port, "GET", "/r", { headers: { range: "bytes=600000-" } }); + expect([past.status, past.headers["content-range"]]).toEqual([416, "bytes */600000"]); + const multi = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1,5-6" } }); + expect([multi.status, multi.body.length]).toEqual([200, 600000]); + await server.close(); + }); + + it("ignores ranges unless asked; HEAD sends headers only and reads nothing", async () => { + const { response, reads } = bytesOf(data); + const server = await answering(response); + const ranged = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ranged.status, ranged.headers["accept-ranges"]]).toEqual([200, undefined]); + reads.length = 0; + const head = await send(server.port, "HEAD", "/r"); + expect([head.status, head.headers["content-length"], head.body.length, reads]).toEqual([200, "600000", 0, []]); + await server.close(); + }); + + it("names a download with an ASCII fallback and the UTF-8 name, control and bidi characters dropped", async () => { + const server = await answering(bytesOf(Buffer.from("x"), { download: 'Grüße "1"\r\n\u202etxt.exe' }).response); + const answer = await send(server.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe( + `attachment; filename="Gr__e _1_txt.exe"; filename*=UTF-8''Gr%C3%BC%C3%9Fe%20%221%22txt.exe`, + ); + await server.close(); + }); + + it("is an attachment unless inline; an inline one gets a sandbox CSP", async () => { + const attached = await answering(bytesOf(Buffer.from("x")).response); + expect((await send(attached.port, "GET", "/r")).headers["content-disposition"]).toBe("attachment"); + await attached.close(); + const shown = await answering(bytesOf(Buffer.from("x"), { inline: true }).response); + const answer = await send(shown.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe("inline"); + expect(answer.headers["content-security-policy"]).toMatch(/; sandbox$/); + await shown.close(); + }); + + it("refuses a foreign request (another site's) 403, reading nothing", async () => { + const { response, reads } = bytesOf(Buffer.from("secret")); + const server = await answering(response); + const foreign = await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "cross-site" } }); + expect([foreign.status, reads]).toEqual([403, []]); + expect((await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "same-origin" } })).status).toBe(200); + await server.close(); + }); + + it("without a size: chunked, read with growing offsets until an empty chunk, no ranges (R6)", async () => { + const pieces = [Buffer.from("ab"), Buffer.from("cde"), Buffer.alloc(0)]; + const offsets: number[] = []; + const server = await answering({ + kind: "bytes", + mime: "text/plain", + ranges: true, + read: async (offset) => { + offsets.push(offset); + return pieces.shift()!; + }, + }); + const answer = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ + answer.status, + answer.body.toString(), + answer.headers["transfer-encoding"], + answer.headers["accept-ranges"], + ]).toEqual([200, "abcde", "chunked", undefined]); + expect(offsets).toEqual([0, 2, 5]); + await server.close(); + }); + + it("destroys the connection on a short or a throwing read, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /short": async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => Buffer.from("abc"), + }), + // Astra's build review: a throwing read hung the response, unreported. + "GET /throws": async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => { + throw new Error("read failed"); + }, + }), + }, + }), + }); + await expect(send(server.port, "GET", "/short")).rejects.toThrow(); + await expect(send(server.port, "GET", "/throws")).rejects.toThrow(); + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /short" }, + { kind: "response_failed", pattern: "GET /throws", error: { message: "read failed" } }, + ]); + await server.close(); + }); + + it("stops reading when the client disconnects: the read's context is aborted", async () => { + let reads = 0; + let signal: AbortSignal | undefined; + const server = await answering({ + kind: "bytes", + mime: "application/octet-stream", + size: 1024 * 1024 * 1024, + read: async (_offset, length, ctx: Ctx) => { + reads++; + signal = ctx.abortSignal; + return Buffer.alloc(length); + }, + }); + await new Promise((resolve) => { + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/r", agent: false }, (response) => { + response.once("data", () => { + request.destroy(); + resolve(); + }); + }); + request.end(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const after = reads; + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(signal?.aborted).toBe(true); + expect(reads).toBe(after); + expect(after).toBeLessThan(1000); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/router.test.ts b/packages/chord/test/next-node/http/router.test.ts new file mode 100644 index 000000000..9854820c6 --- /dev/null +++ b/packages/chord/test/next-node/http/router.test.ts @@ -0,0 +1,228 @@ +// Tests of the router (slice 10 §2, D2, D8): patterns, decoding (one rule for `:name` and rest segments), precedence, +// `HEAD` per path, each route's own param names, route options, conflicts, wraps resolved by phase, contributor order, +// scope and route, allowed hosts, framers, relays, CSP sources, sockets. + +import { describe, expect, it } from "vitest"; +import type { RegistryConflict, RegistryTable } from "../../../src/next/registries/index.ts"; +import { + composeRouter, + type HttpContribution, + type HttpRoute, + type HttpWrap, + type RouteMatch, + type Router, +} from "../../../src/next-node/http/router.ts"; + +/** A route named `name` (its handler answers its name). */ +const route = + (name: string): HttpRoute => + async () => ({ kind: "html", body: name }); + +/** The router of `entries` (plugin, contribution), with the conflicts it reported. */ +function compose(...entries: [string, HttpContribution][]): { router: Router; conflicts: RegistryConflict[] } { + const conflicts: RegistryConflict[] = []; + const table: RegistryTable = entries.map(([plugin, value]) => ({ plugin, version: 1, value })); + return { router: composeRouter(table, { report: (conflict) => conflicts.push(conflict) }), conflicts }; +} + +/** The matched route's pattern and params, or what `route` answered. */ +function matched(router: Router, method: string, path: string) { + const match = router.route(method, path); + if (match === undefined || match === "bad_path") return match; + return { pattern: match.pattern, params: match.params }; +} + +describe("patterns and precedence (D8)", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /notes": route("list"), + "GET /notes/:id": route("one"), + "GET /notes/new": route("new"), + "GET /notes/:id/export": route("export"), + "GET /files/*": route("files"), + "GET /files/special/:name": route("special"), + "GET /": route("root"), + }, + }, + ]); + + it("matches exact segments, :name and a trailing *, the most specific from the left", () => { + expect(matched(router, "GET", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(matched(router, "GET", "/notes/new")).toEqual({ pattern: "GET /notes/new", params: {} }); + expect(matched(router, "GET", "/notes/7")).toEqual({ pattern: "GET /notes/:id", params: { id: "7" } }); + expect(matched(router, "GET", "/notes/7/export")).toMatchObject({ params: { id: "7" } }); + expect(matched(router, "GET", "/files/special/x")).toEqual({ + pattern: "GET /files/special/:name", + params: { name: "x" }, + }); + // The exact branch fails deeper: the rest takes it. + expect(matched(router, "GET", "/files/special/x/y")).toEqual({ + pattern: "GET /files/*", + params: { "*": "special/x/y" }, + }); + expect(matched(router, "GET", "/")).toEqual({ pattern: "GET /", params: {} }); + }); + + it("a /* matches its prefix with an empty rest; :name needs a non-empty segment", () => { + expect(matched(router, "GET", "/files")).toEqual({ pattern: "GET /files/*", params: { "*": "" } }); + expect(matched(router, "GET", "/notes/")).toBeUndefined(); + }); + + // Opus's build review F4, security audit H5: `%2F` in `:name` reached a file path. + it("percent-decodes :name and rest segments; one decoding to /, \\, NUL, . or .. is a bad path", () => { + expect(matched(router, "GET", "/notes/a%20b")).toMatchObject({ params: { id: "a b" } }); + expect(matched(router, "GET", "/files/a%20b/c")).toMatchObject({ params: { "*": "a b/c" } }); + for (const bad of ["a%2Fb", "..%2F..%2Fsecret", "a%5Cb", "a%00b", "%2e%2e", "%2E", ".", ".."]) { + expect(router.route("GET", `/notes/${bad}`)).toBe("bad_path"); + expect(router.route("GET", `/files/x/${bad}`)).toBe("bad_path"); + } + expect(router.route("GET", "/notes/%E0%A4%A")).toBe("bad_path"); + }); + + it("falls back to the GET route for HEAD; another method or path has none", () => { + expect(matched(router, "HEAD", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(router.route("POST", "/notes")).toBeUndefined(); + expect(router.route("GET", "/nope")).toBeUndefined(); + }); + + // Opus's build review F2, Astra's: one explicit HEAD route anywhere hid every GET fallback. + it("falls back to GET per path when another path has an explicit HEAD route", () => { + const { router } = compose([ + "@app/a", + { routes: { "HEAD /download": route("head"), "GET /download": route("get"), "GET /notes": route("notes") } }, + ]); + expect(matched(router, "HEAD", "/download")).toMatchObject({ pattern: "HEAD /download" }); + expect(matched(router, "HEAD", "/notes")).toMatchObject({ pattern: "GET /notes" }); + }); + + // Opus's build review F3, Astra's: the first route's param name was used for every route at that position. + it("names each route's params by its own pattern", () => { + const { router } = compose(["@app/a", { routes: { "GET /:id/a": route("a"), "GET /:name/b": route("b") } }]); + expect(matched(router, "GET", "/v/a")).toEqual({ pattern: "GET /:id/a", params: { id: "v" } }); + expect(matched(router, "GET", "/v/b")).toEqual({ pattern: "GET /:name/b", params: { name: "v" } }); + }); + + it("takes a route with options: page and crossSite", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /p": { handler: route("p"), page: true }, + "POST /hook": { handler: route("h"), crossSite: true }, + }, + }, + ]); + expect(router.route("GET", "/p")).toMatchObject({ page: true, crossSite: false }); + expect(router.route("POST", "/hook")).toMatchObject({ page: false, crossSite: true }); + expect(router.route("GET", "/notes")).toBeUndefined(); + }); +}); + +describe("conflicts", () => { + it("reports one pattern from two contributions (also with other param names); the first by plugin name wins", () => { + const { router, conflicts } = compose( + ["@app/a", { routes: { "GET /x/:id": route("a") } }], + ["@app/b", { routes: { "GET /x/:key": route("b"), BAD: route("bad") }, sockets: { nope: () => {} } }], + ); + expect((router.route("GET", "/x/1") as RouteMatch).plugin).toBe("@app/a"); + expect(conflicts).toEqual([ + { message: 'route "GET /x/:key" is contributed twice', plugins: ["@app/a", "@app/b"] }, + { message: 'route "BAD" is no "METHOD /pattern"', plugins: ["@app/b"] }, + { message: 'socket "nope" is no pattern', plugins: ["@app/b"] }, + ]); + }); + + it("refuses a route for a method a Web Request cannot carry, reported", () => { + const { router, conflicts } = compose([ + "@app/a", + { routes: { "TRACE /x": route("t"), "CONNECT /y": route("c") } }, + ]); + expect(router.route("TRACE", "/x")).toBeUndefined(); + expect(conflicts).toEqual([ + { message: 'route "TRACE /x": TRACE is served for no resource (501)', plugins: ["@app/a"] }, + { message: 'route "CONNECT /y": CONNECT is served for no resource (501)', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("wraps (D2)", () => { + const wrap = (name: string, phase: HttpWrap["phase"], extra: Partial = {}): HttpWrap => ({ + name, + phase, + wrap: (request, next) => next(request), + ...extra, + }); + + it("orders by phase (outermost first), then contributor order; own wraps around their plugin's routes only", () => { + const { router } = compose( + [ + "@app/a", + { + routes: { "GET /a": route("a") }, + wraps: [wrap("a-transform", "transform"), wrap("a-observe", "observe")], + }, + ], + [ + "@app/b", + { + routes: { "GET /b": route("b") }, + wraps: [wrap("b-all", "headers", { scope: "all" }), wrap("b-own", "observe")], + }, + ], + ); + const names = (path: string) => (router.route("GET", path) as RouteMatch).wraps.map(({ wrap }) => wrap.name); + expect(names("/a")).toEqual(["a-observe", "b-all", "a-transform"]); + expect(names("/b")).toEqual(["b-own", "b-all"]); + }); + + it("narrows to one route; a route nobody contributes is a reported conflict", () => { + const { router, conflicts } = compose([ + "@app/a", + { + routes: { "GET /a": route("a"), "GET /b": route("b") }, + wraps: [wrap("only-a", "observe", { route: "GET /a" }), wrap("ghost", "observe", { route: "GET /c" })], + }, + ]); + expect((router.route("GET", "/a") as RouteMatch).wraps.map(({ wrap }) => wrap.name)).toEqual(["only-a"]); + expect((router.route("GET", "/b") as RouteMatch).wraps).toEqual([]); + expect(conflicts).toEqual([ + { message: 'wrap "ghost" names route "GET /c", which nobody contributes', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("hosts and sockets", () => { + it("collects hosts lowercased, framers, relays and CSP sources; matches sockets with their largest frame", () => { + const terminal = () => {}; + const relay = { name: "radius", verify: () => undefined }; + const contribution: HttpContribution = { + hosts: ["Notes.Example.com"], + sockets: { "/terminal/:id": terminal }, + maxPayload: 10, + framers: ["https://frame.example.com"], + relays: [relay], + csp: { "img-src": ["https://cdn.example.com"] }, + }; + const { router } = compose( + ["@app/a", contribution], + ["@app/b", { hosts: ["b.example.com"], csp: { "img-src": ["data:"] } }], + ); + expect([...router.hosts]).toEqual(["notes.example.com", "b.example.com"]); + expect([[...router.framers], router.relays, [...router.csp]]).toEqual([ + ["https://frame.example.com"], + [relay], + [["img-src", ["https://cdn.example.com", "data:"]]], + ]); + expect(router.socket("/terminal/7")).toMatchObject({ + pattern: "/terminal/:id", + plugin: "@app/a", + params: { id: "7" }, + maxPayload: 10, + }); + expect(router.socket("/terminal/..")).toBe("bad_path"); + expect(router.socket("/nope")).toBeUndefined(); + expect(router.socketHandlers.has(terminal)).toBe(true); + }); +}); diff --git a/packages/chord/test/next-node/http/support.ts b/packages/chord/test/next-node/http/support.ts new file mode 100644 index 000000000..8a4b82b59 --- /dev/null +++ b/packages/chord/test/next-node/http/support.ts @@ -0,0 +1,204 @@ +// Test support for the HTTP component (slice 10 §2): a server host with the component, the app's configuration and +// authenticator, and plugin facets contributing to `Http`; raw requests (any `Host`, any body) and WebSocket clients +// against its listener. + +import { request as httpRequest, type IncomingHttpHeaders } from "node:http"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Address } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Facet, type Host, type HostError } from "../../../src/next/host/index.ts"; +import { + type Authenticator, + Authenticators, + createHttp, + Http, + type HttpComponent, + type HttpContribution, + type HttpError, + type HttpOptions, +} from "../../../src/next-node/http/index.ts"; + +/** The principals of these tests. */ +export type TestPrincipal = + | { readonly kind: "server" } + | { readonly kind: "owner" } + | { readonly kind: "guest"; readonly id: string }; + +/** A running server: its host, its component, its port once loaded, what both reported. */ +export type Served = { + readonly host: Host; + readonly http: HttpComponent; + /** The listener's port (after `load`). */ + readonly port: number; + readonly errors: HttpError[]; + readonly hostErrors: HostError[]; + /** The core facets and the plugins, as loaded. */ + facets(plugins?: Readonly>>): { name: string; facet: Facet }[]; + close(): Promise; +}; + +/** A plugin facet contributing `contribution` to `Http`. */ +export function contributing(contribution: HttpContribution): Facet { + return defineFacet((_host, self) => self.contribute(Http, contribution)); +} + +/** + * Builds the component on a new server host (listening on loopback, a free port), then loads it with the app's + * authenticator (`@app/auth`: the `x-principal` header names one, else anonymous) and `plugins` (by name) unless + * `load` is false; `config` overrides the settings (`listen: undefined`: no listener). + */ +export async function served( + plugins: Readonly>> = {}, + options: { + readonly config?: Omit; + readonly authenticate?: Authenticator; + readonly load?: boolean; + readonly readyTimeout?: number; + readonly address?: Address; + } = {}, +): Promise }> { + const errors: HttpError[] = []; + const hostErrors: HostError[] = []; + const host = createHost({ + principal: { kind: "server" }, + onError: (error) => hostErrors.push(error), + ...(options.readyTimeout === undefined ? {} : { readyTimeout: options.readyTimeout }), + ...(options.address === undefined ? {} : { address: options.address }), + }); + const config: Omit = { listen: { host: "127.0.0.1", port: 0 }, ...options.config }; + const http = createHttp(host, { ...config, onError: (error) => errors.push(error) }, context); + const authenticate = options.authenticate ?? ((request) => principalOf(request.headers)); + const core = [ + { name: "chord.http", facet: http.facet }, + { + name: "@app/auth", + facet: defineFacet((_host, self) => self.contribute(Authenticators, authenticate)), + }, + ]; + const facets = (more = plugins) => [...core, ...Object.entries(more).map(([name, facet]) => ({ name, facet }))]; + let port = 0; + const load = async (): Promise => { + await host.load(facets(), context); + if (config.listen !== undefined) port = (await http.listening(context)).port; + }; + if (options.load !== false) await load(); + return { + host, + http, + get port() { + return port; + }, + errors, + hostErrors, + facets, + load, + close: async () => { + await http.close(context); + await host.close(context); + }, + }; +} + +/** The principal an `x-principal` header names: `owner`, `guest:`; none without it. */ +function principalOf(headers: Headers): TestPrincipal | undefined { + const named = headers.get("x-principal"); + if (named === "owner") return { kind: "owner" }; + if (named?.startsWith("guest:")) return { kind: "guest", id: named.slice(6) }; + return undefined; +} + +/** A response as a test reads it. */ +export type Answer = { readonly status: number; readonly headers: IncomingHttpHeaders; readonly body: Buffer }; + +/** + * A request to `port` with raw control: `Host` defaults to `127.0.0.1:`; `body` is sent as is; `signal` aborts it. + */ +export function send( + port: number, + method: string, + path: string, + options: { + readonly headers?: Readonly>; + readonly body?: string | Buffer; + readonly agent?: false; + } = {}, +): Promise { + return new Promise((resolve, reject) => { + const request = httpRequest( + { + host: "127.0.0.1", + port, + method, + path, + agent: false, + headers: { host: `127.0.0.1:${port}`, ...options.headers }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => + resolve({ status: response.statusCode ?? 0, headers: response.headers, body: Buffer.concat(chunks) }), + ); + response.on("error", reject); + }, + ); + request.on("error", reject); + request.end(options.body); + }); +} + +/** A WebSocket client to `path` on `port` (`Host` loopback), with `headers`; resolves once open, rejects when refused. */ +export function socketTo( + port: number, + path: string, + headers: Readonly> = {}, + options: { readonly maxPayload?: number } = {}, +): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(`ws://127.0.0.1:${port}${path}`, { headers, ...options }); + // Frames sent at once by the server arrive with the upgrade: queued from the start. + inboxes.set(socket, messagesOf(socket)); + socket.once("open", () => resolve(socket)); + socket.once("unexpected-response", (_request, response) => { + reject(new Error(`refused ${response.statusCode}`)); + socket.terminate(); + }); + socket.once("error", reject); + }); +} + +/** The messages of every socket `socketTo` opened, queued from its start. */ +const inboxes = new WeakMap }>(); + +/** The next text message of a socket `socketTo` opened. */ +export function nextMessage(socket: WebSocket): Promise { + return inboxes.get(socket)!.next(); +} + +/** The text messages `socket` receives, from now on, in order: `next()` waits for the next one. */ +export function messagesOf(socket: WebSocket): { next(): Promise } { + const queued: string[] = []; + const waiting: ((text: string) => void)[] = []; + socket.on("message", (data) => { + const text = String(data); + const waiter = waiting.shift(); + if (waiter === undefined) queued.push(text); + else waiter(text); + }); + return { + next: () => { + const text = queued.shift(); + return text === undefined ? new Promise((resolve) => waiting.push(resolve)) : Promise.resolve(text); + }, + }; +} + +/** The close code and reason of `socket`'s close. */ +export function closeOf(socket: WebSocket): Promise<{ readonly code: number; readonly reason: string }> { + return new Promise((resolve) => { + if (socket.readyState === socket.CLOSED) resolve({ code: 1006, reason: "" }); + socket.once("close", (code, reason) => resolve({ code, reason: reason.toString() })); + }); +} + +export { context }; diff --git a/packages/chord/test/next-node/http/upgrades.test.ts b/packages/chord/test/next-node/http/upgrades.test.ts new file mode 100644 index 000000000..f4de24124 --- /dev/null +++ b/packages/chord/test/next-node/http/upgrades.test.ts @@ -0,0 +1,458 @@ +// Tests of upgraded connections (slice 10 §2, D6, Q6; decisions "allowed hosts only through contributions", "plugin +// sockets get a heartbeat"): the links endpoint (`host.accept`, options per socket, refused sockets, the 1 MiB frame +// default, the output cap), plugin sockets (handed to their handler, an oversized frame closing only that socket, a +// throwing handler), connections terminated at once when their socket handler or host leaves (a raw peer can act no +// more), an upgrade checked again after authenticating, the heartbeat, close bounded for a peer that never answers, +// and nothing kept of a closed socket or its `ws` server. + +import { once } from "node:events"; +import { connect, type Socket } from "node:net"; +import { describe, expect, it, vi } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { arr, hostKind, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { webSocketLink } from "../../../src/next/link/index.ts"; +import { Http, type HttpContribution } from "../../../src/next-node/http/index.ts"; +import { limitedSocket } from "../../../src/next-node/http/upgrades.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { closeOf, context, contributing, nextMessage, send, served, socketTo, type TestPrincipal } from "./support.ts"; + +const Server = hostKind("server"); + +/** Who the server sees calling: served at the server's address. */ +const Who = remote("test.who", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { who: method(undefined, str()) } } }, +}); + +const serveWho = defineFacet((host) => + host.serve(Who, () => ({ + variant: "main", + members: { who: async (ctx) => JSON.stringify(ctx.principal ?? null) }, + })), +); + +/** A plugin socket that echoes text, with its handler's request params and principal sent first. */ +const echo = contributing({ + sockets: { + "/echo/:room": (socket, request, ctx) => { + socket.send(`${request.params.room} as ${ctx.principal?.kind ?? "nobody"}`); + socket.on("message", (data) => socket.send(String(data))); + }, + }, + maxPayload: 64, +}); + +describe("the links endpoint", () => { + it("accepts a link as the authenticated principal, with the options links.options gave that socket", async () => { + const options: unknown[] = []; + const server = await served( + { "@app/who": serveWho }, + { + address: Server(), + config: { + links: { + path: "/chord", + options: (request, principal) => { + options.push([request.path, principal]); + return { maxPayload: 1024 * 1024 }; + }, + }, + }, + }, + ); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + // The link wraps the socket at once: the server's `hello` comes with the upgrade. + const socket = new WebSocket(`ws://127.0.0.1:${server.port}/chord`, { headers: { "x-principal": "owner" } }); + tab.connect(webSocketLink(socket as never), { principal: { kind: "server" } }, context); + const who = await tab.get(Who, {}, context); + expect(await who.who(context)).toBe('{"kind":"owner"}'); + expect(options).toEqual([["/chord", { kind: "owner" }]]); + await tab.close(context); + await server.close(); + }); + + it("refuses a socket without a principal, from another Origin or Host, foreign, or whose links.options throws (403)", async () => { + const server = await served( + { "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { + config: { + links: { + path: "/chord", + options: (request) => { + if (request.headers.has("x-fail")) throw new Error("options"); + return {}; + }, + }, + }, + }, + ); + const { port } = server; + const owner = { "x-principal": "owner" }; + const refusals = [ + socketTo(port, "/chord"), + socketTo(port, "/chord", { ...owner, origin: "http://evil.com" }), + socketTo(port, "/chord", { ...owner, origin: "null" }), + socketTo(port, "/chord", { ...owner, origin: `HTTP://127.0.0.1:${port}` }), + socketTo(port, "/chord", { ...owner, host: "evil.com" }), + socketTo(port, "/chord", { ...owner, "sec-fetch-site": "cross-site" }), + socketTo(port, "/chord", { ...owner, "x-fail": "1" }), + ]; + for (const refusal of refusals) await expect(refusal).rejects.toThrow("refused 403"); + const same = await socketTo(port, "/chord", { ...owner, origin: `http://127.0.0.1:${port}` }); + same.close(); + expect(server.errors).toEqual([{ kind: "hook_failed", hook: "links.options", error: new Error("options") }]); + await server.close(); + }); + + it("takes frames of at most 1 MiB unless links.options says otherwise (Opus's audit H2)", async () => { + const server = await served({}, { config: { links: { path: "/chord" } } }); + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.send(Buffer.alloc(1024 * 1024 + 1)); + expect((await closed).code).toBe(1009); + await server.close(); + }); + + it("cuts off a client that stopped reading: past 4 MiB unsent beyond the frame being sent", () => { + const sent: number[] = []; + let terminated = 0; + const fake = { + OPEN: 1, + readyState: 1, + bufferedAmount: 0, + binaryType: "nodebuffer", + send: (data: Uint8Array) => void sent.push(data.byteLength), + terminate: () => void terminated++, + }; + const limited = limitedSocket(fake as never); + limited.send(new Uint8Array(10)); + fake.bufferedAmount = 5 * 1024 * 1024; + limited.send(new Uint8Array(10)); + expect([sent, terminated]).toEqual([[10], 1]); + }); +}); + +describe("plugin sockets", () => { + it("hands the socket and its request to the handler in a scope with its principal; no principal is refused", async () => { + const server = await served({ "@app/echo": echo }); + await expect(socketTo(server.port, "/echo/lobby")).rejects.toThrow("refused 403"); + await expect(socketTo(server.port, "/nope", { "x-principal": "owner" })).rejects.toThrow("refused 404"); + const socket = await socketTo(server.port, "/echo/lobby", { "x-principal": "owner" }); + expect(await nextMessage(socket)).toBe("lobby as owner"); + socket.send("hi"); + expect(await nextMessage(socket)).toBe("hi"); + socket.close(); + await server.close(); + }); + + it("an oversized frame closes only that socket (1009), never the server", async () => { + const server = await served({ "@app/echo": echo }); + const [big, other] = await Promise.all([ + socketTo(server.port, "/echo/a", { "x-principal": "owner" }), + socketTo(server.port, "/echo/b", { "x-principal": "owner" }), + ]); + await Promise.all([nextMessage(big), nextMessage(other)]); + const closed = closeOf(big); + big.send("x".repeat(100)); + expect((await closed).code).toBe(1009); + other.send("still here"); + expect(await nextMessage(other)).toBe("still here"); + other.close(); + await server.close(); + }); + + it("a throwing handler is reported socket_failed and closes its socket (1011)", async () => { + const server = await served({ + "@app/bad": contributing({ + sockets: { + "/bad": () => { + throw new Error("handler"); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/bad", { "x-principal": "owner" }); + expect((await closeOf(socket)).code).toBe(1011); + expect(server.errors).toMatchObject([{ kind: "socket_failed", pattern: "/bad", plugin: "@app/bad" }]); + await server.close(); + }); +}); + +describe("revocation", () => { + it("terminates a plugin's sockets at once when a reload removes its contribution, sealing their scopes; others stay (Q6)", async () => { + let signal: AbortSignal | undefined; + const removed = contributing({ + sockets: { + "/removed": (_socket, _request, ctx) => { + signal = ctx.abortSignal; + }, + }, + }); + const other = contributing({ sockets: { "/other": () => {} } }); + const server = await served({ "@app/removed": removed, "@app/other": other }); + const doomed = await socketTo(server.port, "/removed", { "x-principal": "owner" }); + const kept = await socketTo(server.port, "/other", { "x-principal": "owner" }); + const closed = closeOf(doomed); + await server.host.reload(server.facets({ "@app/other": other }), context); + expect(signal?.aborted).toBe(true); + expect((await closed).code).toBe(1006); + expect(kept.readyState).toBe(kept.OPEN); + kept.close(); + await server.close(); + }); + + it("terminates links and sockets through a host no longer contributed; loopback ones stay", async () => { + const server = await served( + { "@app/echo": echo, "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { config: { links: { path: "/chord" } } }, + ); + const through = { "x-principal": "owner", host: "a.example.com" }; + const link = await socketTo(server.port, "/chord", through); + const socket = await socketTo(server.port, "/echo/a", through); + const loopback = await socketTo(server.port, "/echo/b", { "x-principal": "owner" }); + const closes = [closeOf(link), closeOf(socket)]; + await server.host.reload(server.facets({ "@app/echo": echo }), context); + expect((await Promise.all(closes)).map(({ code }) => code)).toEqual([1006, 1006]); + expect(loopback.readyState).toBe(loopback.OPEN); + expect((await send(server.port, "GET", "/", { headers: { host: "a.example.com" } })).status).toBe(403); + loopback.close(); + await server.close(); + }); + + it("keeps a writable contribution's own sockets when only its hosts change", async () => { + const Tunnel = local("app.tunnel", { version: 1, params: {}, members: { name: method(arr(str())) } }); + const handler = (): void => {}; + const tunnel = defineFacet((host) => + host.serve(Tunnel, (_, self) => { + const contribution = self.writable({ hosts: [], sockets: { "/own": handler } }); + self.contribute(Http, contribution); + return { + name: async (hosts) => { + contribution.set({ hosts, sockets: { "/own": handler } }); + return null; + }, + }; + }), + ); + const server = await served({ "@app/tunnel": tunnel }); + const handle = await server.host.get(Tunnel, {}, context); + const socket = await socketTo(server.port, "/own", { "x-principal": "owner" }); + await handle.name(["t.example.com"], context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(socket.readyState).toBe(socket.OPEN); + socket.close(); + await server.close(); + }); + + // Astra's security audit A1: a policy close left the peer able to invoke the plugin's handlers. + it("a raw peer whose host or contribution left can send nothing more to the handler", async () => { + const calls: string[] = []; + const sockets = contributing({ + sockets: { "/echo": (socket) => void socket.on("message", (data) => calls.push(String(data))) }, + }); + const tunnel = contributing({ hosts: ["t.example"] }); + const server = await served({ "@app/sockets": sockets, "@app/tunnel": tunnel }); + const throughHost = await rawSocket(server.port, "t.example", "/echo"); + const local = await rawSocket(server.port, `localhost:${server.port}`, "/echo"); + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + throughHost.write(frame("after the host left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.host.reload(server.facets({}), context); + local.write(frame("after the contribution left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(calls).toEqual([]); + throughHost.destroy(); + local.destroy(); + await server.close(); + }); + + // Astra's security audit A2: an upgrade authenticating across the removal of its host was still handed out. + it("checks an upgrade again after authenticating: one whose host left meanwhile is refused", async () => { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + const sockets = contributing({ sockets: { "/echo": () => {} } }); + const server = await served( + { "@app/sockets": sockets, "@app/tunnel": contributing({ hosts: ["t.example"] }) }, + { + authenticate: async () => { + started.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const opening = socketTo(server.port, "/echo", { host: "t.example" }); + await started.promise; + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + gate.resolve(); + await expect(opening).rejects.toThrow("refused 403"); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Astra's recheck R1, Sol's N5: a client gone while its upgrade authenticated left that work running. + // Fable's second recheck: only a reset (RST) was seen; a clean disconnect (FIN, a tab closing) was not. + it.each(["ends its connection", "resets its connection"] as const)( + "aborts an upgrade's authentication when its client %s", + async (how) => { + const entered = Promise.withResolvers(); + let signal: AbortSignal | undefined; + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { + authenticate: async (_request, ctx) => { + signal = ctx.abortSignal; + entered.resolve(); + return new Promise(() => {}); + }, + }, + ); + const socket = connect(server.port, "127.0.0.1"); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write( + `GET /s HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n`, + ); + await entered.promise; + const aborted = new Promise((resolve) => signal?.addEventListener("abort", resolve, { once: true })); + if (how === "ends its connection") socket.end(); + else socket.resetAndDestroy(); + await aborted; + expect(signal?.aborted).toBe(true); + socket.destroy(); + await server.close(); + }, + ); +}); + +describe("heartbeat", () => { + it("pings a plugin socket after 15 s of silence and terminates it 15 s later without an answer", async () => { + const server = await served({ + "@app/quiet": contributing({ + sockets: { "/quiet": (socket) => void socket.on("message", (data) => socket.send(data)) }, + }), + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const silent = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + autoPong: false, + }); + const answering = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + }); + await Promise.all([once(silent, "open"), once(answering, "open")]); + const pinged = Promise.all([once(silent, "ping"), once(answering, "ping")]); + await vi.advanceTimersByTimeAsync(15_000); + await pinged; + // The answering client's pong travels for real (only timers are fake): an echo after it shows it arrived. + const echoed = once(answering, "message"); + answering.send("after the pong"); + await echoed; + const closed = closeOf(silent); + await vi.advanceTimersByTimeAsync(15_000); + expect((await closed).code).toBe(1006); + expect(answering.readyState).toBe(answering.OPEN); + answering.close(); + } finally { + vi.useRealTimers(); + } + await server.close(); + }); +}); + +describe("close", () => { + // Sol's build review: close waited on a peer that never acknowledged its close frame. + it("terminates a socket that never acknowledges its close at closeTimeout", async () => { + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { config: { closeTimeout: 20 } }, + ); + const raw = await rawSocket(server.port, `127.0.0.1:${server.port}`, "/s"); + raw.pause(); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + raw.destroy(); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's build review: a `ws` server was kept per frame size for the component's life. + it("keeps no ws server once its socket closed", async () => { + const refs: WeakRef[] = []; + const original = WebSocketServer.prototype.handleUpgrade; + WebSocketServer.prototype.handleUpgrade = function (this: WebSocketServer, ...args: Parameters) { + refs.push(new WeakRef(this)); + return original.apply(this, args); + }; + let maxPayload = 1024; + const server = await served( + {}, + { config: { links: { path: "/chord", options: () => ({ maxPayload: maxPayload++ }) } } }, + ); + try { + for (let i = 0; i < 8; i++) { + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + } + await settledHeap(); + await expectCollected(...refs); + } finally { + WebSocketServer.prototype.handleUpgrade = original; + await server.close(); + } + }); + + it("keeps nothing of a closed plugin socket", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/kept": contributing({ + sockets: { + "/kept": (socket, request, ctx) => { + refs.push(new WeakRef(socket), new WeakRef(request), new WeakRef(ctx)); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/kept", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + await new Promise((resolve) => setTimeout(resolve, 50)); + await expectCollected(...refs); + await server.close(); + }); +}); + +/** A raw TCP client upgraded to the plugin socket `path` through `host` as the owner. */ +async function rawSocket(port: number, host: string, path: string): Promise { + const socket = connect(port, "127.0.0.1"); + await once(socket, "connect"); + socket.on("error", () => {}); + socket.write( + `GET ${path} HTTP/1.1\r\nHost: ${host}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: AAAAAAAAAAAAAAAAAAAAAA==\r\nSec-WebSocket-Version: 13\r\nx-principal: owner\r\n\r\n`, + ); + const [answer] = await once(socket, "data"); + expect(String(answer)).toContain("101 Switching Protocols"); + return socket; +} + +/** A masked client text frame. */ +function frame(text: string): Buffer { + const bytes = Buffer.from(text); + const mask = Buffer.from([1, 2, 3, 4]); + return Buffer.concat([ + Buffer.from([0x81, 0x80 | bytes.length]), + mask, + Buffer.from(bytes.map((byte, index) => byte ^ mask[index % 4]!)), + ]); +} diff --git a/packages/chord/test/next-node/plugins/build.test.ts b/packages/chord/test/next-node/plugins/build.test.ts new file mode 100644 index 000000000..7224734ba --- /dev/null +++ b/packages/chord/test/next-node/plugins/build.test.ts @@ -0,0 +1,685 @@ +// Tests of building (slice 7c §2): every row of the import table, failures as results, source maps relative to the +// plugin and digests stable across two checkout paths, inputs hashed as esbuild read them. + +import { createHash } from "node:crypto"; +import { mkdir, readFile, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuildOptions, + type BuiltFacet, + buildFacet, + discoverPlugins, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +const NODE: BuildOptions = { hostPackages: ["@earendil-works/chord/next"], platform: "node" }; +const BROWSER: BuildOptions = { + hostPackages: ["@earendil-works/chord/next", "react/jsx-runtime"], + platform: "browser", + publicPath: "/assets/", +}; + +/** The plugins under `root`, and the one named `name`. */ +async function discovered(root: string, name: string): Promise<{ plugins: readonly Plugin[]; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + return { plugins, plugin: plugins.find((each) => each.name === name)! }; +} + +/** `built`'s script, failing the test with its errors when it failed. */ +function script(built: BuiltFacet): Extract { + if (built.kind === "failed") throw new Error(`build failed: ${built.errors.join("\n")}`); + return built; +} + +/** The errors of a build that must fail. */ +function errors(built: BuiltFacet): readonly string[] { + if (built.kind !== "failed") throw new Error("the build succeeded"); + return built.errors; +} + +/** The sources of `built`'s inline source map. */ +function sourcesOf(built: Extract): string[] { + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + return JSON.parse(Buffer.from(inline, "base64").toString()).sources; +} + +describe("building a facet: the import table", () => { + it("inlines the plugin's own files and another plugin's file (copied); keeps host packages external", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + const real = await realpath(root); + expect(built).toMatchObject({ plugin: "@test/notes", facet: "worker", assets: [] }); + expect(built.source.startsWith("(function (require, module, exports) {")).toBe(true); + expect(built.source).toContain('require("@earendil-works/chord/next")'); + expect(built.copied).toEqual([join(real, "contracts", "contract.ts")]); + expect(built.inputs.map(({ file }) => file)).toEqual([ + join(real, "contracts", "contract.ts"), + join(real, "notes", "label.ts"), + join(real, "notes", "worker.ts"), + ]); + // Relative to the plugin (a file whose code minified away maps nothing and is left out). + expect(sourcesOf(built)).toEqual(expect.arrayContaining(["worker.ts", "../contracts/contract.ts"])); + for (const source of sourcesOf(built)) expect(source.startsWith("/")).toBe(false); + // Positions only: no copy of the sources' text in the script (Opus N3). + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + expect(JSON.parse(Buffer.from(inline, "base64").toString())).not.toHaveProperty("sourcesContent"); + }); + + it("hashes each input as esbuild read it; the digest is the source's hash", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + for (const { file, hash } of built.inputs) { + expect(hash).toBe( + createHash("sha256") + .update(await readFile(file)) + .digest("hex"), + ); + } + expect(built.digest).toBe(createHash("sha256").update(built.source).digest("hex")); + }); + + it("gives the same script and digest from two checkout paths (relative source maps)", async () => { + const first = await tempRoot(["contracts", "notes"]); + const second = await tempRoot(["contracts", "notes"]); + const builds = await Promise.all( + [first, second].map(async (root) => { + const { plugins, plugin } = await discovered(root, "@test/notes"); + return script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + }), + ); + expect(builds[0]!.digest).toBe(builds[1]!.digest); + expect(builds[0]!.source).toBe(builds[1]!.source); + }); + + it("inlines a package of its dependencies; keeps chord.external packages and Node builtins external on Node", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "deps", + "@test/deps", + { + "worker.ts": [ + 'import { pad } from "pad-dep";', + 'import native from "native-dep";', + 'import { join } from "node:path";', + 'import { readFileSync } from "fs";', + 'import { pluginFile } from "@earendil-works/chord/next/plugins";', + "export default { setup: () => [pad, native, join, readFileSync, pluginFile] };", + ].join("\n"), + }, + { dependencies: { "pad-dep": "1.0.0" }, chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "pad-dep", { "index.js": 'exports.pad = (s) => "pad:" + s;' }); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/deps"); + const built = script(await buildFacet(plugin, "worker", NODE, context)); + expect(built.source).toContain('"pad:"'); + for (const external of ["native-dep", "node:path", "fs", "@earendil-works/chord/next/plugins"]) { + expect(built.source).toContain(`require("${external}")`); + } + // Packages are no inputs: the watcher ignores node_modules. + expect(built.inputs.map(({ file }) => file)).toEqual([join(await realpath(dir), "worker.ts")]); + }); + + it("refuses a chord.external package and a Node builtin in a browser build", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "web", + "@test/web", + { + "web.ts": + 'import native from "native-dep";\nimport { join } from "node:path";\nexport default { setup: () => [native, join] };', + }, + { chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/web"); + expect(errors(await buildFacet(plugin, "web", BROWSER, context))).toEqual([ + 'web.ts:1:20: "native-dep": chord.external packages are for Node builds', + 'web.ts:2:22: "node:path": a Node builtin in a browser build', + ]); + }); + + it("collects a browser build's CSS into one stylesheet and its assets into content-hashed files", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.tsx": [ + 'import "./notes.css";', + 'import iconUrl from "./assets/icon.svg?url";', + "const Icon = () => ;", + "export default { setup: () => Icon };", + ].join("\n"), + "notes.css": ".notes { color: red; }", + "assets/icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + expect(built.css?.name).toMatch(/^web-[A-Z0-9]+\.css$/); + expect(built.css?.text).toContain(".notes{color:red}"); + expect(built.assets.map(({ name }) => name)).toEqual([expect.stringMatching(/^icon-[A-Z0-9]+\.svg$/)]); + expect(new TextDecoder().decode(built.assets[0]!.bytes)).toBe(""); + expect(built.source).toContain(`"/assets/${built.assets[0]!.name}"`); + expect(built.source).toContain('require("react/jsx-runtime")'); + }); + + it("refuses CSS and assets in a Node build", async () => { + const root = await tempRoot(); + await writePlugin(root, "n", "@test/n", { + "worker.ts": 'import "./a.css";\nimport url from "./icon.svg?url";\nexport default { setup: () => url };', + "a.css": "a{}", + "icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/n"); + const found = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(found).toEqual([ + 'worker.ts:1:8: "./a.css": stylesheets and assets are for browser builds', + expect.stringMatching(/^worker\.ts:2:17: "\.\/icon\.svg\?url": assets by URL are for browser builds/), + ]); + }); + + it("refuses import.meta.url, __dirname, __filename, a dynamic import() and top-level await, pointing to pluginFile", async () => { + const root = await tempRoot(); + await writePlugin(root, "bad", "@test/bad", { + "worker.ts": "export const a = import.meta.url;\nexport default { setup: () => [__dirname, __filename] };", + "server.ts": 'export default { setup: () => import("./other.ts") };', + "web.ts": "await Promise.resolve();\nexport default { setup: () => {} };", + "other.ts": "export const x = 1;", + }); + const { plugin } = await discovered(root, "@test/bad"); + const meta = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(meta.length).toBeGreaterThan(0); + for (const error of meta) + expect(error).toMatch(/^worker\.ts:\d+:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/); + // Without `import.meta` in the file: each free `__dirname` and `__filename` at its own line (example 37). + await writeFiles(join(root, "bad"), { + "worker.ts": "// The globals.\nconsole.log(__dirname);\nexport default { setup: () => __filename };", + }); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:2:13: import\.meta\.url, __dirname and __filename .*pluginFile/), + expect.stringMatching(/^worker\.ts:3:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/), + ]); + expect(errors(await buildFacet(plugin, "server", NODE, context))).toEqual([ + expect.stringMatching( + /^server\.ts:1:\d+: import\("\.\/other\.ts"\): a plugin script has no dynamic import .*pluginFile/, + ), + ]); + expect(errors(await buildFacet(plugin, "web", NODE, context))).toEqual([ + "web.ts:1:1: a plugin script has no top-level await (a facet declares synchronously)", + ]); + }); + + it("refuses a path leaving the root, a file in no plugin, a package not in its dependencies, a package in a copied file", async () => { + const root = await tempRoot(["contracts"]); + const away = await tempRoot(); + await writeFiles(away, { "far.ts": "export const far = 1;" }); + await writePlugin(root, "out", "@test/out", { + "worker.ts": [ + 'import { base } from "../base.ts";', + `import { far } from ${JSON.stringify(join(away, "far.ts"))};`, + 'import x from "left-out";', + "export default { setup: () => [base, far, x] };", + ].join("\n"), + }); + await writeFiles(root, { "base.ts": "export const base = 1;" }); + await writePlugin(root, "copier", "@test/copier", { + "worker.ts": 'import { y } from "../shared/shared.ts";\nexport default { setup: () => y };', + }); + await writePlugin(root, "shared", "@test/shared", { + "shared.ts": 'import y from "some-package";\nexport { y };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const out = plugins.find(({ name }) => name === "@test/out")!; + expect(errors(await buildFacet(out, "worker", { ...NODE, plugins }, context))).toEqual([ + 'worker.ts:1:22: "../base.ts": not a file of a plugin in this root', + expect.stringMatching( + /^worker\.ts:2:21: ".*far\.ts": outside the plugin's root \(base code is reached through a host package, other repositories' plugins through dependencies\)$/, + ), + 'worker.ts:3:15: "left-out": neither a host package nor in the dependencies of plugin "@test/out"', + ]); + const copier = plugins.find(({ name }) => name === "@test/copier")!; + expect(errors(await buildFacet(copier, "worker", { ...NODE, plugins }, context))).toEqual([ + '../shared/shared.ts:1:15: "some-package": a file copied from plugin "@test/shared" may import only host packages and files of its own plugin', + ]); + }); + + it("inlines another plugin's files of the same root by relative path; refuses one of another root and names by plugin name", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(first, "contracts", "@test/contracts", { + "contract.ts": 'export const name = "contract";', + "other.ts": 'import { name } from "./contract.ts";\nexport const other = name;', + }); + await writePlugin(second, "elsewhere", "@test/elsewhere", { "contract.ts": 'export const name = "far";' }); + await writePlugin(first, "user", "@test/user", { + "worker.ts": [ + 'import { name } from "../contracts/contract.ts";', + 'import { other } from "../contracts/other.ts";', + "export default { setup: () => [name, other] };", + ].join("\n"), + }); + await writePlugin(first, "far", "@test/far", { + "worker.ts": `import { name } from ${JSON.stringify(join(second, "elsewhere", "contract.ts"))};\nexport default { setup: () => name };`, + }); + await writePlugin(first, "named", "@test/named", { + "worker.ts": 'import { name } from "@test/contracts/contract";\nexport default { setup: () => name };', + }); + const { plugins } = await discoverPlugins([first, second], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, plugins }, context); + const real = await realpath(first); + expect(script(await build("@test/user")).copied).toEqual([ + join(real, "contracts", "contract.ts"), + join(real, "contracts", "other.ts"), + ]); + expect(errors(await build("@test/far"))).toEqual([ + expect.stringMatching( + /^worker\.ts:1:22: ".*": outside the plugin's root \(base code is reached through a host package/, + ), + ]); + expect(errors(await build("@test/named"))).toEqual([ + 'worker.ts:1:22: "@test/contracts/contract": neither a host package nor in the dependencies of plugin "@test/named"', + ]); + }); + + it("returns a syntax error and a missing facet as failed results, never a throw", async () => { + const root = await tempRoot(); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + const { plugin } = await discovered(root, "@test/syntax"); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:1:\d+: Expected /), + ]); + expect(await buildFacet(plugin, "server", NODE, context)).toEqual({ + kind: "failed", + plugin: "@test/syntax", + facet: "server", + errors: ['plugin "@test/syntax" has no server facet'], + }); + }); +}); + +/** The app's API of slice 12's tests: two versions, a shared module and per-kind modules. */ +const API = { name: "@acme/app", versions: ["v3", "v2"] } as const; + +/** Each kind's host packages with the API: v3's shared and kind modules, and v2. */ +function apiKind(kind: string): BuildOptions { + return { + hostPackages: ["@acme/app/v3", `@acme/app/v3/${kind}`, "@acme/app/v2"], + platform: kind === "web" ? "browser" : "node", + api: API, + kinds: KINDS, + }; +} + +describe("building a facet: the app's API (slice 12 §1)", () => { + it("keeps a listed version and the kind's module external, and the plugins component's pluginFile", async () => { + const root = await tempRoot(); + await writePlugin(root, "good", "@test/good", { + "worker.ts": [ + 'import { defineFacet } from "@acme/app/v3";', + 'import { Tools } from "@acme/app/v3/worker";', + 'import { pluginFile } from "@earendil-works/chord/next/plugins";', + "export default defineFacet(() => [Tools, pluginFile]);", + ].join("\n"), + }); + const { plugin } = await discovered(root, "@test/good"); + const built = script(await buildFacet(plugin, "worker", apiKind("worker"), context)); + expect(built.source).toContain('require("@acme/app/v3/worker")'); + }); + + it("refuses an unlisted version, no version, another kind's module and Chord itself", async () => { + const root = await tempRoot(); + await writePlugin(root, "bad", "@test/bad", { + "worker.ts": [ + 'import { a } from "@acme/app/v1";', + 'import { b } from "@acme/app";', + 'import { c } from "@acme/app/v3/web";', + 'import { d } from "@earendil-works/chord/next";', + "export default { setup: () => [a, b, c, d] };", + ].join("\n"), + }); + const { plugin } = await discovered(root, "@test/bad"); + // Chord is refused even when the app lists it as a host package. + const options = { + ...apiKind("worker"), + hostPackages: [...apiKind("worker").hostPackages, "@earendil-works/chord/next"], + }; + expect(errors(await buildFacet(plugin, "worker", options, context))).toEqual([ + 'worker.ts:1:19: "@acme/app/v1": @acme/app/v1 is not available: this app provides v3, v2', + 'worker.ts:2:19: "@acme/app": import a version of the app\'s API (this app provides v3, v2)', + 'worker.ts:3:19: "@acme/app/v3/web": not available to worker facets', + 'worker.ts:4:19: "@earendil-works/chord/next": plugins import Chord through the app\'s API ("@acme/app/v3"), never directly', + ]); + }); + + it("refuses two versions in one facet, naming both with their importers", async () => { + const root = await tempRoot(); + await writePlugin(root, "mixed", "@test/mixed", { + "worker.ts": + 'import { a } from "@acme/app/v3";\nimport { b } from "./old.ts";\nexport default { setup: () => [a, b] };', + "old.ts": 'export { b } from "@acme/app/v2";', + }); + const { plugin } = await discovered(root, "@test/mixed"); + expect(errors(await buildFacet(plugin, "worker", apiKind("worker"), context))).toEqual([ + "one facet imports one version of @acme/app: this one imports @acme/app/v2 (old.ts) and @acme/app/v3 (worker.ts)", + ]); + }); + + it("names the app's API in the refusal of a path leaving the root", async () => { + const root = await tempRoot(); + const away = await tempRoot(); + await writeFiles(away, { "base.ts": "export const base = 1;" }); + await writePlugin(root, "out", "@test/out", { + "worker.ts": `import { base } from ${JSON.stringify(join(away, "base.ts"))};\nexport default { setup: () => base };`, + }); + const { plugin } = await discovered(root, "@test/out"); + expect(errors(await buildFacet(plugin, "worker", apiKind("worker"), context))).toEqual([ + expect.stringMatching(/base code is reached through the app's API \("@acme\/app\/v3"\)/), + ]); + }); +}); + +describe("building a facet: per-kind files (slice 12 §3)", () => { + it("allows a file of the facet's own kind; refuses one of another kind, naming the chain that reached it", async () => { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { + "contract.ts": 'export const shared = "shared";', + "contract.worker.ts": 'export const tools = "tools";', + "contract.web.tsx": 'export const panel = "panel";', + "view.ts": 'export { panel } from "./contract.web.tsx";', + "worker.ts": [ + 'import { shared } from "./contract.ts";', + 'import { tools } from "./contract.worker.ts";', + "export default { setup: () => [shared, tools] };", + ].join("\n"), + "server.ts": 'import { panel } from "./view.ts";\nexport default { setup: () => panel };', + }); + const { plugins, plugin } = await discovered(root, "@test/notes"); + script(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context)); + expect(errors(await buildFacet(plugin, "server", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'view.ts:1:23: "./contract.web.tsx": a web file, imported into a server facet (server.ts → view.ts → contract.web.tsx)', + ]); + }); + + it("checks another plugin's copied files; never a package's file names (pdfjs's pdf.worker.mjs)", async () => { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { "contract.web.tsx": 'export const panel = "panel";' }); + const user = await writePlugin( + root, + "user", + "@test/user", + { + "server.ts": 'import { panel } from "../notes/contract.web.tsx";\nexport default { setup: () => panel };', + "worker.ts": + 'import { list } from "@bob/notes/web";\nimport pdf from "pdfjs-like";\nexport default { setup: () => [list, pdf] };', + }, + { dependencies: { "@bob/notes": "1.0.0", "pdfjs-like": "1.0.0" } }, + ); + await writePackage( + user, + "@bob/notes", + { "contract.ts": "export const list = 1;", "contract.web.tsx": "export const list = 2;" }, + { exports: { ".": "./contract.ts", "./web": "./contract.web.tsx" } }, + ); + await writePackage( + user, + "pdfjs-like", + { + "index.mjs": 'export { default } from "./build/pdf.worker.mjs";', + "build/pdf.worker.mjs": "export default 1;", + }, + { type: "module", main: "index.mjs" }, + ); + const { plugins, plugin } = await discovered(root, "@test/user"); + const options = { ...NODE, kinds: KINDS, plugins }; + expect(errors(await buildFacet(plugin, "server", options, context))).toEqual([ + 'server.ts:1:23: "../notes/contract.web.tsx": a web file, imported into a server facet (server.ts → ../notes/contract.web.tsx)', + ]); + script(await buildFacet(plugin, "worker", options, context)); + }); + + it("refuses another kind's facet file and a per-kind name of any case (Opus L1, M2)", async () => { + const root = await tempRoot(); + await writePlugin(root, "f", "@test/f", { + "worker.ts": + 'import { a } from "./web.tsx";\nimport { b } from "./contract.Web.ts";\nexport default { setup: () => [a, b] };', + "web.tsx": "export const a = 1;", + "contract.Web.ts": "export const b = 2;", + }); + const { plugins, plugin } = await discovered(root, "@test/f"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'worker.ts:1:19: "./web.tsx": a web file, imported into a worker facet (worker.ts → web.tsx)', + 'worker.ts:2:19: "./contract.Web.ts": a web file, imported into a worker facet (worker.ts → contract.Web.ts)', + ]); + }); + + it("names the chain of an import cycle without hanging (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "cycle", "@test/cycle", { + "worker.ts": 'import { value } from "./a.ts";\nexport default { setup: () => value };', + "a.ts": 'import "./worker.ts";\nexport { value } from "./bridge.ts";', + "bridge.ts": 'export { value } from "./bad.web.ts";', + "bad.web.ts": "export const value = 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/cycle"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'bridge.ts:1:23: "./bad.web.ts": a web file, imported into a worker facet (worker.ts → a.ts → bridge.ts → bad.web.ts)', + ]); + }); +}); + +describe("building a facet: roots under node_modules and plugin packages (slice 12 fix round 2: Opus N1, N2)", () => { + it("builds a plugin of a root that lies under a node_modules (an app installed from npm with its plugins)", async () => { + const app = await tempRoot(); + const root = join(app, "node_modules", "@acme", "app", "plugins"); + await writePlugin(root, "notes", "@acme/notes", { + "worker.ts": + 'import { x } from "./helper.ts";\nimport { c } from "../contracts/contract.ts";\nexport default { setup: () => [x, c] };', + "helper.ts": "export const x = 1;", + }); + await writePlugin(root, "contracts", "@acme/contracts", { "contract.ts": "export const c = 1;" }); + const { plugins, plugin } = await discovered(root, "@acme/notes"); + expect(script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)).copied).toEqual([ + join(await realpath(root), "contracts", "contract.ts"), + ]); + }); + + it("checks per-kind files in a package that is plugin code (a chord field), not in a library", async () => { + const root = await tempRoot(); + const user = await writePlugin( + root, + "user", + "@test/user", + { + "worker.ts": + 'import { panel } from "@bob/plugins/notes/contract.web.tsx";\nexport default { setup: () => panel };', + "server.ts": 'import { a } from "@bob/plugins/notes/web.tsx";\nexport default { setup: () => a };', + }, + { dependencies: { "@bob/plugins": "1.0.0" } }, + ); + await writeFiles(join(user, "node_modules", "@bob", "plugins"), { + "package.json": JSON.stringify({ name: "@bob/plugins", chord: { plugins: ["notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "notes/contract.web.tsx": 'export const panel = "panel";', + "notes/web.tsx": "export const a = 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/user"); + const options = { ...NODE, kinds: KINDS, plugins }; + expect(errors(await buildFacet(plugin, "worker", options, context))).toEqual([ + 'worker.ts:1:23: "@bob/plugins/notes/contract.web.tsx": a web file, imported into a worker facet (worker.ts → node_modules/@bob/plugins/notes/contract.web.tsx)', + ]); + expect(errors(await buildFacet(plugin, "server", options, context))).toEqual([ + 'server.ts:1:19: "@bob/plugins/notes/web.tsx": a web file, imported into a server facet (server.ts → node_modules/@bob/plugins/notes/web.tsx)', + ]); + }); +}); + +describe("building a facet: libraries inside plugin packages (slice 12 fix round 3: Opus R2)", () => { + it("leaves a library unchecked by kind, also one nested in a plugin package's own node_modules", async () => { + const root = await tempRoot(); + const user = await writePlugin( + root, + "p", + "@test/p", + { "server.ts": 'import x from "@bob/plugins/notes/contract.ts";\nexport default { setup: () => x };' }, + { dependencies: { "@bob/plugins": "1.0.0" } }, + ); + await writeFiles(join(user, "node_modules", "@bob", "plugins"), { + "package.json": JSON.stringify({ name: "@bob/plugins", chord: { plugins: ["./notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes", dependencies: { "pdfjs-like": "1.0.0" } }), + "notes/contract.ts": 'import pdf from "pdfjs-like";\nexport default pdf;', + "notes/node_modules/pdfjs-like/package.json": JSON.stringify({ + name: "pdfjs-like", + type: "module", + main: "index.mjs", + }), + "notes/node_modules/pdfjs-like/index.mjs": 'export { default } from "./build/pdf.worker.mjs";', + "notes/node_modules/pdfjs-like/build/pdf.worker.mjs": "export default 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/p"); + script(await buildFacet(plugin, "server", { ...NODE, kinds: KINDS, plugins }, context)); + }); +}); + +describe("building a facet: what lies outside the plugin (slice 12 fix round 1: Opus M1, Sol)", () => { + it("refuses a package file whose real path is outside the plugin's node_modules", async () => { + const app = await tempRoot(); + await writeFiles(app, { + "src/core.ts": "export const SECRET = 'base code';", + "src/package.json": '{ "name": "base", "main": "core.ts" }', + }); + const root = join(app, "plugins"); + const a = await writePlugin( + root, + "a", + "@test/a", + { "worker.ts": 'import { SECRET } from "base";\nexport default { setup: () => SECRET };' }, + { dependencies: { base: "file:../../src" } }, + ); + await mkdir(join(a, "node_modules"), { recursive: true }); + // npm's `file:` dependency: a link out of node_modules. + await symlink(join(app, "src"), join(a, "node_modules", "base"), "dir"); + const b = await writePlugin( + root, + "b", + "@test/b", + { "worker.ts": 'import { S } from "dep";\nexport default { setup: () => S };' }, + { dependencies: { dep: "1.0.0" } }, + ); + await writePackage(b, "dep", { "index.js": 'export { SECRET as S } from "../../../../src/core.ts";' }); + const c = await writePlugin( + root, + "c", + "@test/c", + { "worker.ts": 'import { S } from "dep";\nexport default { setup: () => S };' }, + { dependencies: { dep: "1.0.0" } }, + ); + await writePackage(c, "dep", { "package.json": '{ "name": "dep", "main": "index.js" }' }); + // A link inside the package pointing out of it. + await symlink(join(app, "src", "core.ts"), join(c, "node_modules", "dep", "index.js")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, plugins }, context); + const outside = /: resolves outside the plugin's node_modules \(.*src\/core\.ts\)$/; + expect(errors(await build("@test/a"))).toEqual([expect.stringMatching(outside)]); + expect(errors(await build("@test/b"))).toEqual([expect.stringMatching(outside)]); + expect(errors(await build("@test/c"))).toEqual([expect.stringMatching(outside)]); + }); + + it("refuses a facet file linked out of its plugin, or to a file of another kind (Sol)", async () => { + const root = await tempRoot(); + const away = await tempRoot(); + await writeFiles(away, { "facet.ts": "export default { setup: () => {} };" }); + const out = await writePlugin(root, "out", "@test/out", {}); + await symlink(join(away, "facet.ts"), join(out, "worker.ts")); + const kind = await writePlugin(root, "kind", "@test/kind", { "web.tsx": "export default { setup: () => {} };" }); + await symlink(join(kind, "web.tsx"), join(kind, "worker.ts")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, kinds: KINDS, plugins }, context); + expect(errors(await build("@test/out"))).toEqual([ + expect.stringMatching(/^worker\.ts: the facet file links out of its plugin/), + ]); + expect(errors(await build("@test/kind"))).toEqual([ + "worker.ts: links to a web file (web.tsx), not a worker facet", + ]); + }); + + it("refuses a Node builtin in a copied file (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "other", "@test/other", { + "contract.ts": 'import { basename } from "node:path";\nexport const value = basename("x/y");', + }); + await writePlugin(root, "user", "@test/user", { + "worker.ts": 'import { value } from "../other/contract.ts";\nexport default { setup: () => value };', + }); + const { plugins, plugin } = await discovered(root, "@test/user"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, plugins }, context))).toEqual([ + '../other/contract.ts:1:26: "node:path": a file copied from plugin "@test/other" may import only host packages and files of its own plugin', + ]); + }); +}); + +describe("building a facet: packages through their exports (slice 12 §2)", () => { + it("inlines a dependency's shared and per-kind contract files as its exports map them", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "stats", + "@test/stats", + { + "worker.ts": 'import { Notes } from "@bob/notes";\nexport default { setup: () => Notes };', + "web.tsx": 'import { Panel } from "@bob/notes/web";\nexport default { setup: () => Panel };', + }, + { dependencies: { "@bob/notes": "1.0.0" } }, + ); + await writePackage( + dir, + "@bob/notes", + { + "contract.ts": 'export const Notes = "notes v1";', + "contract.web.tsx": 'export const Panel = "panel v1";', + }, + { exports: { ".": "./contract.ts", "./web": "./contract.web.tsx" }, type: "module" }, + ); + const { plugin } = await discovered(root, "@test/stats"); + const worker = script(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS }, context)); + expect(worker.source).toContain("notes v1"); + const web = script(await buildFacet(plugin, "web", { ...BROWSER, kinds: KINDS }, context)); + expect(web.source).toContain("panel v1"); + // Packages are not the watcher's inputs. + expect(worker.inputs.map(({ file }) => file)).toEqual([join(await realpath(dir), "worker.ts")]); + }); +}); + +describe("building a facet: single-file plugins (slice 12 §4)", () => { + it("allows host packages only, refusing other files, packages and builtins with a pointer to a directory", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.server.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});', + "bad.server.ts": [ + 'import { a } from "./hello.server.ts";', + 'import b from "some-package";', + 'import { join } from "node:path";', + "export default { setup: () => [a, b, join] };", + ].join("\n"), + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "server", { ...NODE, plugins }, context); + expect(sourcesOf(script(await build("hello")))).toEqual(["hello.server.ts"]); + const hint = + 'a single-file plugin imports only host packages (the app\'s API); make it a directory "bad/" with a package.json'; + expect(errors(await build("bad"))).toEqual([ + `bad.server.ts:1:19: "./hello.server.ts": ${hint}`, + `bad.server.ts:2:15: "some-package": ${hint}`, + `bad.server.ts:3:22: "node:path": ${hint}`, + ]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/discover.test.ts b/packages/chord/test/next-node/plugins/discover.test.ts new file mode 100644 index 000000000..2fdfd55d3 --- /dev/null +++ b/packages/chord/test/next-node/plugins/discover.test.ts @@ -0,0 +1,272 @@ +// Tests of discovery (slice 7c §1, slice 12 §1, §2, §4): names from package.json, the first root winning a duplicate, +// directories reported (no or an invalid package.json, an invalid name, two files of one kind), contracts-only plugins, +// symlinks, required roots, checkouts of several plugins, packages' chord.plugins manifests, single-file plugins. + +import { mkdir, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +describe("discovery", () => { + it("names a plugin by its package.json, lists its facet files by kind, its dependencies and externals", async () => { + const root = await tempRoot(["contracts", "notes"]); + const dir = await writePlugin( + root, + "deps-dir", + "@test/deps", + { "worker.ts": "", "store.ts": "" }, + { + dependencies: { marked: "1.0.0" }, + chord: { external: ["better-sqlite3"] }, + }, + ); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + const real = await realpath(root); + const of = { root: real, required: false, layout: "directory" } as const; + expect(plugins).toEqual([ + { name: "@test/contracts", dir: join(real, "contracts"), ...of, facets: {}, dependencies: [], external: [] }, + { + name: "@test/deps", + dir: await realpath(dir), + ...of, + facets: { worker: join(real, "deps-dir", "worker.ts") }, + dependencies: ["marked"], + external: ["better-sqlite3"], + }, + { + name: "@test/notes", + dir: join(real, "notes"), + ...of, + facets: { server: join(real, "notes", "server.ts"), worker: join(real, "notes", "worker.ts") }, + dependencies: [], + external: [], + }, + ]); + }); + + it("reports a directory without package.json, with an invalid one, an invalid name or two files of a kind", async () => { + const root = await tempRoot(); + await mkdir(join(root, "bare")); + await writeFiles(join(root, "broken"), { "package.json": "{ not json" }); + await writePlugin(root, "invalid", "Not A Name", {}); + await writePlugin(root, "two", "@test/two", { "worker.ts": "", "worker.js": "" }); + await writePlugin(root, "good", "@test/good", { "web.tsx": "" }); + // Not plugins and not reported: hidden directories and a root's own packages. + await mkdir(join(root, ".git")); + await mkdir(join(root, "node_modules")); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@test/good"]); + const real = await realpath(root); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { path: join(real, "bare"), error: "no package.json" }, + { path: join(real, "broken"), error: expect.stringMatching(/^package\.json: /) }, + { path: join(real, "invalid"), error: 'package.json: "Not A Name" is not a valid plugin name' }, + { path: join(real, "two"), error: "two worker facets: worker.ts, worker.js" }, + ]); + }); + + it("keeps the first root's plugin of a duplicate name and reports the other; a missing root is empty", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(first, "a", "@test/same", { "worker.ts": "" }); + await writePlugin(second, "b", "@test/same", { "server.ts": "" }); + const { plugins, problems } = await discoverPlugins( + [first, join(first, "missing"), second], + { kinds: KINDS }, + context, + ); + expect(plugins.map(({ dir }) => dir)).toEqual([join(await realpath(first), "a")]); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { + path: join(await realpath(second), "b"), + error: `plugin "@test/same" is in ${join(await realpath(first), "a")} already`, + }, + ]); + }); + + it("follows a symlinked plugin directory to its real path", async () => { + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin(elsewhere, "linked", "@test/linked", { "worker.ts": "" }); + await symlink(dir, join(root, "link")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map((each) => [each.name, each.dir])).toEqual([["@test/linked", await realpath(dir)]]); + expect(plugins[0]!.facets).toEqual({ worker: join(await realpath(dir), "worker.ts") }); + }); +}); + +describe("discovery: roots and checkouts (slice 12)", () => { + it("marks the plugins of a required root and records each plugin's root", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + await writePlugin(app, "core", "@test/core", { "server.ts": "" }); + await writePlugin(user, "extra", "@test/extra", { "server.ts": "" }); + const { plugins } = await discoverPlugins([{ path: app, required: true }, user], { kinds: KINDS }, context); + expect(plugins.map(({ name, root, required }) => [name, root, required])).toEqual([ + ["@test/core", await realpath(app), true], + ["@test/extra", await realpath(user), false], + ]); + }); + + it("lists the plugins of a checkout of several (a directory without package.json), within its root", async () => { + const root = await tempRoot(); + await writePlugin(join(root, "repo"), "notes", "@test/notes", { "worker.ts": "" }); + await writePlugin(join(root, "repo"), "search", "@test/search", { "worker.ts": "" }); + await mkdir(join(root, "repo", "docs")); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(problems).toEqual([]); + expect(plugins.map(({ name, dir, root: of }) => [name, dir, of])).toEqual([ + ["@test/notes", join(real, "repo", "notes"), real], + ["@test/search", join(real, "repo", "search"), real], + ]); + }); +}); + +describe("discovery: packages and their chord.plugins manifests (slice 12 §2)", () => { + it("lists the folders chord.plugins names: paths, globs and exclusions", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "bob"), { + "package.json": JSON.stringify({ + name: "@bob/plugins", + chord: { plugins: ["./notes/", "extra/*", "!extra/old", "deep/**/p?"] }, + }), + "server.ts": "", + }); + await writePlugin(join(root, "bob"), "notes", "@bob/notes", { "worker.ts": "" }); + await writePlugin(join(root, "bob"), "unlisted", "@bob/unlisted", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "extra"), "search", "@bob/search", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "extra"), "old", "@bob/old", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "deep", "a", "b"), "p1", "@bob/p1", { "worker.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + const real = await realpath(root); + expect(plugins.map(({ name, dir }) => [name, dir.slice(real.length + 1)])).toEqual([ + ["@bob/notes", "bob/notes"], + ["@bob/p1", "bob/deep/a/b/p1"], + ["@bob/search", "bob/extra/search"], + ]); + }); + + it("lists the package itself as `.`, and reports a listed folder inside another listed one (Opus N5)", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "one"), { + "package.json": JSON.stringify({ name: "@bob/one", chord: { plugins: ["."] } }), + "worker.ts": "", + }); + await writeFiles(join(root, "two"), { + "package.json": JSON.stringify({ name: "@bob/two", chord: { plugins: [".", "./sub"] } }), + "sub/package.json": JSON.stringify({ name: "@bob/sub" }), + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@bob/one"]); + expect(problems.map(({ error }) => error)).toEqual([ + 'package.json: chord.plugins lists "sub" inside the listed "."', + ]); + }); + + it("reports a manifest that is no list, lists nothing, leaves the package or names no directory", async () => { + const root = await tempRoot(); + const manifest = (plugins: unknown) => JSON.stringify({ name: "@bob/x", chord: { plugins } }); + await writeFiles(root, { + "a/package.json": manifest({ "@bob/notes": "github:bob/plugins#v1:notes" }), + "b/package.json": manifest(["nothing/*"]), + "c/package.json": manifest(["../a"]), + "d/package.json": manifest(["./file.txt"]), + "d/file.txt": "", + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(plugins).toEqual([]); + expect(problems.map(({ path, error }) => [path.slice(real.length + 1), error])).toEqual([ + ["a", "package.json: chord.plugins is not a list of plugin folders"], + ["b", "package.json: chord.plugins lists no plugin folder"], + ["c", 'package.json: chord.plugins "../a" leaves the package'], + ["d/file.txt", "listed in chord.plugins, not a directory"], + ]); + }); + + it("by convention: a package with facet files at its top is one plugin, else its subfolders with a package.json", async () => { + const root = await tempRoot(); + await writePlugin(root, "single", "@test/single", { "worker.ts": "", "fixtures/package.json": "{}" }); + await writeFiles(join(root, "repo"), { "package.json": JSON.stringify({ name: "@test/repo" }) }); + await writePlugin(join(root, "repo"), "notes", "@test/notes", { "worker.ts": "" }); + await writePlugin(root, "contracts-only", "@test/contracts", { "contract.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + expect(plugins.map(({ name }) => name)).toEqual(["@test/contracts", "@test/notes", "@test/single"]); + }); + + it("reports a required root that does not exist, and records each problem's root", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "broken"), { "package.json": "{ not json" }); + const missing = join(root, "missing"); + const { problems } = await discoverPlugins( + [ + { path: root, required: true }, + { path: missing, required: true }, + ], + { kinds: KINDS }, + context, + ); + expect(problems).toEqual([ + { + path: join(await realpath(root), "broken"), + error: expect.stringMatching(/^package\.json: /), + root: await realpath(root), + required: true, + }, + { path: missing, error: "the required root does not exist", root: missing, required: true }, + ]); + }); +}); + +describe("discovery: single-file plugins (slice 12 §4)", () => { + it("groups .. files at a root's top by name, one facet per file, without package.json", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.server.ts": "", + "hello.web.tsx": "", + "clock.worker.mjs": "", + "notes.md": "", + "helper.ts": "", + "other.thing.ts": "", + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(problems).toEqual([]); + const of = { dir: real, root: real, required: false, layout: "file", dependencies: [], external: [] }; + expect(plugins).toEqual([ + { name: "clock", ...of, facets: { worker: join(real, "clock.worker.mjs") } }, + { + name: "hello", + ...of, + facets: { server: join(real, "hello.server.ts"), web: join(real, "hello.web.tsx") }, + }, + ]); + }); + + it("reports an invalid name and two files of a kind; a directory plugin of the same name wins", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "Bad.server.ts": "", + "twice.server.ts": "", + "twice.server.js": "", + "notes.server.ts": "", + }); + const dir = await writePlugin(root, "zz-notes", "notes", { "server.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(plugins.map(({ name, layout, dir: at }) => [name, layout, at])).toEqual([ + ["notes", "directory", await realpath(dir)], + ]); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { path: join(real, "Bad.server.ts"), error: '"Bad" is not a valid unscoped plugin name' }, + { path: join(real, "twice.server.js"), error: "two server facets: twice.server.js, twice.server.ts" }, + { path: join(real, "notes.server.ts"), error: `plugin "notes" is in ${await realpath(dir)} already` }, + ]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/evaluate.test.ts b/packages/chord/test/next-node/plugins/evaluate.test.ts new file mode 100644 index 000000000..7186475ff --- /dev/null +++ b/packages/chord/test/next-node/plugins/evaluate.test.ts @@ -0,0 +1,275 @@ +// Tests of evaluation (slice 7c §3): host packages as the host's instances (`instanceof ChordError` across a plugin), +// default and named imports of ESM and CommonJS host packages, chord.external packages and builtins from the plugin, +// `pluginFile` and its `..` refusal (and a single-file plugin's, slice 12 §4), an import-time throw, a non-facet export, +// a refused require, a computed import(), the filename. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + type Plugin, + pluginFile, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected } from "../../next/support.ts"; +import * as esmHost from "../fixtures/esm-host.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +const HOSTS = { ...HOST_PACKAGES, "esm-host": esmHost, "cjs-host": { named: "cjs named" } }; + +/** The fixtures' `Files` contract, as the `contracts` fixture declares it. */ +const Files = local("test.files", { version: 1, params: {}, members: { text: str() } }); + +/** Builds `name`'s `kind` facet under `root` for Node and returns the script and the plugin. */ +async function built(root: string, name: string, kind = "worker"): Promise<{ script: BuiltScript; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins.find((each) => each.name === name)!; + const result = await buildFacet( + plugin, + kind, + { hostPackages: Object.keys(HOSTS), platform: "node", plugins }, + context, + ); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + return { script: result, plugin }; +} + +/** A facet with extra members a test reads. */ +type Probe = { readonly setup: () => void; readonly [name: string]: unknown }; + +describe("evaluating a facet", () => { + it("gives host packages as the host's instances: a ChordError crosses the plugin as one", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import { ChordError, defineFacet } from "@earendil-works/chord/next";', + "const facet = defineFacet(() => {});", + 'export default { ...facet, made: new ChordError({ code: "unavailable", message: "m" }), is: (e: unknown) => e instanceof ChordError };', + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.made).toBeInstanceOf(ChordError); + expect(isChordError(facet.made, "unavailable")).toBe(true); + expect((facet.is as (e: unknown) => boolean)(new ChordError({ code: "unavailable", message: "x" }))).toBe(true); + }); + + it("answers default and named imports of an ESM and a CommonJS host package", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import esm, { named as esmNamed } from "esm-host";', + 'import * as esmAll from "esm-host";', + 'import cjs, { named as cjsNamed } from "cjs-host";', + "export default { setup: () => {}, values: [esm, esmNamed, esmAll.named, cjs, cjsNamed] };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["esm default", "esm named", "esm named", { named: "cjs named" }, "cjs named"]); + }); + + it("loads chord.external packages and Node builtins from the plugin's directory", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "p", + "@test/p", + { + "worker.ts": + 'import ext from "ext-dep";\nimport { basename } from "node:path";\nexport default { setup: () => {}, values: [ext, basename("/a/b")] };', + }, + { chord: { external: ["ext-dep"] } }, + ); + await writePackage(dir, "ext-dep", { "index.js": 'module.exports = "from the plugin";' }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["from the plugin", "b"]); + }); + + it("gives each facet pluginFile for its own plugin's directory and refuses .. out of it", async () => { + const root = await tempRoot(["contracts", "files"]); + const { script, plugin } = await built(root, "@test/files"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/files", facet }], context); + expect((await host.get(Files, {}, context)).text.value).toBe("a file of the plugin"); + await host.close(context); + await writePlugin(root, "escape", "@test/escape", { + "worker.ts": + 'import { pluginFile } from "@earendil-works/chord/next/plugins";\nexport default { setup: () => {}, escape: () => pluginFile("../files/assets/data.txt") };', + }); + const escaping = await built(root, "@test/escape"); + const probe = evaluateFacet(escaping.script, { + hostPackages: HOSTS, + plugin: escaping.plugin, + }) as unknown as Probe; + expect(() => (probe.escape as () => string)()).toThrow( + 'pluginFile("../files/assets/data.txt"): outside the directory of plugin "@test/escape"', + ); + // Outside an evaluated facet it has none. + expect(() => pluginFile("x")).toThrow("only a plugin's facet script"); + }); + + it("refuses pluginFile in a single-file plugin: it has no directory of its own (slice 12 §4)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "lone.worker.ts": + 'import { pluginFile } from "@earendil-works/chord/next/plugins";\nexport default { setup: () => {}, read: () => pluginFile("x.txt") };', + }); + const { script, plugin } = await built(root, "lone"); + const probe = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(() => (probe.read as () => string)()).toThrow( + 'pluginFile("x.txt"): single-file plugin "lone" has no directory of its own', + ); + }); + + it("throws an import-time throw, a non-facet export and a refused require, naming the facet", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'throw new Error("import time");\nexport default { setup: () => {} };', + "server.ts": "export default 42;", + }); + const thrown = await built(root, "@test/p"); + expect(() => evaluateFacet(thrown.script, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow("import time"); + const notFacet = await built(root, "@test/p", "server"); + expect(() => evaluateFacet(notFacet.script, { hostPackages: HOSTS, plugin: notFacet.plugin })).toThrow( + 'plugin "@test/p" (server): its default export is not a facet (defineFacet)', + ); + const handmade: BuiltScript = { + ...thrown.script, + source: '(function (require, module, exports) { require("nope"); })', + }; + expect(() => evaluateFacet(handmade, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow( + 'plugin "@test/p" (worker): "nope" is not a host package, a Node builtin or one of its chord.external packages', + ); + }); + + it("runs a computed import() through the facet's require: an unknown one names the facet (Astra)", async () => { + const root = await tempRoot(); + await writePlugin(root, "dynamic", "@test/dynamic", { + "worker.ts": [ + 'import { defineFacet } from "@earendil-works/chord/next";', + "const later = (specifier: string) => import(specifier);", + "export default { ...defineFacet(() => {}), later };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/dynamic"); + // The fixture attaches its computed import to the facet. + const { later } = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as { + later: (specifier: string) => Promise<{ readonly named?: string }>; + }; + await expect(later("missing-package")).rejects.toThrow( + 'plugin "@test/dynamic" (worker): "missing-package" is not a host package', + ); + expect((await later("cjs-host")).named).toBe("cjs named"); + }); + + it("runs a script under a stable filename per plugin, kind and digest", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'export default { setup: () => {}, fail: () => { throw new Error("x"); } };', + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + let stack = ""; + try { + (facet.fail as () => void)(); + } catch (error) { + stack = (error as Error).stack!; + } + expect(stack).toContain(`/chord-plugins/@test/p/worker.ts?${script.digest}:`); + }); +}); + +describe("a computed require refuses what the build refuses (slice 12 fix round 1, Sol)", () => { + it("refuses another version of the app's API, Chord itself, and builtins in a single-file plugin", async () => { + const root = await tempRoot(); + const api = { name: "@acme/app", versions: ["v3", "v2"] }; + const modules = { "@acme/app/v3": { value: 3 }, "@acme/app/v2": { value: 2 }, "@earendil-works/chord/next": {} }; + const join = (...parts: string[]) => `[${parts.map((part) => JSON.stringify(part)).join(", ")}].join("/")`; + await writePlugin(root, "mixed", "@test/mixed", { + "worker.ts": [ + 'import { value } from "@acme/app/v3";', + `export default { setup: () => {}, other: () => require(${join("@acme/app", "v2")}), chord: () => require(${join("@earendil-works", "chord", "next")}), same: () => require(${join("@acme/app", "v3")}).value + value };`, + ].join("\n"), + }); + await writeFiles(root, { + "lone.worker.ts": 'export default { setup: () => {}, builtin: () => require(["node", "path"].join(":")) };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const options = { hostPackages: Object.keys(modules), platform: "node", api, kinds: KINDS, plugins } as const; + const evaluated = async (name: string) => { + const plugin = plugins.find((each) => each.name === name)!; + const built = await buildFacet(plugin, "worker", options, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return evaluateFacet(built, { hostPackages: modules, plugin }) as unknown as Probe; + }; + const mixed = await evaluated("@test/mixed"); + expect((mixed.same as () => number)()).toBe(6); + expect(() => (mixed.other as () => unknown)()).toThrow( + '"@acme/app/v2": this script uses @acme/app/v3: one facet uses one version of the API', + ); + expect(() => (mixed.chord as () => unknown)()).toThrow("plugins import Chord through the app's API"); + // Without a static import the first version required pins the facet's (Opus N3). + await writePlugin(root, "computed", "@test/computed", { + "worker.ts": `export default { setup: () => {}, both: () => [require(${join("@acme/app", "v3")}).value, require(${join("@acme/app", "v2")}).value] };`, + }); + const fresh = await discoverPlugins([root], { kinds: KINDS }, context); + const computedPlugin = fresh.plugins.find((each) => each.name === "@test/computed")!; + const computedBuilt = await buildFacet(computedPlugin, "worker", { ...options, plugins: fresh.plugins }, context); + if (computedBuilt.kind === "failed") throw new Error(computedBuilt.errors.join("\n")); + const computed = evaluateFacet(computedBuilt, { + hostPackages: modules, + plugin: computedPlugin, + }) as unknown as Probe; + expect(() => (computed.both as () => unknown)()).toThrow( + '"@acme/app/v2": this script uses @acme/app/v3: one facet uses one version of the API', + ); + const lone = await evaluated("lone"); + expect(() => (lone.builtin as () => unknown)()).toThrow( + '"node:path": a single-file plugin requires only host packages', + ); + }); +}); + +describe("ESM host packages' wrappers", () => { + it("wraps a namespace once, and keeps no namespace nobody else holds", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'import * as ns from "fake-esm";\nexport default { setup: () => {}, ns };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const result = await buildFacet(plugin, "worker", { hostPackages: ["fake-esm"], platform: "node" }, context); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + // What `Object.prototype.toString` calls a module namespace. + const evaluate = (namespace: object) => + evaluateFacet(result, { hostPackages: { "fake-esm": namespace }, plugin }) as unknown as Probe; + // Wrapping reads each export once: twice evaluated, read once. + let reads = 0; + const namespace = { + [Symbol.toStringTag]: "Module", + get named() { + reads++; + return 1; + }, + }; + expect([ + (evaluate(namespace).ns as { named: number }).named, + (evaluate(namespace).ns as { named: number }).named, + ]).toEqual([1, 1]); + expect(reads).toBe(1); + const ref = (() => { + const dropped = { [Symbol.toStringTag]: "Module", named: 2 }; + expect((evaluate(dropped).ns as { named: number }).named).toBe(2); + return new WeakRef(dropped); + })(); + await expectCollected(ref); + }); +}); diff --git a/packages/chord/test/next-node/plugins/install.test.ts b/packages/chord/test/next-node/plugins/install.test.ts new file mode 100644 index 000000000..d45849d44 --- /dev/null +++ b/packages/chord/test/next-node/plugins/install.test.ts @@ -0,0 +1,302 @@ +// Tests of installing (slice 7c §5, slice 12 §2, §5): the npm arguments, run in the plugin's real directory, the command +// overridable; a failure and an abort reject; git dependencies with a ref, a plugin inside a repository of several +// reached by path; plugins installed from git and npm (a package's chord.plugins manifest, the refusals, claims and +// cleanup). Git repositories and tarballs are local: no network. + +import { execFileSync } from "node:child_process"; +import { mkdir, readdir, readFile, realpath, symlink, utimes } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { + buildFacet, + discoverPlugins, + installDependencies, + installPlugin, +} from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { gitRepository, KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A fake npm (a Node script) that records its arguments and directory, and exits with `code` after `delayMs`. */ +async function fakeNpm(dir: string, code: number, delayMs = 0): Promise { + const script = join(dir, "fake-npm.mjs"); + await writeFiles(dir, { + "fake-npm.mjs": [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(join(dir, "record.json"))}, JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd() }));`, + `setTimeout(() => { if (${code} !== 0) console.error("npm said no"); process.exit(${code}); }, ${delayMs});`, + ].join("\n"), + }); + return [process.execPath, script]; +} + +describe("installDependencies", () => { + it("runs npm install without dev dependencies, peers, scripts, audit or funding, in the plugin's real directory", async () => { + const tools = await tempRoot(); + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin( + elsewhere, + "deps", + "@test/deps", + { "worker.ts": "" }, + { dependencies: { a: "1.0.0" } }, + ); + await symlink(dir, join(root, "deps")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context, { npmCommand: await fakeNpm(tools, 0) }); + expect(JSON.parse(await readFile(join(tools, "record.json"), "utf8"))).toEqual({ + args: ["install", "--omit=dev", "--legacy-peer-deps", "--ignore-scripts", "--no-audit", "--no-fund"], + cwd: await realpath(dir), + }); + }); + + it("rejects with the command's output when it fails, and when the context aborts", async () => { + const tools = await tempRoot(); + const root = await tempRoot(); + await writePlugin(root, "deps", "@test/deps", {}); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + expect( + await rejection(installDependencies(plugin, context, { npmCommand: await fakeNpm(tools, 1) })), + ).toMatchObject({ + message: expect.stringMatching(/^plugin "@test\/deps": .* install failed \(exit 1\): npm said no$/), + }); + const { context: aborted, cancel } = withCancel(context); + const running = installDependencies(plugin, aborted, { npmCommand: await fakeNpm(tools, 0, 5_000) }); + setTimeout(() => cancel(new Error("stop")), 50); + expect(await rejection(running)).toMatchObject({ name: "AbortError" }); + }); +}); + +/** The worker facet of plugin `name` under `root`, built for Node with Chord as the host package. */ +async function workerOf(root: string, name: string): Promise { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins.find((each) => each.name === name)!; + const options = { hostPackages: ["@earendil-works/chord/next"], platform: "node", plugins } as const; + const built = await buildFacet(plugin, "worker", options, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return built.source; +} + +/** A contracts package `@bob/notes` whose contract says `label`. */ +function notesPackage(label: string): Record { + return { + "package.json": JSON.stringify({ name: "@bob/notes", version: "1.0.0", exports: { ".": "./contract.ts" } }), + "contract.ts": `export const label = ${JSON.stringify(label)};`, + }; +} + +describe("installDependencies: git dependencies (slice 12 §2)", () => { + it("installs a git dependency pinned to a commit with npm; the build inlines that commit's file", async () => { + const first = await gitRepository("notes", notesPackage("notes at v1"), "v1"); + await gitRepository("notes", notesPackage("notes at v2"), "v2", first); + const root = await tempRoot(); + await writePlugin( + root, + "stats", + "@test/stats", + { "worker.ts": 'import { label } from "@bob/notes";\nexport default { setup: () => label };' }, + { dependencies: { "@bob/notes": `git+${first.url}#${first.commit}` } }, + ); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context); + const source = await workerOf(root, "@test/stats"); + expect(source).toContain("notes at v1"); + expect(source).not.toContain("notes at v2"); + }, 60_000); + + it("reaches a plugin inside a multi-plugin repository through a dependency on the whole repository, by path", async () => { + const repository = await gitRepository( + "plugins", + { + "package.json": JSON.stringify({ name: "@bob/plugins", version: "1.0.0", chord: { plugins: ["notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "notes/contract.ts": 'export const label = "notes inside a repository of several";', + }, + "v1", + ); + const root = await tempRoot(); + await writePlugin( + root, + "stats", + "@test/stats", + { + "worker.ts": + 'import { label } from "@bob/plugins/notes/contract.ts";\nexport default { setup: () => label };', + }, + { dependencies: { "@bob/plugins": `git+${repository.url}#${repository.commit}` } }, + ); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context); + expect(await workerOf(root, "@test/stats")).toContain("notes inside a repository of several"); + }, 60_000); +}); + +describe("installPlugin (slice 12 §5)", () => { + it("clones a repository whose top is a plugin into /, at a tag, and installs its dependencies", async () => { + const repository = await gitRepository( + "hello", + { "package.json": JSON.stringify({ name: "@bob/hello" }), "worker.ts": "export default { setup: () => {} };" }, + "v1", + ); + const root = await tempRoot(); + const tools = await tempRoot(); + const npmCommand = await fakeNpm(tools, 0); + expect(await installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand })).toEqual([ + "@bob/hello", + ]); + expect(JSON.parse(await readFile(join(tools, "record.json"), "utf8")).cwd).toBe( + await realpath(join(root, "hello")), + ); + // No updating in place. + expect( + await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand })), + ).toMatchObject({ + message: `installPlugin: ${join(root, "hello")} exists already`, + }); + }); + + it("clones a repository of several plugins (its subdirectories), pinned to a commit", async () => { + const first = await gitRepository( + "bob-plugins", + { + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "search/package.json": JSON.stringify({ name: "@bob/search" }), + }, + "v1", + ); + await gitRepository("bob-plugins", { "later/package.json": JSON.stringify({ name: "@bob/later" }) }, "v2", first); + const root = await tempRoot(); + const tools = await tempRoot(); + const names = await installPlugin({ git: first.url, ref: first.commit }, root, context, { + npmCommand: await fakeNpm(tools, 0), + }); + expect([...names].sort()).toEqual(["@bob/notes", "@bob/search"]); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@bob/notes", "@bob/search"]); + }); + + it("installs an npm package (here a local tarball) into /", async () => { + const source = await tempRoot(); + await writeFiles(source, { + "package.json": JSON.stringify({ name: "@bob/clock", version: "1.2.0" }), + "worker.ts": "export default { setup: () => {} };", + }); + const packed = execFileSync("npm", ["pack", "--silent", "--pack-destination", source], { + cwd: source, + encoding: "utf8", + }).trim(); + const root = await tempRoot(); + const names = await installPlugin({ npm: "@bob/clock", version: `file:${join(source, packed)}` }, root, context); + expect(names).toEqual(["@bob/clock"]); + expect(JSON.parse(await readFile(join(root, "@bob", "clock", "package.json"), "utf8"))).toMatchObject({ + version: "1.2.0", + }); + }, 60_000); + + it("installs exactly the folders a package's chord.plugins lists", async () => { + const repository = await gitRepository( + "bob-plugins", + { + "package.json": JSON.stringify({ + name: "@bob/plugins", + chord: { plugins: ["plugins/*", "!plugins/draft"] }, + }), + "plugins/notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "plugins/draft/package.json": JSON.stringify({ name: "@bob/draft" }), + "tools/package.json": JSON.stringify({ name: "@bob/tools" }), + }, + "v1", + ); + const root = await tempRoot(); + const tools = await tempRoot(); + const names = await installPlugin({ git: repository.url, ref: "v1" }, root, context, { + npmCommand: await fakeNpm(tools, 0), + }); + expect(names).toEqual(["@bob/notes"]); + }); + + it("refuses a ref, a repository and a package name an option could hide in (Sol P1)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + const marker = join(root, "..", "upload-pack-ran"); + for (const source of [ + { git: repository.url, ref: `--upload-pack=touch ${marker}` }, + { git: "--upload-pack=x", ref: "v1" }, + { npm: "--registry=http://evil", version: "1.0.0" }, + { npm: "x", version: "--foreground-scripts" }, + { npm: "../escaped", version: "1.0.0" }, + ]) { + expect(await rejection(installPlugin(source, root, context))).toMatchObject({ + message: expect.stringMatching(/^installPlugin: .* (is refused|is not an npm package name)$/), + }); + } + expect(await readdir(root)).toEqual([]); + expect(await readdir(join(root, "..")).then((names) => names.includes("upload-pack-ran"))).toBe(false); + }); + + it("refuses a plugin name another root has, and leaves nothing behind (Opus M4)", async () => { + const first = await tempRoot(); + const root = await tempRoot(); + await writePlugin(first, "zeta", "@test/same", {}); + const repository = await gitRepository("alpha", { "package.json": JSON.stringify({ name: "@test/same" }) }, "v1"); + const roots = [first, root]; + expect( + await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context, { roots })), + ).toMatchObject({ + message: `installPlugin: plugin "@test/same" is in ${join(await realpath(first), "zeta")} already`, + }); + expect(await readdir(root)).toEqual([]); + }); + + it("claims its directory first: of two concurrent installs one wins, the other removes nothing of it (Sol)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + const tools = await tempRoot(); + const npmCommand = await fakeNpm(tools, 0); + const both = await Promise.allSettled([ + installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand }), + installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand }), + ]); + expect(both.map((each) => each.status).sort()).toEqual(["fulfilled", "rejected"]); + expect(both.find((each) => each.status === "rejected")).toMatchObject({ + reason: { message: `installPlugin: ${join(root, "hello")} exists already` }, + }); + expect(await readdir(join(root, "hello"))).toContain("package.json"); + }); + + it("removes the scope directory it made when a scoped npm install fails (Opus L2)", async () => { + const root = await tempRoot(); + const tools = await tempRoot(); + expect( + await rejection( + installPlugin({ npm: "@bob/notes", version: "1.0.0" }, root, context, { + npmCommand: await fakeNpm(tools, 1), + }), + ), + ).toMatchObject({ message: expect.stringContaining("pack @bob/notes@1.0.0 failed") }); + expect(await readdir(root)).toEqual([]); + }); + + it("removes staging directories an install crashed out of, once an hour old (Opus N6)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + await mkdir(join(root, ".chord-install-old")); + await mkdir(join(root, ".chord-install-recent")); + const hourAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + await utimes(join(root, ".chord-install-old"), hourAgo, hourAgo); + const tools = await tempRoot(); + await installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand: await fakeNpm(tools, 0) }); + expect((await readdir(root)).sort()).toEqual([".chord-install-recent", "hello"]); + }); + + it("removes what arrived when it holds no plugin", async () => { + const repository = await gitRepository("empty", { "readme.md": "" }, "v1"); + const root = await tempRoot(); + expect(await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context))).toMatchObject({ + message: expect.stringMatching(/^installPlugin: .*empty: no package\.json$/), + }); + expect(await readdir(root)).toEqual([]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/plugins.test.ts b/packages/chord/test/next-node/plugins/plugins.test.ts new file mode 100644 index 000000000..2aa3854de --- /dev/null +++ b/packages/chord/test/next-node/plugins/plugins.test.ts @@ -0,0 +1,816 @@ +// Tests of the plugins component (slice 7c §4, §5, §7): per plugin at startup and reload (a failing build, evaluation, +// setup; unchanged plugins untouched; added and removed plugins; the closure rule through builds), `rollout` (order, +// stopping at a rejecting step, marks), installs through the component, the app's API and required roots (slice 12), GC +// over 200 reloads, and end to end a server and a worker loading plugin directories, an edit rolled out, a broken edit +// kept old on both hosts. + +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Host, type ReloadReport } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + createPlugins, + type PluginError, + type Plugins, + type RolloutHost, + type RolloutStep, +} from "../../../src/next-node/plugins/index.ts"; +import { + changesOf, + delivered, + expectCollected, + rejection, + settledHeap, + transported, + valuesOf, +} from "../../next/support.ts"; +import { gitRepository, HOST_PACKAGES, kindsOf, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +/** The fixtures' contracts, as the `contracts` fixture declares them. */ +const Worker = hostKind("test.worker"); +const Notes = remote("test.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); +const Index = local("test.index", { version: 1, params: {}, members: { label: str() } }); +const Item = local("test.item", { version: 1, params: {}, members: { label: str() } }); + +/** A plugin `dir` named `@test/` serving `Item` labelled `label` from its worker facet. */ +async function itemPlugin(root: string, dir: string, label: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The report of an in-process step's one host. */ +function reportOf(hosts: readonly RolloutHost[]): ReloadReport { + const [entry] = hosts; + if (entry === undefined || !("report" in entry)) throw new Error("no report"); + return entry.report; +} + +/** The component over `root`, its reports collected. */ +function component(root: string): { plugins: Plugins; reported: unknown[] } { + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + onError: (e) => reported.push(e), + }, + context, + ); + return { plugins, reported }; +} + +/** The failures of a report as plugin names. */ +const failedNames = (report: { readonly failed: readonly { readonly plugin: string }[] }) => + report.failed.map(({ plugin }) => plugin); + +describe("per plugin at startup", () => { + it("leaves out a plugin whose build fails, whose evaluation throws or whose setup throws; loads the others and core", async () => { + const root = await tempRoot(); + await itemPlugin(root, "good", "good"); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + await writePlugin(root, "throws", "@test/throws", { + "worker.ts": 'throw new Error("at import");\nexport default {};', + }); + await writePlugin(root, "setup", "@test/setup", { + "worker.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => { throw new Error("in setup"); });', + }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + const core = defineFacet((h) => h.serve(Item, () => ({ label: constant("core") }))); + const report = await plugins.load( + host, + { kind: "worker", core: [{ name: "@app/core", facet: core }], modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + [ + "@test/syntax", + expect.stringMatching(/^plugin "@test\/syntax": its worker facet failed to build:\nworker\.ts:1:/), + ], + ["@test/throws", "at import"], + ["@test/setup", 'plugin "@test/setup": in setup'], + ]); + const Good = local("test.good", { version: 1, params: {}, members: { label: str() } }); + expect((await host.get(Good, {}, context)).label.value).toBe("good"); + expect((await host.get(Item, {}, context)).label.value).toBe("core"); + await host.close(context); + plugins.close(); + }); +}); + +describe("per plugin at reload", () => { + /** A host loaded with plugins `a`, `b` and `c` (worker facets), and the component. */ + async function loaded() { + const root = await tempRoot(); + for (const dir of ["a", "b", "c"]) await itemPlugin(root, dir, `${dir}1`); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + return { root, plugins, host, label }; + } + + it("rebuilds one plugin of three: only it changes; the others are passed as the same facets, untouched", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "b", "b2"); + const built = await plugins.build(["@test/b"], context); + expect(built.facets.map(({ plugin, facet }) => [plugin, facet])).toEqual([["@test/b", "worker"]]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: [] }); + expect(report.objects).toEqual([{ object: "test.b", kind: "swapped" }]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b2", "c1"]); + // Built again unchanged: the same digest, nothing reloaded. + const again = await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(again.plugins).toEqual({ added: [], changed: [], removed: [] }); + await host.close(context); + plugins.close(); + }); + + it("keeps the loaded facet of a plugin whose build fails or whose evaluation throws; the others apply", async () => { + const { root, plugins, host, label } = await loaded(); + await writeFiles(join(root, "a"), { "worker.ts": "export default {" }); + await writeFiles(join(root, "b"), { "worker.ts": 'throw new Error("at import");\nexport default {};' }); + await itemPlugin(root, "c", "c2"); + const report = await plugins.reload( + host, + await plugins.build(["@test/a", "@test/b", "@test/c"], context), + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/c"], removed: [] }); + expect(failedNames(report)).toEqual(["@test/a", "@test/b"]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b1", "c2"]); + await host.close(context); + plugins.close(); + }); + + it("adds a plugin discovered since, removes one no longer there; an added plugin that fails is left out", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "d", "d1"); + await writePlugin(root, "e", "@test/e", { "worker.ts": "export default {" }); + await rm(join(root, "c"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build([], context); + expect(built.removed).toEqual(["@test/c"]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: ["@test/d"], changed: [], removed: ["@test/c"] }); + expect(failedNames(report)).toEqual(["@test/e"]); + expect(await label("d")).toBe("d1"); + await host.close(context); + plugins.close(); + }); + + it("keeps a consumer edited together with a provider whose new build fails (the closure rule through builds)", async () => { + const root = await tempRoot(); + const provider = (version: number) => + [ + 'import { constant, defineFacet, local, int } from "@earendil-works/chord/next";', + `export const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + `export default defineFacet((host) => host.serve(P, () => ({ n: constant(${version}) })));`, + ].join("\n"); + const consumer = (version: number) => + [ + 'import { defineFacet, local, int, str } from "@earendil-works/chord/next";', + `const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + 'const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } });', + `export default defineFacet((host) => host.serve(C, (_, self) => ({ label: self.derive([self.get(P, {}).n], ([n]) => "v${version} " + n) })));`, + ].join("\n"); + await writePlugin(root, "provider", "@test/provider", { "worker.ts": provider(1) }); + await writePlugin(root, "consumer", "@test/consumer", { "worker.ts": consumer(1) }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } }); + const shown = await host.get(C, {}, context); + await writeFiles(join(root, "provider"), { "worker.ts": `${provider(2)}\nexport const broken = ;` }); + await writeFiles(join(root, "consumer"), { "worker.ts": consumer(2) }); + const report = await plugins.reload( + host, + await plugins.build(["@test/provider", "@test/consumer"], context), + context, + ); + expect(failedNames(report)).toEqual(["@test/provider", "@test/consumer"]); + expect((report.failed[1]!.error as Error).message).toBe( + 'plugin "@test/consumer": needs test.provider v2, which plugin "@test/provider" serves on its old code', + ); + expect(shown.label.value).toBe("v1 1"); + await host.close(context); + plugins.close(); + }); +}); + +describe("rollout", () => { + /** A step recording its name into `order`, answering `report` for one host (or `hosts`), or rejecting with `error`. */ + const step = ( + name: string, + order: string[], + outcome: { report?: ReloadReport; hosts?: readonly RolloutHost[]; error?: Error }, + kinds: readonly string[] = ["worker"], + ): RolloutStep => ({ + name, + kinds, + reload: async () => { + order.push(name); + if (outcome.error !== undefined) throw outcome.error; + return { hosts: outcome.hosts ?? [{ host: name, report: outcome.report! }] }; + }, + }); + const report = (failed: readonly string[]): ReloadReport => ({ + plugins: { added: [], changed: [], removed: [] }, + failed: failed.map((plugin) => ({ plugin, error: new Error("x") })), + objects: [], + }); + + it("runs the steps in order, clears the marks of plugins applied on every step, keeps a plugin failed on any", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await writeFile(join(root, "b", "worker.ts"), "// b\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a", "@test/b"] }), { + timeout: 5_000, + }); + const built = await plugins.build(["@test/a", "@test/b"], context); + const order: string[] = []; + const result = await plugins.rollout( + [step("server", order, { report: report([]) }), step("worker", order, { report: report(["@test/b"]) })], + built, + context, + ); + expect([order, result.stopped, result.steps.map(({ name }) => name)]).toEqual([ + ["server", "worker"], + undefined, + ["server", "worker"], + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/b"] }); + plugins.close(); + }); + + it("counts a host's error as the step's built plugins failing there; a host stopped cleanly dirties nothing (D12)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const stopped: RolloutHost[] = [{ host: "session|S1", stopped: true }]; + await plugins.rollout( + [step("workers", [], { hosts: stopped })], + await plugins.build(["@test/a"], context), + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + await writeFile(join(root, "a", "worker.ts"), "// a again\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const crashed: RolloutHost[] = [ + { host: "session|S1", report: report([]) }, + { host: "session|S2", error: new Error("crashed") }, + ]; + const result = await plugins.rollout( + [step("workers", [], { hosts: crashed })], + await plugins.build(["@test/a"], context), + context, + ); + expect(result.steps).toEqual([{ name: "workers", hosts: crashed }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + plugins.close(); + }); + + it("keeps dirty a plugin whose built kind no step shipped", async () => { + const root = await tempRoot(); + await writePlugin(root, "a", "@test/a", { "worker.ts": "export default { setup: () => {} };", "server.ts": "" }); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "export default { setup: () => {} }; // edited\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + await plugins.rollout([step("worker", [], { report: report([]) })], built, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await plugins.rollout( + [step("worker", [], { report: report([]) }), step("server", [], { report: report([]) }, ["server"])], + built, + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + plugins.close(); + }); + + it("stops at a step that rejects, clearing no mark; a change during the build keeps its mark", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// 1\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + const order: string[] = []; + const stopped = await plugins.rollout( + [step("server", order, { error: new Error("closed") }), step("worker", order, { report: report([]) })], + built, + context, + ); + expect([order, stopped.stopped?.step, (stopped.stopped?.error as Error).message]).toEqual([ + ["server"], + "server", + "closed", + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + // Saved again after the build: applied, the mark stays (the build did not include the new content). + const later = await plugins.build(["@test/a"], context); + await writeFile(join(root, "a", "worker.ts"), "// 2\n"); + await vi.waitFor(async () => { + await plugins.rollout([step("server", [], { report: report([]) })], later, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + }); + plugins.close(); + }); +}); + +describe("current sets (slice 10 D11)", () => { + it("scripts(kind): built on first use; a build changes no set; a rollout applies it right before a step of its kind", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await writePlugin(root, "b", "@test/b", { "server.ts": "export default { setup: () => {} };" }); + const { plugins } = component(root); + const digests = async () => + (await plugins.scripts("worker", context)).map(({ plugin, built }) => [ + plugin.name, + built.kind === "script" ? built.digest.slice(0, 8) : built.kind, + ]); + const first = await digests(); + expect(first.map(([name]) => name)).toEqual(["@test/a"]); + await itemPlugin(root, "a", "a2"); + const built = await plugins.build(["@test/a"], context); + expect(await digests()).toEqual(first); + const rejecting: RolloutStep = { + name: "server", + kinds: ["server"], + reload: async () => { + throw new Error("closed"); + }, + }; + const shipped: RolloutStep = { name: "workers", kinds: ["worker"], reload: async () => ({ hosts: [] }) }; + await plugins.rollout([rejecting, shipped], built, context); + expect(await digests()).toEqual(first); + await plugins.rollout([shipped], built, context); + expect(await digests()).not.toEqual(first); + plugins.close(); + }); +}); + +describe("installs through the component", () => { + it("clears the plugin's install mark and dirties it", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: (error) => reported.push(error), + }, + context, + ); + await plugins.build([], context); + await writeFile( + join(root, "a", "package.json"), + JSON.stringify({ name: "@test/a", type: "module", dependencies: {} }), + ); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ install: [], dirty: ["@test/a"] }); + expect(reported).toEqual([]); + plugins.close(); + }); +}); + +describe("the app's API, required roots and installs (slice 12 §1, §5)", () => { + it("checks plugins' imports against the app's API, and its required root makes load reject", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + const facet = (version: string) => + `import { defineFacet } from "@acme/app/${version}";\nexport default defineFacet(() => {});`; + await writePlugin(app, "core", "@test/core", { "worker.ts": facet("v3") }); + await writePlugin(app, "old", "@test/old", { "worker.ts": facet("v1") }); + await writePlugin(user, "mine", "@test/mine", { "worker.ts": facet("v1") }); + const modules = { "@acme/app/v3": { defineFacet } }; + const plugins = createPlugins( + { + roots: [{ path: app, required: true }, user], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: kindsOf(modules), + onError: () => {}, + }, + context, + ); + const host = createHost({ onError: () => {} }); + const error = await rejection(plugins.load(host, { kind: "worker", core: [], modules }, context)); + expect(error).toMatchObject({ name: "PluginError" }); + expect((error as PluginError).required.map(({ plugin }) => plugin)).toEqual(["@test/old"]); + expect((error as PluginError).failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/mine", "@test/old"]); + expect(String((error as PluginError).required[0]!.error)).toContain( + '"@acme/app/v1": @acme/app/v1 is not available: this app provides v3', + ); + await host.close(context); + plugins.close(); + }); + + it("rejects load when a required root is missing, holds a broken package or loses a name to an earlier root (Opus M3)", async () => { + const facet = 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});'; + const broken = await tempRoot(); + await writePlugin(broken, "core", "@app/core", { "server.ts": facet }); + await writeFile(join(broken, "core", "package.json"), "{ not json"); + const user = await tempRoot(); + const app = await tempRoot(); + await writePlugin(user, "core", "@app/core", { "server.ts": facet }); + await writePlugin(app, "core", "@app/core", { "server.ts": facet }); + const cases = [ + { roots: [{ path: broken, required: true }], problem: /package\.json: / }, + { roots: [{ path: join(broken, "missing"), required: true }], problem: /the required root does not exist/ }, + { roots: [user, { path: app, required: true }], problem: /plugin "@app\/core" is in .* already/ }, + ]; + for (const { roots, problem } of cases) { + const plugins = createPlugins({ roots, kinds: kindsOf(HOST_PACKAGES), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + const error = await rejection( + plugins.load(host, { kind: "server", core: [], modules: HOST_PACKAGES }, context), + ); + expect(error).toMatchObject({ name: "PluginError" }); + expect((error as PluginError).problems.map(({ error: why }) => why)).toEqual([expect.stringMatching(problem)]); + await host.close(context); + plugins.close(); + } + }); + + it("installs a plugin into one of its roots and marks set; refuses a directory that is no root", async () => { + const root = await tempRoot(); + const repository = await gitRepository( + "hello", + { "package.json": JSON.stringify({ name: "@bob/hello" }), "worker.ts": "export default { setup: () => {} };" }, + "v1", + ); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: () => {}, + }, + context, + ); + await plugins.build([], context); + expect(await plugins.installPlugin({ git: repository.url, ref: "v1" }, root, context)).toEqual(["@bob/hello"]); + expect(plugins.state.value).toMatchObject({ set: true }); + const built = await plugins.build([], context); + expect(Object.keys(built.plugins)).toEqual(["@bob/hello"]); + expect(await rejection(plugins.installPlugin({ git: repository.url, ref: "v1" }, tools, context))).toMatchObject({ + message: `plugins.installPlugin: ${tools} is not one of the roots`, + }); + plugins.close(); + }); +}); + +describe("plugins whose files move, and removed plugins (slice 12 fix round 2, Sol)", () => { + it("rebuilds a single-file plugin that gains a same-named directory (the directory wins) without being named", async () => { + const root = await tempRoot(); + const facet = (label: string) => + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + 'const Item = local("test.hello", { version: 1, params: {}, members: { label: str() } });', + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"); + await writeFiles(root, { "hello.worker.ts": facet("from a file") }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const Hello = local("test.hello", { version: 1, params: {}, members: { label: str() } }); + expect((await host.get(Hello, {}, context)).label.value).toBe("from a file"); + await writePlugin(root, "hello", "hello", { "worker.ts": facet("from a directory") }); + const built = await plugins.build([], context); + expect(Object.keys(built.plugins)).toEqual(["hello"]); + await plugins.rollout([plugins.step("worker", host)], built, context); + expect((await host.get(Hello, {}, context)).label.value).toBe("from a directory"); + await host.close(context); + plugins.close(); + }); + + it("forgets what a removed plugin's builds read: a change there no longer dirties it", async () => { + const root = await tempRoot(); + await writePlugin(root, "contracts", "@test/contracts", { "contract.ts": "export const c = 1;" }); + await writePlugin(root, "user", "@test/user", { + "worker.ts": 'import { c } from "../contracts/contract.ts";\nexport default { setup: () => c };', + }); + const { plugins } = component(root); + await plugins.build(["@test/user"], context); + await rm(join(root, "user"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + // The removal rolled out (no host here): its mark clears. + await plugins.rollout([], await plugins.build([], context), context); + expect(plugins.state.value.dirty).toEqual([]); + await writeFile(join(root, "contracts", "contract.ts"), "export const c = 2;"); + await vi.waitFor(() => expect(plugins.state.value.dirty).toContain("@test/contracts"), { timeout: 5_000 }); + expect(plugins.state.value.dirty).not.toContain("@test/user"); + plugins.close(); + }); +}); + +describe("GC", () => { + it("collects old facets over 200 reloads; the heap stays flat after warm-up", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (closure: object) => void made.push(new WeakRef(closure)) }; + const source = (label: string) => + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + 'const Item = local("test.gc", { version: 1, params: {}, members: { label: str() } });', + // A closure per evaluation, which only this evaluation's code holds. + `const state = { label: ${JSON.stringify(label)}, big: new Array(10_000).fill(${JSON.stringify(label)}) };`, + "made(state);", + "export default defineFacet((host) => host.serve(Item, () => ({ label: constant(state.label) })));", + ].join("\n"); + await writePlugin(root, "gc", "@test/gc", { "worker.ts": source("a") }); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules }, context); + const first = await plugins.build(["@test/gc"], context); + await writeFiles(join(root, "gc"), { "worker.ts": source("b") }); + const second = await plugins.build(["@test/gc"], context); + const reload = async (count: number) => { + for (let i = 0; i < count; i++) await plugins.reload(host, i % 2 === 0 ? second : first, context); + }; + await reload(20); + const warm = await settledHeap(); + await reload(200); + const after = await settledHeap(); + expect(after - warm).toBeLessThan(2 * 1024 * 1024); + await expectCollected(...made.slice(0, made.length - 2)); + await host.close(context); + plugins.close(); + }); +}); + +describe("end to end: a server and a worker", () => { + it("loads plugin directories, rolls an edit out (server, then worker), and keeps a broken edit old on both", async () => { + type P = { readonly kind: "server" } | { readonly kind: "worker" }; + const root = await tempRoot(["contracts", "notes"]); + const { plugins } = component(root); + const [up, down] = memoryLinkPair(); + const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + worker.connect(up, { principal: { kind: "server" } }, context); + const server = createHost

({ principal: { kind: "server" }, onError: () => {} }); + server.route( + Worker, + { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, + context, + ); + await plugins.load(worker, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await plugins.load(server, { kind: "server", core: [], modules: HOST_PACKAGES }, context); + const notes = await server.get(Notes, {}, context); + const index = await server.get(Index, {}, context); + expect([notes.label.value, index.label.value]).toEqual(["v1", "v1"]); + await notes.add(context); + const frames = changesOf(await notes.label.watch(context)); + const steps = [plugins.step("server", server as Host

), plugins.step("worker", worker as Host

)]; + + await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const rolled = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect( + rolled.steps.map(({ name, hosts }) => [name, reportOf(hosts).plugins.changed, reportOf(hosts).objects]), + ).toEqual([ + ["server", ["@test/notes"], [{ object: "test.index", kind: "swapped" }]], + ["worker", ["@test/notes"], [{ object: "test.notes", kind: "swapped" }]], + ]); + await transported(); + await delivered(); + expect(valuesOf(frames)).toEqual(["v2"]); + expect([notes.label.value, notes.count.value, index.label.value]).toEqual(["v2", 1, "v2"]); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + + await writeFile(join(root, "notes", "label.ts"), "export const label = ;\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const broken = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect(broken.steps.map(({ name, hosts }) => [name, failedNames(reportOf(hosts))])).toEqual([ + ["server", ["@test/notes"]], + ["worker", ["@test/notes"]], + ]); + await transported(); + expect([notes.label.value, index.label.value, notes.state.value]).toEqual(["v2", "v2", { kind: "live" }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }); + await server.close(context); + await worker.close(context); + plugins.close(); + }); +}); + +describe("fix round 1", () => { + it("serializes reloads of one host: two started together keep both changes (Astra, Sol, Opus S1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await itemPlugin(root, "a", "a2"); + const builtA = await plugins.build(["@test/a"], context); + await itemPlugin(root, "b", "b2"); + const builtB = await plugins.build(["@test/b"], context); + const reports = await Promise.all([plugins.reload(host, builtA, context), plugins.reload(host, builtB, context)]); + expect(reports.map(({ plugins: { changed } }) => changed)).toEqual([["@test/a"], ["@test/b"]]); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + expect([await label("a"), await label("b")]).toEqual(["a2", "b2"]); + await host.close(context); + plugins.close(); + }); + + it("applies the removals of the build it was given, never a later discovery's (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + await rm(join(root, "b"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const later = await plugins.build([], context); + expect([first.removed, later.removed]).toEqual([[], ["@test/b"]]); + const report = await plugins.reload(host, first, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/a"], removed: [] }); + expect((await plugins.reload(host, later, context)).plugins.removed).toEqual(["@test/b"]); + await host.close(context); + plugins.close(); + }); + + it("adopts what a reload committed when its wait aborts after the commit: a later reload keeps it (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const Item = local("test.a", { version: 1, params: {}, members: { label: str() } }); + const item = await host.get(Item, {}, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + // The abort lands right after the reload's commit (its publication batch), while it waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const controller = new AbortController(); + const aborting = withAbortSignal(controller.signal, context); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) controller.abort(); + }); + expect(await rejection(plugins.reload(host, first, aborting))).toMatchObject({ + code: "aborted", + details: { committed: { added: [], changed: ["@test/a"], removed: [], failed: [] } }, + }); + hook.mockRestore(); + expect(item.label.value).toBe("a2"); + await itemPlugin(root, "b", "b2"); + await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(item.label.value).toBe("a2"); + await host.close(context); + plugins.close(); + }); + + it("leaves a plugin out of a host whose kind its rebuilt facets no longer have (Opus recheck R1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + // The plugin keeps a server facet only. + await rm(join(root, "a", "worker.ts")); + await writeFiles(join(root, "a"), { + "server.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});', + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect(built.removed).toEqual([]); + expect((await plugins.reload(host, built, context)).plugins).toEqual({ + added: [], + changed: [], + removed: ["@test/a"], + }); + await host.close(context); + plugins.close(); + }); + + it("reloads a structured clone of a build as the build itself: it is plain data (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const item = await host.get(local("test.a", { version: 1, params: {}, members: { label: str() } }), {}, context); + await itemPlugin(root, "a", "a2"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect((await plugins.reload(host, structuredClone(built), context)).plugins.changed).toEqual(["@test/a"]); + expect(item.label.value).toBe("a2"); + // The marks clear only through the build object itself (a copy carries no snapshot). + const result = await plugins.rollout([plugins.step("worker", host)], structuredClone(built), context); + expect(result.stopped).toBeUndefined(); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await host.close(context); + plugins.close(); + }); + + it("reads a changed package.json again at the next build: a new dependency is accepted (Astra, Sol)", async () => { + const root = await tempRoot(); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const dir = await writePlugin(root, "a", "@test/a", { + "worker.ts": 'import { label } from "./helper.ts";\nexport default { setup: () => {}, label };', + "helper.ts": 'export const label = "old";', + }); + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: () => {}, + }, + context, + ); + expect((await plugins.build([], context)).facets.map(({ kind }) => kind)).toEqual(["script"]); + await writeFiles(dir, { + "package.json": JSON.stringify({ name: "@test/a", type: "module", dependencies: { "new-dep": "1.0.0" } }), + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await writePackage(dir, "new-dep", { "index.js": 'exports.label = "new";' }); + await writeFiles(dir, { "helper.ts": 'export { label } from "new-dep";' }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ set: false }); + const built = await plugins.build(["@test/a"], context); + expect(built.facets).toEqual([expect.objectContaining({ kind: "script", plugin: "@test/a" })]); + expect(plugins.discovery.plugins[0]!.dependencies).toEqual(["new-dep"]); + plugins.close(); + }); +}); + +describe("the reload queues and garbage collection", () => { + it("lets a rejected new facet go once its report is dropped, the kept old one live (Sol)", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (value: object) => made.push(new WeakRef(value)) }; + const source = (bad: boolean) => + [ + 'import { constant, defineFacet, int, local } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + `const state = { bad: ${bad}, big: new Array(10000).fill("payload") };`, + "made(state);", + 'const Type = local("test.failed-gc", { version: 1, params: {}, members: { n: int() } });', + "export default defineFacet((h) => {", + '\tif (state.bad) throw new Error("bad setup");', + "\th.serve(Type, () => ({ n: constant(state.big.length) }));", + "});", + ].join("\n"); + const dir = await writePlugin(root, "failed", "@test/failed", { "worker.ts": source(false) }); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules }, context); + await writeFiles(dir, { "worker.ts": source(true) }); + const built = await plugins.build(["@test/failed"], context); + // The report (its failure's error holds the new facet's closure) is dropped here; the queues must not keep it. + expect((await plugins.reload(host, built, context)).failed.map(({ plugin }) => plugin)).toEqual(["@test/failed"]); + await expectCollected(made[1]!); + expect(made[0]!.deref()).toBeDefined(); + await host.close(context); + plugins.close(); + }); +}); diff --git a/packages/chord/test/next-node/plugins/runtime.test.ts b/packages/chord/test/next-node/plugins/runtime.test.ts new file mode 100644 index 000000000..d0115f276 --- /dev/null +++ b/packages/chord/test/next-node/plugins/runtime.test.ts @@ -0,0 +1,170 @@ +// Tests of the plugin runtime (slice 10 §3, D11): loading and reloading hosts with complete sets of shipped scripts +// (unchanged digests kept as the same facet, removed = loaded minus shipped, failed builds and evaluations kept old), +// a required root's failed plugin rejecting `load`, and that importing it loads no esbuild. + +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + buildFacet, + createPluginRuntime, + discoverPlugins, + PluginError, + type PluginScript, +} from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { HOST_PACKAGES, tempRoot, writePlugin } from "./support.ts"; + +/** Writes plugin `@test/` serving `test.` labelled `label` from its worker facet (or `source` as it is). */ +async function plugin(root: string, dir: string, label: string, source?: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": + source ?? + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The worker scripts of every plugin under `root`, built now: what the server would ship. */ +async function shipped(root: string): Promise { + const { plugins } = await discoverPlugins([root], { kinds: ["worker"] }, context); + const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; + return Promise.all( + plugins.map(async (each) => ({ plugin: each, built: await buildFacet(each, "worker", options, context) })), + ); +} + +/** The label `test.` shows on `host`. */ +async function label(host: ReturnType, dir: string): Promise { + const Item = local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }); + return (await host.get(Item, {}, context)).label.value; +} + +describe("loading and reloading with complete sets", () => { + it("loads a set and leaves out a plugin whose build or evaluation failed, reported with the host's report", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "broken", "", "export const = ;"); + await plugin(root, "throws", "", 'throw new Error("at import");'); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + const report = await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/broken", "@test/throws"]); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); + + it("reloads a complete set: same digests unchanged, a changed one applied, a loaded one not shipped removed", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "b", "b1"); + await plugin(root, "c", "c1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "b", "b2"); + const next = (await shipped(root)).filter(({ plugin }) => plugin.name !== "@test/c"); + const report = await runtime.reload(host, next, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: ["@test/c"] }); + expect([await label(host, "a"), await label(host, "b")]).toEqual(["a1", "b2"]); + await host.close(context); + }); + + it("keeps the loaded facet of a plugin whose new build or evaluation failed; an added failing one is left out", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "a", "", "export const = ;"); + await plugin(root, "new", "", 'throw new Error("at import");'); + const report = await runtime.reload(host, await shipped(root), context); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/a", "@test/new"]); + expect(report.plugins).toEqual({ added: [], changed: [], removed: [] }); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); +}); + +describe("required roots at load (slice 12 §1)", () => { + it("rejects with a PluginError naming a required plugin that failed, after the host's load settled", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + await plugin(app, "core", "core"); + await plugin(app, "broken", "", "export const = ;"); + await plugin(user, "flaky", "", 'throw new Error("at import");'); + const { plugins } = await discoverPlugins([{ path: app, required: true }, user], { kinds: ["worker"] }, context); + const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; + const scripts = await Promise.all( + plugins.map(async (each) => ({ plugin: each, built: await buildFacet(each, "worker", options, context) })), + ); + const host = createHost({ onError: () => {} }); + const error = await rejection( + createPluginRuntime().load(host, { kind: "worker", core: [], scripts, modules: HOST_PACKAGES }, context), + ); + expect(error).toBeInstanceOf(PluginError); + const failure = error as PluginError; + expect(failure.message).toMatch( + /^required plugin failed at load: "@test\/broken": plugin "@test\/broken": its worker facet failed to build/, + ); + expect(failure.required.map(({ plugin }) => plugin)).toEqual(["@test/broken"]); + expect(failure.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/broken", "@test/flaky"]); + // The load settled: the required root's other plugin is loaded. + expect(await label(host, "core")).toBe("core"); + await host.close(context); + }); + + it("only reports a failed plugin of a root that is not required", async () => { + const user = await tempRoot(); + await plugin(user, "flaky", "", 'throw new Error("at import");'); + const host = createHost({ onError: () => {} }); + const report = await createPluginRuntime().load( + host, + { kind: "worker", core: [], scripts: await shipped(user), modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin }) => plugin)).toEqual(["@test/flaky"]); + await host.close(context); + }); +}); + +describe("the runtime's imports", () => { + it("loads no esbuild: a worker that only evaluates never imports it", () => { + const runtime = fileURLToPath(new URL("../../../src/next-node/plugins/runtime.ts", import.meta.url)); + // A hook refusing esbuild, then the runtime imported. + const script = [ + 'import { registerHooks } from "node:module";', + "registerHooks({ resolve(specifier, context, next) {", + ' if (specifier === "esbuild") throw new Error("esbuild imported");', + " return next(specifier, context);", + "} });", + `const { createPluginRuntime } = await import(${JSON.stringify(runtime)});`, + "createPluginRuntime();", + 'console.log("ok");', + ].join("\n"); + const output = execFileSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", "--input-type=module", "--eval", script], + { encoding: "utf8" }, + ); + expect(output.trim()).toBe("ok"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/stack.test.ts b/packages/chord/test/next-node/plugins/stack.test.ts new file mode 100644 index 000000000..92ec48fc6 --- /dev/null +++ b/packages/chord/test/next-node/plugins/stack.test.ts @@ -0,0 +1,123 @@ +// Tests of stack traces (slice 7c §3, decision Q1 (b)): `mapStack` names plugin files and lines through each +// evaluated script's source map; nothing global; a script's map goes with its facet. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + mapStack, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writePlugin } from "./support.ts"; + +/** A plugin `notes` whose worker facet's `fail` throws from `helper.ts` line 3. */ +async function failing(): Promise<{ script: BuiltScript; plugin: Plugin }> { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { + "worker.ts": 'import { helper } from "./helper.ts";\nexport default { setup: () => {}, fail: () => helper() };', + "helper.ts": + '// A helper.\nexport function helper(): never {\n\tthrow new Error("helper failed at " + Date.now());\n}', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet( + plugin, + "worker", + { hostPackages: Object.keys(HOST_PACKAGES), platform: "node" }, + context, + ); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return { script: built, plugin }; +} + +/** The error `facet`'s `fail` throws. */ +function thrownBy(facet: object): unknown { + try { + (facet as { readonly fail: () => void }).fail(); + } catch (error) { + return error; + } + throw new Error("fail did not throw"); +} + +describe("mapStack", () => { + it("maps a plugin script's positions to the plugin's files and lines; other lines stay; nothing global", async () => { + const { script, plugin } = await failing(); + const prepare = Error.prepareStackTrace; + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const error = thrownBy(facet); + const mapped = mapStack(error); + expect(mapped).toMatch(/at \S+ \(notes\/helper\.ts:3:8\)/); + expect(mapped).toMatch(/notes\/worker\.ts:2:\d+/); + expect(mapped).not.toContain("/chord-plugins/"); + // The raw stack is untouched: no `Error.prepareStackTrace` of the component's. + expect((error as Error).stack).toContain("/chord-plugins/@test/notes/worker.ts?"); + expect(Error.prepareStackTrace).toBe(prepare); + expect(mapStack("not an error")).toBe("not an error"); + }); + + it("lets a script's source map go with its facet", async () => { + const { script, plugin } = await failing(); + const error = await (async () => { + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + return { error: thrownBy(facet), ref: new WeakRef(facet) }; + })(); + expect(mapStack(error.error)).toContain("notes/helper.ts:3:8"); + await expectCollected(error.ref); + // The finalization runs after the collection. + for (let round = 0; round < 10 && mapStack(error.error).includes("notes/helper.ts"); round++) await settledHeap(); + expect(mapStack(error.error)).toContain("/chord-plugins/@test/notes/worker.ts?"); + }); +}); + +describe("mapStack, fix round 1", () => { + it("maps an import-time throw (the map is registered before the script runs) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "evaluation", "@test/evaluation", { + "worker.ts": + '// An import-time throw.\n\nthrow new Error("at import");\nexport default { setup: () => {} };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("evaluation/worker.ts:3:"); + }); + + it("maps a throw while its default export is read (a getter) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "getter", "@test/getter", { + "worker.ts": '// A getter failure.\nexport default {\n\tget setup() { throw new Error("getter boom"); } };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("getter/worker.ts:3:"); + }); + + it("keeps a script's map while any evaluation of it lives: two hosts share it (Sol, Opus N3)", async () => { + const { script, plugin } = await failing(); + const first = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const second = (() => new WeakRef(evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin })))(); + await expectCollected(second); + for (let round = 0; round < 5; round++) await settledHeap(); + expect(mapStack(thrownBy(first))).toContain("notes/helper.ts:3:8"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/support.ts b/packages/chord/test/next-node/plugins/support.ts new file mode 100644 index 000000000..2d4b10d3d --- /dev/null +++ b/packages/chord/test/next-node/plugins/support.ts @@ -0,0 +1,104 @@ +// Test support for the plugins component (slice 7c, slice 12): fixture plugins copied into temporary roots, plugins, npm +// packages and local bare git repositories written there (no network), and the host packages the fixtures import. + +import { execFileSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach } from "vitest"; +import * as chordNext from "../../../src/next/index.ts"; +import type { HostPackages, KindDefinition } from "../../../src/next-node/plugins/index.ts"; + +/** The fixture plugins' directory. */ +export const FIXTURES = fileURLToPath(new URL("../fixtures/plugins/", import.meta.url)); + +/** The host packages the fixtures import: this process's Chord. */ +export const HOST_PACKAGES: HostPackages = { "@earendil-works/chord/next": chordNext }; + +/** The facet kinds of the fixtures' app. */ +export const KINDS = ["server", "worker", "web"] as const; + +/** The fixtures' app's kinds as the component takes them: Node servers and workers, a browser web, `modules`' names. */ +export function kindsOf(modules: HostPackages): Readonly> { + const hostPackages = Object.keys(modules); + return { + server: { platform: "node", hostPackages }, + worker: { platform: "node", hostPackages }, + web: { platform: "browser", hostPackages }, + }; +} + +const made: string[] = []; + +afterEach(async () => { + for (const dir of made.splice(0)) await rm(dir, { recursive: true, force: true }); +}); + +/** A temporary directory removed after the test; `fixtures` (names of fixture plugins) copied into it. */ +export async function tempRoot(fixtures: readonly string[] = []): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-plugins-")); + made.push(root); + for (const name of fixtures) await cp(join(FIXTURES, name), join(root, name), { recursive: true }); + return root; +} + +/** Writes `files` (relative path to text) into `dir`, creating directories. */ +export async function writeFiles(dir: string, files: Readonly>): Promise { + for (const [path, text] of Object.entries(files)) { + await mkdir(dirname(join(dir, path)), { recursive: true }); + await writeFile(join(dir, path), text); + } +} + +/** Writes a plugin named `name` into `root/dir`: its `package.json` (with `manifest`'s fields) and `files`. */ +export async function writePlugin( + root: string, + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const path = join(root, dir); + await writeFiles(path, { "package.json": JSON.stringify({ name, type: "module", ...manifest }), ...files }); + return path; +} + +/** Writes an npm package `name` into `dir`'s `node_modules`: its `package.json` (CommonJS unless `type`) and files. */ +export async function writePackage( + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const main = Object.keys(files)[0]!; + const json = JSON.stringify({ name, version: "1.0.0", main, ...manifest }); + await writeFiles(join(dir, "node_modules", name), { "package.json": json, ...files }); +} + +/** + * A local bare git repository `.git` in a temporary directory with `files` committed (one commit), tagged `tag`: + * its `file://` URL and the commit. A call with `previous` adds another commit on top of that repository. + */ +export async function gitRepository( + name: string, + files: Readonly>, + tag: string, + previous?: { readonly work: string; readonly bare: string }, +): Promise<{ readonly url: string; readonly commit: string; readonly work: string; readonly bare: string }> { + const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.email=test@example.com", "-c", "user.name=test", ...args], { + cwd, + encoding: "utf8", + }).trim(); + const work = previous?.work ?? (await tempRoot()); + if (previous === undefined) git(work, "init", "--quiet"); + await writeFiles(work, files); + git(work, "add", "--all"); + git(work, "commit", "--quiet", "--message", tag); + git(work, "tag", tag); + const bare = previous?.bare ?? join(await tempRoot(), `${name}.git`); + if (previous === undefined) git(work, "clone", "--quiet", "--bare", work, bare); + else git(work, "push", "--quiet", "--tags", bare, "HEAD:refs/heads/master"); + return { url: `file://${bare}`, commit: git(work, "rev-parse", "HEAD"), work, bare }; +} diff --git a/packages/chord/test/next-node/plugins/testing.test.ts b/packages/chord/test/next-node/plugins/testing.test.ts new file mode 100644 index 000000000..60143d531 --- /dev/null +++ b/packages/chord/test/next-node/plugins/testing.test.ts @@ -0,0 +1,219 @@ +// Tests of testing a plugin alone (slice 12 §6): the plugin built as production builds it against the app's API +// modules, loaded on an in-memory host beside a stub serving what it uses; the registries it contributes to that nobody +// serves collected; the contracts it waits for unserved listed; a principal for the host and for a root scope. + +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { defineFacet, int, local, str } from "../../../src/next/index.ts"; +import { testPlugin } from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** The app's API v3 for the tests: Chord's surface re-exported, and a worker module with the app's `Panels`. */ +const Panels = chordNext.defineRegistry<{ readonly title: string }, unknown>("acme.panels", { version: 2 }); +const MODULES = { "@acme/app/v3": { ...chordNext }, "@acme/app/v3/worker": { Panels } }; +const API = { name: "@acme/app", versions: ["v3"] } as const; + +/** The notes contract the stats plugin uses, as the notes plugin declares it. */ +const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** The stats plugin's contract, as a test of it declares it again. */ +const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** A root with the stats plugin: serves `Stats` over `Notes`, contributes a panel, and a tag cloud over `Tags`. */ +async function statsPlugin(): Promise { + const root = await tempRoot(); + return writePlugin(root, "stats", "@acme/stats", { + "worker.ts": [ + 'import { constant, defineFacet, int, local, str } from "@acme/app/v3";', + 'import { Panels } from "@acme/app/v3/worker";', + 'const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { words: int() } });', + 'const Tags = local("acme.tags", { version: 1, params: {}, members: { list: str() } });', + 'const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } });', + 'const Cloud = local("acme.cloud", { version: 1, params: {}, members: { list: str() } });', + "export default defineFacet((host, self) => {", + ' self.contribute(Panels, { title: "Word count" });', + " host.serve(Stats, ({ project }, self) => ({ words: self.get(Notes, { project }).words }));", + " host.serve(Cloud, (_params, self) => ({ list: self.get(Tags, {}).list }));", + "});", + ].join("\n"), + }); +} + +/** A stub of the notes plugin: every project has 42 words. */ +const notesStub = defineFacet((host) => host.serve(Notes, () => ({ words: chordNext.constant(42) }))); + +describe("testPlugin", () => { + it("loads the plugin built against the API beside a stub; collects its contributions and lists what it waits for", async () => { + const dir = await statsPlugin(); + const t = await testPlugin( + dir, + { + kind: "worker", + api: API, + modules: MODULES, + stubs: [{ name: "@acme/notes", facet: notesStub }], + principal: { kind: "user", id: "u1" }, + }, + context, + ); + const stats = await t.host.get(Stats, { project: "p1" }, context); + expect(stats.words.value).toBe(42); + expect(t.contributions(Panels).map(({ plugin, value }) => [plugin, value])).toEqual([ + ["@acme/stats", { title: "Word count" }], + ]); + expect(t.unserved()).toEqual(["acme.tags"]); + expect(await t.root(async (ctx) => ctx.principal, context)).toEqual({ kind: "user", id: "u1" }); + expect(await t.root(async (ctx) => ctx.principal, context, { principal: { kind: "user", id: "u2" } })).toEqual({ + kind: "user", + id: "u2", + }); + expect(() => t.contributions(chordNext.defineRegistry("acme.other", { version: 1 }))).toThrow( + "testPlugin: registry acme.other is not collected", + ); + await t.close(context); + }); + + it("rejects a build production would refuse, and a setup that throws", async () => { + const root = await tempRoot(); + const old = await writePlugin(root, "old", "@acme/old", { + "worker.ts": 'import { defineFacet } from "@acme/app/v1";\nexport default defineFacet(() => {});', + }); + expect(await rejection(testPlugin(old, { kind: "worker", api: API, modules: MODULES }, context))).toMatchObject({ + message: expect.stringContaining('"@acme/app/v1": @acme/app/v1 is not available: this app provides v3'), + }); + const throwing = await writePlugin(root, "throwing", "@acme/throwing", { + "worker.ts": + 'import { defineFacet } from "@acme/app/v3";\nexport default defineFacet(() => { throw new Error("no"); });', + }); + expect( + await rejection(testPlugin(throwing, { kind: "worker", api: API, modules: MODULES }, context)), + ).toMatchObject({ message: expect.stringMatching(/^testPlugin: setup failed: plugin "@acme\/throwing": no$/) }); + }); + + it("builds as production by default: the per-kind check active, api required with versioned modules (Opus M5)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "peek", "@acme/peek", { + "worker.ts": + 'import { defineFacet } from "@acme/app/v3";\nimport { panel } from "./contract.web.ts";\nexport default defineFacet(() => console.log(panel));', + "contract.web.ts": 'export const panel = "panel";', + }); + expect(await rejection(testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context))).toMatchObject({ + message: expect.stringContaining('"./contract.web.ts": a web file, imported into a worker facet'), + }); + expect(await rejection(testPlugin(dir, { kind: "worker", modules: MODULES }, context))).toMatchObject({ + message: 'testPlugin: modules hold the app\'s versioned API ("@acme/app/v3"): pass api, as production does', + }); + }); + + it("tests a single-file plugin and a plugin in a checkout of several (Opus L4)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.worker.ts": [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'const Hello = local("acme.hello", { version: 1, params: {}, members: { text: str() } });', + 'export default defineFacet((host) => host.serve(Hello, () => ({ text: constant("hello") })));', + ].join("\n"), + }); + const single = await testPlugin( + join(root, "hello.worker.ts"), + { kind: "worker", api: API, modules: MODULES }, + context, + ); + const Hello = local("acme.hello", { version: 1, params: {}, members: { text: str() } }); + expect((await single.host.get(Hello, {}, context)).text.value).toBe("hello"); + await single.close(context); + // A checkout of several (its own package.json) beside another plugin of the root it copies from. + await writePlugin(root, "words", "@acme/words", { "contract.ts": 'export const WORDS = "words";' }); + await writeFiles(join(root, "repo"), { "package.json": JSON.stringify({ name: "@bob/repo" }) }); + const inner = await writePlugin(join(root, "repo"), "counter", "@bob/counter", { + "worker.ts": [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'import { WORDS } from "../../words/contract.ts";', + 'const Count = local("bob.count", { version: 1, params: {}, members: { text: str() } });', + "export default defineFacet((host) => host.serve(Count, () => ({ text: constant(WORDS) })));", + ].join("\n"), + }); + const checkout = await testPlugin(inner, { kind: "worker", api: API, modules: MODULES }, context); + const Count = local("bob.count", { version: 1, params: {}, members: { text: str() } }); + expect((await checkout.host.get(Count, {}, context)).text.value).toBe("words"); + await checkout.close(context); + }); + + it("collects until no new registry appears: an object live after the first collection contributes too (Sol)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "chain", "@acme/chain", { + "worker.ts": [ + 'import { constant, defineFacet, defineRegistry, local, str } from "@acme/app/v3";', + 'const A = defineRegistry("acme.a", { version: 1 });', + 'const B = defineRegistry("acme.b", { version: 1 });', + 'const Reader = local("acme.reader", { version: 1, params: {}, members: { s: str() } });', + "export default defineFacet((host, self) => {", + '\tself.contribute(A, "a");', + '\thost.serve(Reader, (_params, self) => { self.registry(A); self.contribute(B, "b"); return { s: constant("x") }; });', + "});", + ].join("\n"), + }); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + const B = chordNext.defineRegistry("acme.b", { version: 1 }); + expect(t.contributions(B).map(({ value }) => value)).toEqual(["b"]); + await t.close(context); + }); + + it("waits for slow activations a collection woke before it looks again (Opus R1)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "slow", "@acme/slow", { + "worker.ts": [ + 'import { constant, defineFacet, defineRegistry, local, str } from "@acme/app/v3";', + 'const A = defineRegistry("acme.a", { version: 1 });', + 'const B = defineRegistry("acme.b", { version: 1 });', + 'const Reader = local("acme.reader", { version: 1, params: {}, members: { s: str() } });', + "export default defineFacet((host, self) => {", + '\tself.contribute(A, "a");', + "\thost.serve(Reader, (_params, self) => {", + "\t\tself.registry(A);", + "\t\tself.open(() => new Promise((resolve) => setTimeout(resolve, 50)), () => {});", + '\t\tself.contribute(B, "b");', + '\t\treturn { s: constant("x") };', + "\t});", + "});", + ].join("\n"), + }); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + const B = chordNext.defineRegistry("acme.b", { version: 1 }); + expect(t.contributions(B).map(({ value }) => value)).toEqual(["b"]); + await t.close(context); + }); + + it("serves a registry contributed at a high version without an adapter per older version (Sol)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "high", "@acme/high", { + "worker.ts": + 'import { defineFacet, defineRegistry } from "@acme/app/v3";\nconst R = defineRegistry("acme.high", { version: 2000000 });\nexport default defineFacet((_host, self) => self.contribute(R, 1));', + }); + const started = performance.now(); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + expect(performance.now() - started).toBeLessThan(1_000); + const R = chordNext.defineRegistry("acme.high", { version: 1 }); + expect(t.contributions(R).map(({ value }) => value)).toEqual([1]); + await t.close(context); + }); + + it("takes no kind from test and declaration files (Opus N6)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.worker.ts": 'import { defineFacet } from "@acme/app/v3";\nexport default defineFacet(() => {});', + "hello.test.ts": "", + "env.d.ts": "", + }); + const t = await testPlugin( + join(root, "hello.worker.ts"), + { kind: "worker", api: API, modules: MODULES }, + context, + ); + expect(Object.keys(t.plugin.facets)).toEqual(["worker"]); + await t.close(context); + }); +}); diff --git a/packages/chord/test/next-node/plugins/watcher.test.ts b/packages/chord/test/next-node/plugins/watcher.test.ts new file mode 100644 index 000000000..f54dcfaef --- /dev/null +++ b/packages/chord/test/next-node/plugins/watcher.test.ts @@ -0,0 +1,225 @@ +// Tests of the watcher (slice 7c §5, slice 12 §4): dirty marks from the files facets used and from plugin directories, +// `install`, `set`, `base`, a change during a build keeping its mark, `node_modules` and `.git` ignored, real file +// events, the marks as a JSON watchable, and single-file plugins' files. + +import { readdirSync, readFileSync } from "node:fs"; +import { realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { type BuiltFacet, discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { PluginWatcher } from "../../../src/next-node/plugins/watcher.ts"; +import { changesOf, delivered } from "../../next/support.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A root with plugins `a` (worker), `b` (server) and the contracts fixture, a base directory, and a watcher of both. */ +async function watched() { + const root = await realpath(await tempRoot(["contracts"])); + const base = await realpath(await tempRoot()); + await writePlugin(root, "a", "@test/a", { "worker.ts": "", "own.ts": "" }); + await writePlugin(root, "b", "@test/b", { "server.ts": "" }); + const failures: unknown[] = []; + const watcher = new PluginWatcher( + { roots: [root], base: [base], kinds: KINDS, failed: (error) => failures.push(error) }, + new Batch(() => {}, undefined), + ); + const discovery = await discoverPlugins([root], { kinds: KINDS }, context); + watcher.know(discovery, watcher.discoverySnapshot()); + const contract = join(root, "contracts", "contract.ts"); + const used = (plugin: string, facet: string, files: readonly string[]): BuiltFacet => ({ + kind: "script", + plugin, + facet, + source: "", + digest: "", + inputs: files.map((file) => ({ file, hash: "" })), + copied: [contract], + assets: [], + }); + watcher.used([ + used("@test/a", "worker", [join(root, "a", "worker.ts"), contract]), + used("@test/b", "server", [join(root, "b", "server.ts"), contract]), + ]); + return { root, base, watcher, failures }; +} + +describe("the watcher's marks", () => { + it("dirties every plugin whose facets used a changed file, and the plugin whose directory contains it", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "contracts", "contract.ts")); + expect(watcher.value).toEqual({ + dirty: ["@test/a", "@test/b", "@test/contracts"], + install: [], + set: false, + base: false, + }); + watcher.close(); + }); + + it("dirties the plugin of a file no facet read yet; asks for an install for a package.json", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "new.ts")); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toEqual({ dirty: ["@test/a"], install: ["@test/b"], set: false, base: false }); + // Installed: the mark clears, and the plugin is dirty (its packages changed). + watcher.installed("@test/b"); + expect(watcher.value).toEqual({ dirty: ["@test/a", "@test/b"], install: [], set: false, base: false }); + watcher.close(); + }); + + it("asks for discovery for a directory under a root and a facet file at a plugin's top; base for base code", async () => { + const { root, base, watcher } = await watched(); + watcher.mark(join(root, "c")); + expect(watcher.value).toMatchObject({ set: true, dirty: [] }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: false }); + watcher.mark(join(root, "b", "web.tsx")); + expect(watcher.value).toMatchObject({ set: true, dirty: ["@test/b"] }); + watcher.mark(join(base, "core.ts")); + expect(watcher.value).toMatchObject({ base: true }); + watcher.close(); + }); + + it("ignores node_modules and .git", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "node_modules", "x", "index.js")); + watcher.mark(join(root, "a", ".git", "HEAD")); + expect(watcher.value).toEqual({ dirty: [], install: [], set: false, base: false }); + watcher.close(); + }); + + it("keeps the mark of a change during a build: only an unchanged mark clears when applied", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "own.ts")); + const snapshot = watcher.snapshot(["@test/a"]); + // Saved again while the build runs. + watcher.mark(join(root, "a", "own.ts")); + watcher.applied(["@test/a"], snapshot); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + expect(watcher.value).toMatchObject({ dirty: [] }); + // A discovery that started before a change of the set keeps `set`. + const seen = watcher.discoverySnapshot(); + watcher.mark(join(root, "d")); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: true }); + watcher.close(); + }); + + it("marks from real file events, publishing the marks to a watch", async () => { + const { root, watcher } = await watched(); + const changes = changesOf(await watcher.watch(context)); + await writeFile(join(root, "a", "node_modules-free.ts"), "export {};"); + await writeFiles(join(root, "b", "node_modules", "pkg"), { "index.js": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + await delivered(); + expect(changes.at(-1)).toMatchObject({ kind: "value", value: { dirty: ["@test/a"] } }); + watcher.close(); + }); +}); + +describe("the watcher and single-file plugins (slice 12 §4)", () => { + it("dirties a single-file plugin for a change of its file; a file appearing or disappearing at a root's top sets set", async () => { + const { root, watcher } = await watched(); + await writeFiles(root, { "hello.server.ts": "" }); + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: true, dirty: [] }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: false, dirty: [] }); + // Its own file: only it is dirty (not every plugin of the root its directory is). + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: false, dirty: ["hello"] }); + await writeFiles(root, { "hello.web.tsx": "" }); + watcher.mark(join(root, "hello.web.tsx")); + expect(watcher.value).toMatchObject({ set: true }); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), watcher.discoverySnapshot()); + await rm(join(root, "hello.server.ts")); + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: true, dirty: ["hello"] }); + watcher.close(); + }); +}); + +describe("the watcher, fix round 1", () => { + it("never lets an older build's applied snapshot clear a newer edit's mark (Astra)", async () => { + const { root, watcher } = await watched(); + const file = join(root, "a", "own.ts"); + watcher.mark(file); + const first = watcher.snapshot(["@test/a"]); + watcher.mark(file); + const second = watcher.snapshot(["@test/a"]); + watcher.applied(["@test/a"], second); + expect(watcher.value).toMatchObject({ dirty: [] }); + watcher.mark(file); + watcher.applied(["@test/a"], first); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.close(); + }); + + it("drops the install mark of a plugin the next discovery no longer has (example 38)", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toMatchObject({ install: ["@test/b"] }); + await rm(join(root, "b"), { recursive: true }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ install: [] }); + watcher.close(); + }); + + it.skipIf(process.platform !== "linux")( + "puts no watch on node_modules or .git (Opus N2): 200 packages add no inotify watch", + async () => { + const root = await realpath(await tempRoot()); + const dir = await writePlugin(root, "a", "@test/a", { "worker.ts": "" }); + for (let i = 0; i < 200; i++) + await writeFiles(join(dir, "node_modules", `pkg${i}`, "lib"), { "index.js": "" }); + await writeFiles(join(dir, ".git", "objects"), { x: "" }); + const before = inotifyWatches(); + const watcher = new PluginWatcher( + { roots: [root], base: [], kinds: KINDS, failed: () => {} }, + new Batch(() => {}, undefined), + ); + // The root and the plugin's directory. + expect(inotifyWatches() - before).toBe(2); + watcher.close(); + }, + ); + + it("watches a directory that appears later, and marks the files in it", async () => { + const { root, watcher } = await watched(); + await writeFiles(join(root, "a", "lib"), { "first.ts": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + await writeFile(join(root, "a", "lib", "second.ts"), "export {};"); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +describe("the watcher and links", () => { + it("survives a link loop created under a root (its stat never escapes the callback) and still marks it (Sol)", async () => { + const { root, watcher } = await watched(); + await symlink("loop", join(root, "a", "loop")); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +/** The inotify watches this process holds (Linux): one line per watch in its inotify descriptors' fdinfo. */ +function inotifyWatches(): number { + let count = 0; + for (const fd of readdirSync("/proc/self/fdinfo")) { + try { + count += readFileSync(`/proc/self/fdinfo/${fd}`, "utf8") + .split("\n") + .filter((line) => line.startsWith("inotify wd:")).length; + } catch { + // Closed meanwhile. + } + } + return count; +} diff --git a/packages/chord/test/next-node/workers/control.test.ts b/packages/chord/test/next-node/workers/control.test.ts new file mode 100644 index 000000000..a7054269a --- /dev/null +++ b/packages/chord/test/next-node/workers/control.test.ts @@ -0,0 +1,55 @@ +// Tests of the control channel's crossings (slice 10 §3): errors cross as `{ message, code?, reason? }` and back, a +// reload report with its errors and parked causes. + +import { describe, expect, it } from "vitest"; +import { ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { + controlOf, + crossed, + crossedReport, + envelope, + uncrossed, + uncrossedReport, +} from "../../../src/next-node/workers/control.ts"; + +describe("errors", () => { + it("cross as message, code and reason, never a stack or cause; come back as ChordErrors when they had a code", () => { + const chord = new ChordError({ code: "unavailable", message: "gone" }, { cause: new Error("inner") }); + expect(crossed(chord)).toEqual({ message: "gone", code: "unavailable" }); + expect(crossed(new Error("plain"))).toEqual({ message: "plain" }); + expect(crossed("text")).toEqual({ message: "text" }); + const back = uncrossed({ message: "gone", code: "released", reason: "closed" }); + expect(isChordError(back, "released") && back.reason).toBe("closed"); + expect(uncrossed({ message: "odd", code: "no-such-code" })).toMatchObject({ message: "odd" }); + }); + + it("crosses a report's failures and parked causes and back", () => { + const report = crossedReport({ + plugins: { added: ["@a/x"], changed: [], removed: [] }, + failed: [{ plugin: "@a/y", error: new Error("broken") }], + objects: [ + { object: "a.one", kind: "swapped" }, + { + object: "a.two", + kind: "parked", + cause: { kind: "waiting", error: new ChordError({ code: "unavailable", message: "w" }) }, + }, + ], + }); + expect(JSON.parse(JSON.stringify(report))).toEqual(report); + const back = uncrossedReport(report); + expect(back.failed[0]!.error).toBeInstanceOf(Error); + expect(back.objects[1]).toMatchObject({ + kind: "parked", + cause: { kind: "waiting", error: { code: "unavailable" } }, + }); + }); +}); + +describe("envelopes", () => { + it("tells control messages from anything else on the channel", () => { + expect(controlOf(envelope({ kind: "started" }))).toEqual({ kind: "started" }); + expect(controlOf(new Uint8Array(1))).toBeUndefined(); + expect(controlOf({ kind: "started" })).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next-node/workers/lock.test.ts b/packages/chord/test/next-node/workers/lock.test.ts new file mode 100644 index 000000000..de1564720 --- /dev/null +++ b/packages/chord/test/next-node/workers/lock.test.ts @@ -0,0 +1,62 @@ +// Tests of process locks (slice 10 §3): one holder at a time, a waiter taking the lock when the holder releases it +// (told by its connection's close, no polling), a waiter's abort. + +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { processLock } from "../../../src/next-node/workers/index.ts"; +import { lockAt } from "../../../src/next-node/workers/lock.ts"; + +describe("processLock", () => { + it("makes a second taker wait until the holder releases, then takes it", async () => { + const name = `chord-test-lock-${process.pid}-${Date.now()}`; + const first = await processLock(name, context); + let taken = false; + const second = processLock(name, context).then((lock) => { + taken = true; + return lock; + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(taken).toBe(false); + const released = Date.now(); + await first.release(); + await (await second).release(); + expect(taken).toBe(true); + expect(Date.now() - released).toBeLessThan(100); + }); + + it("stops waiting when the waiter's context aborts", async () => { + const name = `chord-test-lock-abort-${process.pid}-${Date.now()}`; + const held = await processLock(name, context); + const abort = new AbortController(); + const waiting = processLock(name, withAbortSignal(abort.signal, context)); + abort.abort(new Error("gave up")); + await expect(waiting).rejects.toThrow("gave up"); + await held.release(); + await (await processLock(name, context)).release(); + }); +}); + +describe("socket file locks (macOS, the BSDs)", () => { + it("fail closed on a stale lock file, naming it: two takers never both take it over (Sol)", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-lock-file-")); + const address = join(dir, "lock.sock"); + // What a crashed holder leaves: a path nobody accepts on. + await writeFile(address, ""); + const takers = [lockAt({ kind: "file", address }, context), lockAt({ kind: "file", address }, context)]; + for (const taker of takers) await expect(taker).rejects.toThrow(`lock: ${address} is a stale lock file`); + await rm(dir, { recursive: true, force: true }); + }); + + it("are taken, waited for and released like the others", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-lock-file-")); + const address = join(dir, "lock.sock"); + const first = await lockAt({ kind: "file", address }, context); + const second = lockAt({ kind: "file", address }, context); + await first.release(); + await (await second).release(); + await rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/packages/chord/test/next-node/workers/main.test.ts b/packages/chord/test/next-node/workers/main.test.ts new file mode 100644 index 000000000..017c9df04 --- /dev/null +++ b/packages/chord/test/next-node/workers/main.test.ts @@ -0,0 +1,87 @@ +// Tests of the worker's side (slice 10 §3): the start message, the lock, `started` on `connect`, `ready`; shutdown +// during startup, the parent's disconnect and SIGTERM each run the closers and exit; `fatal` redacted. + +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { hostKind, str } from "../../../src/next/contracts/index.ts"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { type Exit, WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import type { WorkerBehaviour } from "../fixtures/workers/control.ts"; +import { until } from "./support.ts"; + +const ENTRY = fileURLToPath(new URL("../fixtures/workers/worker.ts", import.meta.url)); + +const Session = hostKind("session", { id: str() }); + +/** The fixture worker forked and started with `behaviour`; its control messages and exit collected. */ +function started(behaviour: WorkerBehaviour) { + const control: FromWorker[] = []; + let exit: Exit | undefined; + const child = new WorkerProcess( + { + entry: ENTRY, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + onOutput: () => {}, + }, + { + control: (message) => control.push(message), + linkLost: () => {}, + linkClosed: () => {}, + exited: (value) => { + exit = value; + }, + }, + ); + child.send({ + kind: "start", + address: Session({ id: "S1" }), + start: behaviour, + scripts: [], + lock: `chord-test-main:${process.pid}:${Date.now()}:${Math.random()}`, + }); + return { child, control, exited: () => exit }; +} + +describe("the worker's side", () => { + it("sends started when the app connects, then ready with its load report and idle state", async () => { + const { child, control } = started({}); + await until(() => control.some(({ kind }) => kind === "ready")); + expect(control.map(({ kind }) => kind)).toEqual(["started", "ready"]); + expect(control[1]).toMatchObject({ idle: true, report: { plugins: { added: [] }, failed: [] } }); + child.kill(); + }); + + it("shuts down during its startup: aborts it, runs the closers registered so far in reverse order, exits 0", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-main-")); + const { child, control, exited } = started({ hang: true, log: join(dir, "log") }); + await until(() => control.some(({ kind }) => kind === "started")); + child.send({ kind: "shutdown" }); + await until(() => exited() !== undefined); + expect(exited()).toEqual({ code: 0, signal: null }); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + await rm(dir, { recursive: true, force: true }); + }); + + it("shuts down on SIGTERM and on the parent's disconnect", async () => { + const term = started({}); + await until(() => term.control.some(({ kind }) => kind === "ready")); + process.kill(term.child.pid, "SIGTERM"); + await until(() => term.exited() !== undefined); + expect(term.exited()).toEqual({ code: 0, signal: null }); + const gone = started({}); + await until(() => gone.control.some(({ kind }) => kind === "ready")); + gone.child.link.close(); + await until(() => gone.exited() !== undefined); + expect(gone.exited()).toEqual({ code: 0, signal: null }); + }); + + it("sends fatal with the redacted error and exits 1", async () => { + const { control, exited } = started({ fatal: "no secret-key-42 here" }); + await until(() => exited() !== undefined); + expect(control).toEqual([{ kind: "fatal", error: { message: "no [redacted] here" } }]); + expect(exited()?.code).toBe(1); + }); +}); diff --git a/packages/chord/test/next-node/workers/process.test.ts b/packages/chord/test/next-node/workers/process.test.ts new file mode 100644 index 000000000..b84589f6e --- /dev/null +++ b/packages/chord/test/next-node/workers/process.test.ts @@ -0,0 +1,46 @@ +// Tests of worker processes (slice 10 §3): forked with their output piped (the stderr tail kept), the link made at the +// fork (an early frame queued), control messages told, and the whole tree killed. + +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import { until } from "./support.ts"; + +const CHILD = fileURLToPath(new URL("../fixtures/workers/child.ts", import.meta.url)); + +/** Whether process `pid` lives. */ +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +describe("a worker process", () => { + it("pipes its output, keeps its stderr tail, queues an early frame on its link, tells control messages", async () => { + const output: string[] = []; + const control: FromWorker[] = []; + const child = new WorkerProcess( + { + entry: CHILD, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + onOutput: (stream, chunk) => output.push(`${stream}: ${Buffer.from(chunk).toString().trim()}`), + }, + { control: (message) => control.push(message), linkLost: () => {}, linkClosed: () => {}, exited: () => {} }, + ); + await until(() => control.length > 0 && output.length >= 2); + const frames: number[][] = []; + child.link.onMessage((message) => frames.push([...message])); + expect(frames).toEqual([[1, 2, 3]]); + expect(control).toEqual([{ kind: "started" }]); + expect(child.stderrTail).toBe("an error line\n"); + const grandchild = Number(/grandchild (\d+)/.exec(output.join("\n"))![1]); + expect(alive(grandchild)).toBe(true); + child.kill(); + expect(await child.exited).toEqual({ code: null, signal: "SIGKILL" }); + await until(() => !alive(grandchild), 2_000); + }); +}); diff --git a/packages/chord/test/next-node/workers/support.ts b/packages/chord/test/next-node/workers/support.ts new file mode 100644 index 000000000..4931905d5 --- /dev/null +++ b/packages/chord/test/next-node/workers/support.ts @@ -0,0 +1,151 @@ +// Test support for the workers component (slice 10 §3, §4): a server routing `Session` to forked fixture workers, the +// fixture plugins copied into a temporary root, each address's start data (its behaviour) chosen by the test. + +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { hostKind, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, type Host, type HostError } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { createPlugins, type Plugins } from "../../../src/next-node/plugins/index.ts"; +import { createWorkers, type Workers, type WorkersError } from "../../../src/next-node/workers/index.ts"; +import { Session, type WorkerBehaviour } from "../fixtures/workers/control.ts"; + +/** The server's own kind. */ +export const Server = hostKind("server"); + +/** The fixture plugin's notes at a worker. */ +export const Notes = remote("test.notes", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { label: str() } } }, +}); + +/** The principals of these tests. */ +export type TestPrincipal = + | { readonly kind: "server" } + | { readonly kind: "worker"; readonly id: string } + | { readonly kind: "guest"; readonly id: string }; + +const ENTRY = fileURLToPath(new URL("../fixtures/workers/worker.ts", import.meta.url)); + +const PLUGINS = fileURLToPath(new URL("../fixtures/workers/plugins/", import.meta.url)); + +const made: { readonly root: string; readonly close: () => Promise }[] = []; + +afterEach(async () => { + for (const { root, close } of made.splice(0)) { + await close(); + await rm(root, { recursive: true, force: true }); + } +}); + +/** A server with workers; closed after the test. */ +export type Fleet = { + readonly server: Host; + readonly workers: Workers; + readonly plugins: Plugins; + /** The copy of the fixture plugins. */ + readonly root: string; + readonly errors: WorkersError[]; + readonly hostErrors: HostError[]; + /** The addresses `start` ran for, in order. */ + readonly starts: string[]; + /** The workers' stdout lines, each with its address key. */ + readonly output: string[]; + /** Each address's behaviour (its start data); none: a plain worker. */ + readonly behaviours: Map; + /** Writes `label` into the copy's notes plugin. */ + label(label: string): Promise; +}; + +/** Starts a server routing `Session` to fixture workers; `options` override the workers' defaults. */ +export async function fleet( + options: { + readonly idleMs?: number; + readonly heartbeat?: { readonly ping?: number; readonly close?: number }; + readonly start?: (address: { readonly key: string }, context: Context) => Promise; + readonly principal?: () => TestPrincipal; + readonly authorize?: () => boolean; + } = {}, +): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-workers-")); + await cp(PLUGINS, root, { recursive: true }); + const errors: WorkersError[] = []; + const hostErrors: HostError[] = []; + const starts: string[] = []; + const behaviours = new Map(); + const output: string[] = []; + const modules = { "@earendil-works/chord/next": chordNext }; + const plugins = createPlugins( + { roots: [root], kinds: { worker: { platform: "node", hostPackages: Object.keys(modules) } }, onError: () => {} }, + context, + ); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => hostErrors.push(error), + ...(options.heartbeat === undefined ? {} : { heartbeat: options.heartbeat }), + }); + const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: ENTRY, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: options.authorize ?? (() => true), + principal: options.principal ?? ((address) => ({ kind: "worker", id: address.params.id }) as const), + start: async (address, startContext) => { + starts.push(address.key); + if (options.start !== undefined) return options.start(address, startContext); + return (behaviours.get(address.params.id) ?? {}) as JsonValue; + }, + lock: (address) => `chord-test:${root}:${address.key}`, + idleMs: () => options.idleMs ?? 60_000, + plugins, + onOutput: (address, stream, chunk) => { + if (stream === "stdout") output.push(`${address.key} ${Buffer.from(chunk).toString().trim()}`); + }, + onError: (error) => errors.push(error), + }, + context, + ); + await server.load([], context); + // The worker kind's set made now (discovery, builds): the watcher marks edits from then on. + await plugins.scripts("worker", context); + const close = async () => { + await workers.close(context); + await server.close(context); + plugins.close(); + }; + made.push({ root, close }); + return { + server, + workers, + plugins, + root, + errors, + hostErrors, + starts, + output, + behaviours, + label: (label) => writeFile(join(root, "notes", "label.ts"), `export const label = ${JSON.stringify(label)};\n`), + }; +} + +/** Waits until `ready()` holds, polling every 10 ms, for at most `ms`. */ +export async function until(ready: () => boolean, ms = 10_000): Promise { + const deadline = Date.now() + ms; + while (!ready()) { + if (Date.now() > deadline) throw new Error("until: timed out"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export { context, Session }; diff --git a/packages/chord/test/next-node/workers/workers.test.ts b/packages/chord/test/next-node/workers/workers.test.ts new file mode 100644 index 000000000..d7c980784 --- /dev/null +++ b/packages/chord/test/next-node/workers/workers.test.ts @@ -0,0 +1,605 @@ +// Tests of the workers component (slice 10 §3, §4): starts on demand (the link made at the fork, `started` on +// `connect`, the start timer covering only the start), failures that stay (a crash, a `fatal`, a lost link, a failed +// start: no fork for a new hold, holders `failed { waiting }`) until `retry` or a rollout, idle stops and `busy`, `stop` +// and its suspension, the per-address queue, `close`, the rollout step, and nothing kept of 1,000 addresses. + +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { withAbortSignal } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import type { RolloutHost } from "../../../src/next-node/plugins/index.ts"; +import { expectCollected } from "../../next/support.ts"; +import { Control } from "../fixtures/workers/control.ts"; +import { context, fleet, Notes, Session, type TestPrincipal, until } from "./support.ts"; + +/** A step's hosts as `[host, what]`. */ +function listed(hosts: readonly RolloutHost[]) { + return hosts.map((entry) => [ + entry.host, + "report" in entry ? entry.report.plugins : "stopped" in entry ? "stopped" : (entry.error as Error).message, + ]); +} + +/** The state entry of `id`. */ +const stateOf = (state: unknown, id: string) => + (state as Record)[`session|${id}`]; + +describe("starts on demand", () => { + it("starts a worker for the first hold: starting, then running; its link binds the holder", async () => { + const { server, workers, starts } = await fleet(); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"); + await until(() => notes.state.value.kind === "live"); + expect(notes.label.value).toBe("S1 v1"); + await until(() => stateOf(workers.state.value, "S1")?.status === "running"); + expect(starts).toEqual(["session|S1"]); + notes.release(); + }); + + it("binds through a load longer than the heartbeat: the link is made at the fork, the worker connects first", async () => { + const { server, behaviours } = await fleet({ heartbeat: { ping: 50, close: 200 } }); + behaviours.set("S1", { loadMs: 600 }); + const notes = await server.get(Notes, { id: "S1" }, context); + expect(notes.label.value).toBe("S1 v1"); + notes.release(); + }); +}); + +describe("failures stay until retry or a rollout (R2)", () => { + it("a start that throws fails the address: holders fail waiting with it, a new hold forks nothing; retry starts it", async () => { + let fail = true; + const { server, workers, errors, starts } = await fleet({ + start: async () => { + if (fail) throw new Error("no data dir"); + return {}; + }, + }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind === "failed"); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "worker failed to start" } }, + }); + expect(stateOf(workers.state.value, "S1")).toMatchObject({ + status: "failed", + error: { message: "worker failed to start" }, + }); + const again = server.hold(Notes, { id: "S1" }, context); + await until(() => again.state.value.kind === "failed"); + expect(starts).toEqual(["session|S1"]); + // The full error only for the app's logs (decision "Slice 10 fix round, worker errors in public"). + expect(errors).toMatchObject([ + { kind: "worker_failed", address: "session|S1", error: { message: "no data dir" } }, + ]); + fail = false; + await workers.retry(Session({ id: "S1" }), context); + await until(() => notes.state.value.kind === "live" && again.state.value.kind === "live"); + expect(stateOf(workers.state.value, "S1")?.status).not.toBe("failed"); + notes.release(); + again.release(); + }); + + it("a crash after ready stays failed (worker_exited, worker_failed), nothing restarts", async () => { + const { server, workers, errors, starts } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S1" }, context); + await control.crash(7, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (code 7)"); + expect(errors.map(({ kind }) => kind)).toEqual(["worker_exited", "worker_failed"]); + await until(() => notes.state.value.kind === "failed"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(starts).toEqual(["session|S1"]); + notes.release(); + control.release(); + }); + + it("a fatal crosses redacted: the secret never reaches the public state", async () => { + const { server, workers, errors, behaviours } = await fleet(); + behaviours.set("S1", { fatal: "boom secret-token-123" }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("boom [redacted]"); + // Public: the state; the report's stderr tail is the app's to redact (a later test). + expect(JSON.stringify(workers.state.value)).not.toContain("secret"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: "boom [redacted]" }, + }); + notes.release(); + }); + + it("a link lost while its child lives (a blocked event loop past the heartbeat): killed after the grace, failed", async () => { + const { server, workers, starts, errors } = await fleet({ heartbeat: { ping: 100, close: 300 } }); + const notes = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S1" }, context); + await control.block(20_000, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "stopping", 5_000); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: "worker: link lost" }, + }); + expect(starts).toEqual(["session|S1"]); + notes.release(); + control.release(); + }, 20_000); + + it("an exit before started fails the start; a worker that never connects is killed at the start timer", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { exit: 3 }); + const exited = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker failed to start"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: expect.stringContaining("exited before it was ready (code 3)") }, + }); + exited.release(); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + behaviours.set("S2", { connect: false }); + const silent = server.hold(Notes, { id: "S2" }, context); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S2")?.status).toBe("starting"), { + timeout: 5_000, + }); + await vi.advanceTimersByTimeAsync(60_000); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S2")?.status).toBe("failed"), { timeout: 5_000 }); + expect(stateOf(workers.state.value, "S2")?.error?.message).toBe("worker failed to start"); + expect(errors.findLast(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: expect.stringContaining("did not start within 60 s") }, + }); + silent.release(); + } finally { + vi.useRealTimers(); + } + }); + + it("the start timer covers only the start: a slow load after connecting is not killed (Q4)", async () => { + const { server, workers, behaviours } = await fleet(); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + behaviours.set("S1", { hang: true }); + const notes = server.hold(Notes, { id: "S1" }, context); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"), { + timeout: 5_000, + }); + await new Promise((resolve) => setImmediate(resolve)); + await vi.advanceTimersByTimeAsync(120_000); + expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"); + expect(notes.state.value.kind).toBe("loading"); + notes.release(); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("keep-alive (D9, R3, Q5)", () => { + it("stops a worker idle since idleMs; a hold keeps one running; busy keeps it too", async () => { + const { server, workers } = await fleet({ idleMs: 200 }); + const held = await server.get(Notes, { id: "held" }, context); + const released = await server.get(Notes, { id: "free" }, context); + const control = await server.get(Control, { id: "busy" }, context); + await control.busy(1_500, context); + control.release(); + released.release(); + await until(() => stateOf(workers.state.value, "free") === undefined, 5_000); + await new Promise((resolve) => setTimeout(resolve, 600)); + expect(stateOf(workers.state.value, "held")?.status).toBe("running"); + expect(stateOf(workers.state.value, "busy")?.status).toBe("running"); + await until(() => stateOf(workers.state.value, "busy") === undefined, 5_000); + held.release(); + }, 20_000); + + it("1,000 sequential local calls in a worker nobody holds send no message (Sol)", async () => { + const { server, behaviours, output } = await fleet(); + behaviours.set("S1", { localCalls: 1_000 }); + // Released at once: the calls run 300 ms after the load, with no hold arriving (no reference keeps it busy). + (await server.get(Notes, { id: "S1" }, context)).release(); + await until(() => output.some((line) => line.startsWith("session|S1 local calls sent"))); + expect(output).toContain("session|S1 local calls sent 0"); + }); +}); + +describe("stop", () => { + it("suspends while held: holders fail waiting suspended, nothing starts; forgotten with the last holder (Q3)", async () => { + const { server, workers, starts } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await workers.stop(Session({ id: "S1" }), context); + expect(stateOf(workers.state.value, "S1")).toBeUndefined(); + await until(() => notes.state.value.kind === "failed"); + expect(notes.state.value).toMatchObject({ + cause: { kind: "waiting", error: { message: "session|S1: suspended" } }, + }); + expect(starts).toEqual(["session|S1"]); + notes.release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const again = await server.get(Notes, { id: "S1" }, context); + expect(starts).toEqual(["session|S1", "session|S1"]); + again.release(); + }); + + it("a stop during a start waits for it, then stops; a start during a stop waits for its exit", async () => { + const { server, workers, behaviours, starts } = await fleet(); + behaviours.set("S1", { loadMs: 300 }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + await workers.stop(Session({ id: "S1" }), context); + expect(stateOf(workers.state.value, "S1")).toBeUndefined(); + notes.release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + behaviours.set("S1", { log: undefined }); + const again = await server.get(Notes, { id: "S1" }, context); + const stopping = workers.stop(Session({ id: "S1" }), context); + again.release(); + await stopping; + const third = await server.get(Notes, { id: "S1" }, context); + expect(starts).toHaveLength(3); + third.release(); + }); + + it("runs the worker's closers in reverse order at its stop", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-workers-log-")); + const { server, workers, behaviours } = await fleet(); + behaviours.set("S1", { log: join(dir, "log"), closeMs: 50 }); + const notes = await server.get(Notes, { id: "S1" }, context); + notes.release(); + await workers.stop(Session({ id: "S1" }), context); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + await rm(dir, { recursive: true, force: true }); + }); +}); + +describe("close", () => { + it("cancels a start waiting in start(address): nothing forks; new starts reject; no worker_failed at shutdown", async () => { + let waiting = false; + const { server, workers, errors } = await fleet({ + start: async (address, startContext) => { + if (address.key === "session|wait") { + waiting = true; + await new Promise((_, reject) => + startContext.abortSignal?.addEventListener("abort", () => reject(startContext.abortSignal?.reason)), + ); + } + return {}; + }, + }); + const running = await server.get(Notes, { id: "S1" }, context); + const waits = server.hold(Notes, { id: "wait" }, context); + await until(() => waiting); + await workers.close(context); + expect(workers.state.value).toEqual({}); + expect(errors).toEqual([]); + const late = server.hold(Notes, { id: "late" }, context); + await until(() => late.state.value.kind === "failed"); + expect(late.state.value).toMatchObject({ cause: { error: { code: "released", reason: "closed" } } }); + for (const ref of [running, waits, late]) ref.release(); + }); +}); + +describe("the rollout step (§4)", () => { + it("ships the current set to running workers, retries held failed ones, lists each", async () => { + const { server, workers, plugins, label } = await fleet(); + const one = await server.get(Notes, { id: "S1" }, context); + const two = await server.get(Notes, { id: "S2" }, context); + const control = await server.get(Control, { id: "S3" }, context); + const failed = await server.get(Notes, { id: "S3" }, context); + await control.crash(1, context); + await until(() => stateOf(workers.state.value, "S3")?.status === "failed"); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const result = await plugins.rollout( + [workers.step("workers")], + await plugins.build(plugins.state.value.dirty, context), + context, + ); + const hosts = result.steps[0]!.hosts; + expect(listed(hosts).sort()).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ["session|S2", { added: [], changed: ["@test/notes"], removed: [] }], + ["session|S3", { added: ["@test/notes"], changed: [], removed: [] }], + ]); + await until(() => one.label.value === "S1 v2" && two.label.value === "S2 v2" && failed.label.value === "S3 v2"); + expect(plugins.state.value.dirty).toEqual([]); + for (const ref of [one, two, failed, control]) ref.release(); + }); + + it("lists a worker stopped cleanly during it as stopped and one that crashed with its error (it stays dirty)", async () => { + const { server, workers, plugins, label } = await fleet(); + const one = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S2" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const built = await plugins.build(plugins.state.value.dirty, context); + // Both workers block their event loops: their reloads wait; S1 is stopped meanwhile, S2 crashes before reading it. + const first = await server.get(Control, { id: "S1" }, context); + await first.block(400, context); + await control.blockThenCrash(400, context); + // Their event loops are blocked once their timers ran. + await new Promise((resolve) => setTimeout(resolve, 50)); + const rolling = plugins.rollout([workers.step("workers")], built, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const stopped = workers.stop(Session({ id: "S1" }), context); + const result = await rolling; + await stopped; + expect(listed(result.steps[0]!.hosts).sort()).toEqual([ + ["session|S1", "stopped"], + ["session|S2", "worker: its link is gone"], + ]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + for (const ref of [one, first, control]) ref.release(); + }); + + it("a worker starting during it gets the new set: a start after the rollout loads it", async () => { + const { server, workers, plugins, label } = await fleet(); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + await plugins.rollout( + [workers.step("workers")], + await plugins.build(plugins.state.value.dirty, context), + context, + ); + const notes = await server.get(Notes, { id: "S9" }, context); + expect(notes.label.value).toBe("S9 v2"); + notes.release(); + }); + + it("an aborted wait lists the worker aborted; its late reply keeps the child's slot until it arrives", async () => { + const { server, workers, plugins, label } = await fleet(); + const control = await server.get(Control, { id: "S1" }, context); + const notes = await server.get(Notes, { id: "S1" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const built = await plugins.build(plugins.state.value.dirty, context); + await control.block(300, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const abort = new AbortController(); + const step = workers.step("workers"); + const waited = plugins.rollout([step], built, withAbortSignal(abort.signal, context)); + setTimeout(() => abort.abort(new Error("stop waiting")), 50); + const first = await waited; + expect(listed(first.steps[0]!.hosts)).toEqual([["session|S1", "stop waiting"]]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + await label("v3"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const next = await plugins.rollout([step], await plugins.build(plugins.state.value.dirty, context), context); + expect(listed(next.steps[0]!.hosts)).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ]); + await until(() => notes.label.value === "S1 v3"); + control.release(); + notes.release(); + }); +}); + +describe("GC", () => { + // Real forks are costly (100 took the suite from 103 s to 700 s on one CPU at 1,000): 100 here; the 1,000 failed and + // retried addresses below cover the per-address records and suspensions at scale. + it("100 addresses started and stopped leave no state, no record, no suspension (Astra, Sol)", async () => { + const refs: WeakRef[] = []; + const { server, workers } = await fleet({ + start: async (address) => { + refs.push(new WeakRef(address)); + return {}; + }, + }); + // Twenty at a time: each a real fork, start, hold, stop and exit. + for (let batch = 0; batch < 5; batch++) { + await Promise.all( + Array.from({ length: 20 }, async (_, index) => { + const id = `R${batch * 20 + index}`; + const notes = await server.get(Notes, { id }, context); + notes.release(); + await workers.stop(Session({ id }), context); + }), + ); + } + expect(workers.state.value).toEqual({}); + await expectCollected(...refs); + }, 120_000); + + it("1,000 failed and retried addresses leave no state, no record and no suspension", async () => { + const refs: WeakRef[] = []; + const { server, workers } = await fleet({ + start: async (address) => { + refs.push(new WeakRef(address)); + throw new Error("no worker"); + }, + }); + for (let i = 0; i < 1_000; i++) { + const id = `A${i}`; + const ref = server.hold(Notes, { id }, context); + await until(() => ref.state.value.kind === "failed"); + ref.release(); + await workers.stop(Session({ id }), context); + await workers.retry(Session({ id }), context); + } + expect(workers.state.value).toEqual({}); + await expectCollected(...refs.slice(0, 50)); + }, 120_000); +}); + +describe("fix round 2", () => { + // Astra's recheck R3: a throwing `principal(address)` reached the guest with its message, the child left starting. + it("a throwing principal hook fails the start with the public message; the error stays private", async () => { + const { server, workers, errors } = await fleet({ + principal: () => { + throw new Error("secret token=sk-1 at /home/owner/private"); + }, + }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind === "failed"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { error: { message: "worker failed to start" } }, + }); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")).toMatchObject({ error: { message: "worker failed to start" } }); + // Opus's second recheck: the component's own kill of a failed start is no `worker_exited`. + await until(() => errors.length > 0); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(errors).toMatchObject([ + { kind: "worker_failed", address: "session|S1", error: { message: expect.stringContaining("secret") } }, + ]); + notes.release(); + }); + + it("a throwing authorize refuses the hold (forbidden); the error stays private, reported authorize_failed", async () => { + const { server, errors, starts } = await fleet({ + authorize: () => { + throw new Error("secret lookup"); + }, + }); + const remote = createHost({ onError: () => {} }); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "guest", id: "g" } }, context); + remote.connect(down, { principal: { kind: "server" } }, context); + await remote.load([], context); + const notes = remote.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind !== "loading"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "forbidden" } } }); + expect([errors, starts]).toMatchObject([[{ kind: "authorize_failed", address: "session|S1" }], []]); + notes.release(); + await remote.close(context); + }); +}); + +describe("fix round 1", () => { + it("close during a forked start that never connects returns at once, no failure (Opus F1)", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { connect: false }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "starting"); + const started = Date.now(); + await workers.close(context); + expect(Date.now() - started).toBeLessThan(5_000); + expect(errors).toEqual([]); + notes.release(); + }); + + it("close during an app's start that ignores its signal returns at once; nothing forks after it (Sol)", async () => { + let forked = false; + const { server, workers, errors } = await fleet({ start: () => new Promise(() => {}) }); + const notes = server.hold(Notes, { id: "S1" }, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + await workers.close(context); + forked = stateOf(workers.state.value, "S1") !== undefined; + expect([forked, errors]).toEqual([false, []]); + notes.release(); + }); + + it("a stop during the worker's startup is a stop: its closers run, nothing fails (Astra)", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-workers-log-")); + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { loadMs: 5_000, closeMs: 200, log: join(dir, "log") }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + await workers.stop(Session({ id: "S1" }), context); + expect(errors).toEqual([]); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + notes.release(); + await rm(dir, { recursive: true, force: true }); + }); + + it("a reload queued behind one the child never answers settles when the child exits (Astra)", async () => { + const { server, workers, plugins, label } = await fleet(); + const control = await server.get(Control, { id: "S1" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.length > 0); + const built = await plugins.build(plugins.state.value.dirty, context); + await control.blockThenCrash(600, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const first = plugins.rollout([workers.step("one")], built, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const second = plugins.rollout([workers.step("two")], built, context); + const outcomes = await Promise.all([first, second]); + expect(outcomes.map((result) => listed(result.steps[0]!.hosts))).toEqual([ + [["session|S1", "worker: its link is gone"]], + [["session|S1", "worker: its link is gone"]], + ]); + control.release(); + }); + + it("a plugin whose evaluation failed is shipped again by the next rollout and stays dirty (Astra)", async () => { + const { server, workers, plugins, root } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await writeFile( + join(root, "notes", "label.ts"), + 'throw new Error("evaluation failed");\nexport const label = "v2";\n', + ); + await until(() => plugins.state.value.dirty.length > 0); + const built = await plugins.build(plugins.state.value.dirty, context); + const failed = (result: { steps: readonly { hosts: readonly RolloutHost[] }[] }) => { + const [entry] = result.steps[0]!.hosts; + return entry !== undefined && "report" in entry ? entry.report.failed.map(({ plugin }) => plugin) : entry; + }; + expect(failed(await plugins.rollout([workers.step("one")], built, context))).toEqual(["@test/notes"]); + expect(failed(await plugins.rollout([workers.step("two")], built, context))).toEqual(["@test/notes"]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + expect(notes.label.value).toBe("S1 v1"); + notes.release(); + }); + + it("two rollouts started together each ship their own build; the later one stays (Astra)", async () => { + const { server, workers, plugins, label } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await label("v2"); + const v2 = await plugins.build(["@test/notes"], context); + await label("v3"); + const v3 = await plugins.build(["@test/notes"], context); + const results = await Promise.all([ + plugins.rollout([workers.step("v2")], v2, context), + plugins.rollout([workers.step("v3")], v3, context), + ]); + for (const result of results) { + expect(listed(result.steps[0]!.hosts)).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ]); + } + await until(() => notes.label.value === "S1 v3"); + notes.release(); + }); + + it("reaps a worker that closed its own IPC channel and lives on: killed after the grace, failed (Sol)", async () => { + const { server, workers, behaviours } = await fleet(); + behaviours.set("S1", { disconnectAndBlock: true }); + const notes = await server.get(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); + await until(() => notes.state.value.kind === "failed"); + notes.release(); + }, 15_000); + + it("keeps no reload report once consumed: the reload queue settles to nothing (Sol)", async () => { + const env = await fleet(); + const notes = await env.server.get(Notes, { id: "S1" }, context); + const report = await (async () => { + await env.label("v2"); + const built = await env.plugins.build(["@test/notes"], context); + const result = await env.plugins.rollout([env.workers.step("workers")], built, context); + const [entry] = result.steps[0]!.hosts; + if (entry === undefined || !("report" in entry)) throw new Error("no report"); + return new WeakRef(entry.report); + })(); + await expectCollected(report); + notes.release(); + }); + + it("keeps a worker's stderr out of the public state: only worker_failed carries its tail (Astra)", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { fatal: "boom secret-token-123" }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(JSON.stringify(workers.state.value)).not.toContain("secret"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + const failure = errors.find(({ kind }) => kind === "worker_failed"); + expect(failure).toMatchObject({ stderrTail: expect.stringContaining("secret-token-123") }); + notes.release(); + }); +}); diff --git a/packages/chord/test/next/bench/calls.ts b/packages/chord/test/next/bench/calls.ts new file mode 100644 index 000000000..d8e5dc0f7 --- /dev/null +++ b/packages/chord/test/next/bench/calls.ts @@ -0,0 +1,90 @@ +// Benchmark of the paths slice 8 touched under every object (no registries involved): a local method call (its ctx), +// a commit through a watched derive (the batch, a derive's compute) and activation churn (an object with params made, +// live and ended per get: its attempt's declarations). Median µs per operation over bursts, and heap allocated per +// operation (growth plus what the collections during the burst freed). Runs on HEAD as it is (copy it there). +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/calls.ts +import { performance } from "node:perf_hooks"; +import { GCProfiler } from "node:v8"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const BURSTS = 9; + +const gc = (globalThis as { gc?: () => void }).gc!; + +const Counter = defineState("bench.counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Store = local("bench.store", { + version: 1, + params: {}, + members: { doubled: int(), bump: method(), set: method(int()) }, +}); +const Item = local("bench.item", { version: 1, params: { id: int() }, members: { n: int() } }); + +const host = createHost({ onError: (error) => console.log("onError", error.kind) }); +await host.load( + [ + { + name: "@bench/calls", + facet: defineFacet((h) => { + h.serve(Store, (_, self) => { + const counter = self.state(Counter); + return { + doubled: self.derive([counter], ([c]) => c.n * 2), + bump: async () => null, + set: async (n, ctx) => { + await counter.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Item, ({ id }, self) => ({ n: self.derive([self.get(Store, {}).doubled], ([d]) => d + id) })); + }), + }, + ], + context, +); + +const store = await host.get(Store, {}, context); +const watch = await store.doubled.watch(context); +watch.start(async () => {}); + +/** Median µs per operation over the bursts (after one warm-up burst), and heap bytes allocated per operation. */ +async function measure(runs: number, operation: (i: number) => Promise): Promise { + const times: number[] = []; + let bytes = 0; + let counted = 0; + for (let burst = -1; burst < BURSTS; burst++) { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + const started = performance.now(); + for (let i = 0; i < runs; i++) await operation(i); + const elapsed = performance.now() - started; + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + if (burst < 0) continue; + times.push((elapsed * 1000) / runs); + bytes += after - before + freed; + counted += runs; + } + times.sort((a, b) => a - b); + return `${times[Math.floor(times.length / 2)]!.toFixed(2)} µs, ${Math.round(bytes / counted)} B allocated`; +} + +console.log("local method call:", await measure(20_000, () => store.bump(context))); +console.log("commit through a watched derive:", await measure(10_000, (i) => store.set(i + 1, context))); +console.log( + "activation churn (get, live, release):", + await measure(2_000, async (i) => (await host.get(Item, { id: i }, context)).release()), +); +await watch.stop(); +await host.close(context); diff --git a/packages/chord/test/next/bench/compression.ts b/packages/chord/test/next/bench/compression.ts new file mode 100644 index 000000000..15e7024ce --- /dev/null +++ b/packages/chord/test/next/bench/compression.ts @@ -0,0 +1,163 @@ +// Light benchmark of compression at the origin (slice 4b): a 500 KB snapshot and a 1 MiB call result, uncompressed and +// deflated (level 1) by the server; their bytes on the wire, the deflate at the origin, the inflate at the owner +// (synchronous `nodeCompression`, asynchronous `DecompressionStream`), and end to end over a memory link. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/compression.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { DeflatedPart, inflateAsync } from "../../../src/next/link/internal.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; + +const Server = hostKind("server"); + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); + +const Store = local("bench.store", { version: 1, params: {}, members: { doc: obj({ entries: arr(Entry) }) } }); + +/** A tab's view of the 500 KB document, and a method returning 1 MiB. */ +const View = remote("bench.view", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { owner: { state: { doc: obj({ entries: arr(Entry) }) }, methods: { export: method(undefined, str()) } } }, +}); + +/** Words that make text compress like prose and code (about 3x at level 1), not like a repeated string. */ +const WORDS = "the a model tool call file read write test error result agent session note code run build".split(" "); + +/** About `length` characters of words, from a fixed pseudo-random sequence starting at `seed`. */ +function prose(seed: number, length: number): string { + let state = seed * 7919 + 1; + const words: string[] = []; + for (let total = 0; total < length; total += words.at(-1)!.length + 1) { + state = (state * 1103515245 + 12345) % 2147483648; + const word = WORDS[state % WORDS.length]!; + words.push(state % 5 === 0 ? `${word}${state % 1000}` : word); + } + return words.join(" ").slice(0, length); +} + +/** The 500 KB document: 500 entries of 960 characters, as in the slice 3a benchmark, with prose-like text. */ +const entries = Array.from({ length: 500 }, (_, id) => ({ + id, + text: prose(id, 960), + author: "mario", + at: 1_700_000_000 + id, +})); + +/** The 1 MiB result. */ +const RESULT = prose(1, 1024 * 1024); + +const Doc = defineState("doc", { schema: obj({ entries: arr(Entry) }), version: 1, initial: () => ({ entries }) }); + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => ({ doc: self.state(Doc) })); + host.serve(View, (_, _principal, self) => ({ + variant: "owner", + members: { doc: self.get(Store, {}).doc, export: async () => RESULT }, + })); +}); + +/** A link end counting the bytes its host receives. */ +function counted(link: Link, bytes: { received: number }): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + bytes.received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** The mean milliseconds of `runs` runs of `work`, the fastest of three such loops. */ +async function mean(runs: number, work: () => unknown): Promise { + let best = Number.POSITIVE_INFINITY; + for (let loop = 0; loop < 3; loop++) { + const started = performance.now(); + for (let run = 0; run < runs; run++) await work(); + best = Math.min(best, (performance.now() - started) / runs); + } + return best; +} + +const UTF8 = new TextEncoder(); + +const kib = (bytes: number) => `${(bytes / 1024).toFixed(1)} KiB`; + +const ms = (value: number) => `${value.toFixed(3)} ms`; + +console.log( + "| Payload | JSON | deflated (level 1) | deflate at the origin | inflate, sync (`nodeCompression`) | inflate, async (`DecompressionStream`) |", +); +console.log("| --- | --- | --- | --- | --- | --- |"); +for (const [name, json] of [ + ["500 KB snapshot", JSON.stringify({ entries })], + ["1 MiB result", JSON.stringify(RESULT)], +] as const) { + const bytes = UTF8.encode(json); + const deflated = nodeCompression.deflate(bytes); + const deflate = await mean(20, () => nodeCompression.deflate(bytes)); + const inflate = await mean(20, () => nodeCompression.inflate(deflated, Number.POSITIVE_INFINITY)); + const inflateAsyncMs = await mean(20, () => + inflateAsync(new DeflatedPart(deflated), Number.POSITIVE_INFINITY, new AbortController().signal, context), + ); + console.log( + `| ${name} | ${kib(bytes.byteLength)} | ${kib(deflated.byteLength)} | ${ms(deflate)} | ${ms(inflate)} | ${ms(inflateAsyncMs)} |`, + ); +} + +/** A server compressing with `server` and a client with `client` (none: it inflates asynchronously). */ +async function linked(server: Compression | undefined, client: Compression | undefined) { + const serverHost = createHost

({ + address: Server(), + principal: { kind: "server" }, + compression: server, + onError: console.error, + }); + await serverHost.load([{ name: "@bench/app", facet }], context); + const clientHost = createHost

({ compression: client, onError: console.error }); + await clientHost.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + serverHost.accept(a, { principal: { kind: "owner" } }, context); + clientHost.connect(counted(b, bytes), { principal: { kind: "server" } }, context); + return { server: serverHost, client: clientHost, bytes }; +} + +console.log(); +console.log( + "| End to end over a memory link (binary codec) | snapshot: bytes received | hold to live (mean of 20) | 1 MiB result: bytes received | call to result (mean of 20) |", +); +console.log("| --- | --- | --- | --- | --- |"); +for (const [label, server, client] of [ + ["uncompressed", undefined, undefined], + ["deflated, inflated synchronously (Node client)", nodeCompression, nodeCompression], + ["deflated, inflated asynchronously (browser client)", nodeCompression, undefined], +] as const) { + const hosts = await linked(server, client); + // Warm up: the store and the view activate once. + (await hosts.client.get(View, {}, context)).release(); + const before = hosts.bytes.received; + const view = await hosts.client.get(View, {}, context); + const snapshotBytes = hosts.bytes.received - before; + view.release(); + const hold = await mean(20, async () => (await hosts.client.get(View, {}, context)).release()); + const held = await hosts.client.get(View, {}, context); + const beforeCall = hosts.bytes.received; + await held.export(context); + const resultBytes = hosts.bytes.received - beforeCall; + const call = await mean(20, () => held.export(context)); + console.log(`| ${label} | ${kib(snapshotBytes)} | ${ms(hold)} | ${kib(resultBytes)} | ${ms(call)} |`); + await hosts.client.close(context); + await hosts.server.close(context); +} diff --git a/packages/chord/test/next/bench/derives.ts b/packages/chord/test/next/bench/derives.ts new file mode 100644 index 000000000..7c2785173 --- /dev/null +++ b/packages/chord/test/next/bench/derives.ts @@ -0,0 +1,73 @@ +// Benchmark of a retirement through derives (slice 6a rule 21): a singleton under a diamond of 40 derives (13 layers of +// two derives joined by a third, 2^13 paths from the top to the bottom), its bottom watched; each step retires the +// document and creates it again in one transaction, so every step carries the `retired` flag through the diamond. +// Median and p95 per step over several bursts. The flag reaches each derive once per retirement, not once per path. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/derives.ts +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Self } from "../../../src/next/host/index.ts"; +import type { Watchable } from "../../../src/watch.ts"; + +const STEPS = 1_000; +const RUNS = 7; +const LAYERS = 13; + +const List = defineState("bench.list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => ({ items: [] }), +}); +const Diamond = local("bench.diamond", { version: 1, params: {}, members: { bottom: int(), touch: method() } }); + +/** The diamond over `list`: its bottom derive. */ +function diamond(self: Self, list: SingletonHandle<{ items: string[] }>): Watchable { + let previous: Watchable = self.derive([list], ([value]) => value.items.length); + for (let layer = 0; layer < LAYERS; layer++) { + const left = self.derive([previous], ([n]) => n + 1); + const right = self.derive([previous], ([n]) => n + 2); + previous = self.derive([left, right], ([a, b]) => (a + b) % 1_000); + } + return previous; +} + +/** Per-step times (ms) of `RUNS` bursts of `STEPS` retire-and-recreate transactions, after one warm-up burst. */ +async function bursts(): Promise<{ readonly median: number; readonly p95: number }> { + const times: number[] = []; + for (let run = -1; run < RUNS; run++) { + let list: SingletonHandle<{ items: string[] }> | undefined; + let self: Self | undefined; + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Diamond, (_, declared) => { + self = declared; + list = declared.state(List); + return { bottom: diamond(declared, list), touch: async () => null }; + }), + ); + await host.load([{ name: "@bench/derives", facet }], context); + const reference = await host.get(Diamond, {}, context); + const watch = await reference.bottom.watch(context); + watch.start(async () => {}); + for (let step = 0; step < STEPS; step++) { + const before = performance.now(); + await self!.tx(async (tx) => { + await tx.retire(list!); + (await tx.draft(list!)).items.push(`${step}`); + }, context); + if (run >= 0) times.push(performance.now() - before); + } + await watch.stop(); + reference.release(); + await host.close(context); + } + times.sort((a, b) => a - b); + return { median: times[Math.floor(times.length / 2)]!, p95: times[Math.floor(times.length * 0.95)]! }; +} + +const { median, p95 } = await bursts(); +console.log( + `retire + recreate under a ${1 + 3 * LAYERS}-derive diamond: ${(median * 1000).toFixed(1)} µs median, ${(p95 * 1000).toFixed(1)} µs p95`, +); diff --git a/packages/chord/test/next/bench/http.ts b/packages/chord/test/next/bench/http.ts new file mode 100644 index 000000000..21f3b2274 --- /dev/null +++ b/packages/chord/test/next/bench/http.ts @@ -0,0 +1,120 @@ +// Benchmark of the HTTP component (slice 10 §7): a route's overhead per request against a plain Node handler (keep-alive +// round trips, median µs per request over bursts), and `bytes` throughput for a 100 MiB file (`fileResponse`) against +// `pipeline` of a read stream. +// Run by hand from packages/chord (not in CI), pinned: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/http.ts +import { createReadStream } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { Agent, createServer, request } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { pipeline } from "node:stream/promises"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { createHttp, fileResponse, Http } from "../../../src/next-node/http/index.ts"; + +const BURSTS = 7; +const REQUESTS = 2_000; +const FILE = 100 * 1024 * 1024; + +/** GETs `path` on `port` over `agent`, reading the whole body; resolves with its length. */ +function get(port: number, path: string, agent: Agent): Promise { + return new Promise((resolve, reject) => { + const req = request({ host: "127.0.0.1", port, path, agent }, (res) => { + let length = 0; + res.on("data", (chunk: Buffer) => { + length += chunk.length; + }); + res.on("end", () => resolve(length)); + res.on("error", reject); + }); + req.on("error", reject); + req.end(); + }); +} + +/** The median µs per request of `REQUESTS` sequential keep-alive requests, over bursts. */ +async function perRequest(port: number, path: string): Promise { + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + for (let i = 0; i < 200; i++) await get(port, path, agent); + const times: number[] = []; + for (let burst = 0; burst < BURSTS; burst++) { + const start = performance.now(); + for (let i = 0; i < REQUESTS; i++) await get(port, path, agent); + times.push(((performance.now() - start) * 1000) / REQUESTS); + } + agent.destroy(); + return median(times); +} + +/** MiB/s of downloading `path` once (after one warm-up). */ +async function throughput(port: number, path: string): Promise { + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + await get(port, path, agent); + const times: number[] = []; + for (let burst = 0; burst < 3; burst++) { + const start = performance.now(); + await get(port, path, agent); + times.push(performance.now() - start); + } + agent.destroy(); + return FILE / 1024 / 1024 / (median(times) / 1000); +} + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +const dir = await mkdtemp(join(tmpdir(), "chord-bench-http-")); +const file = join(dir, "big.bin"); +await writeFile(file, Buffer.alloc(FILE, 1)); + +// A plain Node server: the same JSON body, the same file through `pipeline`. +const plain = createServer((req, res) => { + if (req.url === "/file") { + res.writeHead(200, { "Content-Type": "application/octet-stream", "Content-Length": String(FILE) }); + void pipeline(createReadStream(file), res).catch(() => {}); + return; + } + const body = JSON.stringify({ ok: true }); + res.writeHead(200, { "Content-Type": "application/json; charset=utf-8", "Content-Length": String(body.length) }); + res.end(body); +}); +await new Promise((resolve) => plain.listen(0, "127.0.0.1", resolve)); +const plainPort = (plain.address() as { port: number }).port; + +// The component with Chord's own listener: one plugin with the same route and the file. +const host = createHost({ principal: { kind: "server" }, onError: () => {} }); +const http = createHttp(host, { listen: { host: "127.0.0.1", port: 0 }, onError: () => {} }, context); +const plugin = defineFacet((_host, self) => { + self.contribute(Http, { + routes: { + "GET /json": async () => ({ kind: "json", value: { ok: true } }), + "GET /file": async () => fileResponse(file), + }, + }); +}); +await host.load( + [ + { name: "chord.http", facet: http.facet }, + { name: "@bench/routes", facet: plugin }, + ], + context, +); +const { port } = await http.listening(context); + +const plainRequest = await perRequest(plainPort, "/json"); +const routeRequest = await perRequest(port, "/json"); +console.log( + `request round trip: plain ${plainRequest.toFixed(1)} µs, route ${routeRequest.toFixed(1)} µs (overhead ${(routeRequest - plainRequest).toFixed(1)} µs, ${(((routeRequest - plainRequest) / plainRequest) * 100).toFixed(1)}%)`, +); +const plainBytes = await throughput(plainPort, "/file"); +const routeBytes = await throughput(port, "/file"); +console.log(`100 MiB file: pipeline ${plainBytes.toFixed(0)} MiB/s, fileResponse ${routeBytes.toFixed(0)} MiB/s`); + +await http.close(context); +await host.close(context); +plain.close(); +await rm(dir, { recursive: true, force: true }); diff --git a/packages/chord/test/next/bench/jobs.ts b/packages/chord/test/next/bench/jobs.ts new file mode 100644 index 000000000..87cce72d0 --- /dev/null +++ b/packages/chord/test/next/bench/jobs.ts @@ -0,0 +1,164 @@ +// Benchmark of persisted jobs (slice 6b §4, §5): a run's start (its record and state created, resolved after the +// commit) and a state change inside `run` (immediate, and deferred), for a persisted job on a SQLite file against the +// same job in memory. Median and p95 per operation over several bursts. A disk commit is not held to the in-memory +// number: the persisted columns show what a run's records cost on the store. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle, type RunState } from "../../../src/next/jobs/index.ts"; +import { openStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const OPERATIONS = 500; +const RUNS = 7; + +type Progress = { done: number }; + +const InMemoryProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), +}); + +const PersistedProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 64, +}); + +const InMemoryImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: InMemoryProgress, +}); + +const PersistedImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: PersistedProgress, + persist: { resume: "rerun" }, +}); + +const Worker = local("bench.worker", { version: 1, params: {}, members: {} }); + +/** Where a run's `run` parks: its state, and the gate that ends it. */ +type Parked = { readonly state: RunState; readonly end: () => void }; + +/** A host serving `Worker` with the job (persisted on a fresh SQLite file, or in memory), its runs parked. */ +async function served(persisted: boolean) { + const directory = await mkdtemp(join(tmpdir(), "chord-bench-jobs-")); + const store = persisted + ? await openStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "store.sqlite"))), context) + : undefined; + const parked: Parked[] = []; + const waiting: ((parked: Parked) => void)[] = []; + let jobs: JobHandle<{ path: string }, { count: number }, Progress> | undefined; + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Worker, (_, self) => { + jobs = self.job(persisted ? PersistedImport : InMemoryImport, { + run: (_args, ctx) => + new Promise((resolve) => { + const run = { state: ctx.state, end: () => resolve({ count: ctx.state.value.done }) }; + const next = waiting.shift(); + if (next === undefined) parked.push(run); + else next(run); + }), + }); + return {}; + }), + ); + await host.load([{ name: "@bench/jobs", facet }], context); + const reference = await host.get(Worker, {}, context); + return { + jobs: jobs!, + /** The parked `run` of the next run invoked. */ + next: () => new Promise((resolve) => waiting.push(resolve)), + close: async () => { + reference.release(); + await host.close(context); + await store?.close(context); + await rm(directory, { recursive: true, force: true }); + }, + }; +} + +/** Per-operation times (ms) of `RUNS` bursts, after one warm-up burst; `burst` does `OPERATIONS` timed operations. */ +async function bursts( + persisted: boolean, + burst: ( + worker: Awaited>, + time: (operation: () => Promise) => Promise, + ) => Promise, +) { + const times: number[] = []; + for (let run = -1; run < RUNS; run++) { + const worker = await served(persisted); + await burst(worker, async (operation) => { + const before = performance.now(); + await operation(); + if (run >= 0) times.push(performance.now() - before); + }); + await worker.close(); + } + times.sort((a, b) => a - b); + return { median: times[Math.floor(times.length / 2)]!, p95: times[Math.floor(times.length * 0.95)]! }; +} + +/** Starts: each run's start resolved (record and state committed), its `run` parked, all ended after the burst. */ +function starts(persisted: boolean) { + return bursts(persisted, async ({ jobs, next }, time) => { + const parked: Parked[] = []; + for (let index = 0; index < OPERATIONS; index++) { + const invoked = next(); + await time(() => jobs.start({ path: `p${index}` }, context)); + parked.push(await invoked); + } + for (const run of parked) run.end(); + }); +} + +/** State changes: `OPERATIONS` changes of one run's state, immediate or deferred (stored by the outcome). */ +function changes(persisted: boolean, deferred: boolean) { + const options = deferred ? { persistence: { kind: "deferred", withinMs: Infinity } as const } : undefined; + return bursts(persisted, async ({ jobs, next }, time) => { + const invoked = next(); + await jobs.start({ path: "p" }, context); + const { state, end } = await invoked; + for (let index = 0; index < OPERATIONS; index++) { + await time(() => + state.change( + (draft) => { + draft.done = index + 1; + }, + context, + options, + ), + ); + } + end(); + }); +} + +function format({ median, p95 }: { readonly median: number; readonly p95: number }): string { + return `median ${(median * 1000).toFixed(1)} µs, p95 ${(p95 * 1000).toFixed(1)} µs`; +} + +console.log(`start, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await starts(false))}`); +console.log(` persisted: ${format(await starts(true))}`); +console.log(`state change, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await changes(false, false))}`); +console.log(` persisted, immediate: ${format(await changes(true, false))}`); +console.log(` persisted, deferred: ${format(await changes(true, true))}`); diff --git a/packages/chord/test/next/bench/links.ts b/packages/chord/test/next/bench/links.ts new file mode 100644 index 000000000..1a57419bf --- /dev/null +++ b/packages/chord/test/next/bench/links.ts @@ -0,0 +1,507 @@ +// Light benchmarks of remote objects over one memory link (slices 3a, 3b), per codec side by side: wire bytes per +// message; compute (encode and decode time per message, commit-to-frame end to end); memory (heap allocated per message +// while encoding and decoding, retained per hold); drops (3b: reconnects re-holding 1 and 100 references, 1,000 calls +// parked while the link is down). Payloads are modelled on p2: a notes list, a 500 KB conversation document, a reply +// streamed token by token, calls with and without bytes. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/links.ts +import { GCProfiler } from "node:v8"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { BACKGROUND_CONTEXT, type Context, withTelemetry } from "../../../src/context/index.ts"; +import { arr, bytes, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host, type RefState } from "../../../src/next/host/index.ts"; +import { type Codec, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import type { NativeWatchable } from "../../../src/watch.ts"; + +const Server = hostKind("server"); + +type P = { kind: "owner" } | { kind: "server" }; + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); +const Shape = obj({ entries: arr(Entry) }); + +/** The shared store: one document of entries, and the writes the benchmark makes. */ +const Store = local("bench.store", { + version: 1, + params: { name: str() }, + members: { + doc: Shape, + append: method(Entry), + edit: method(obj({ index: int(), text: str() })), + remove: method(int()), + stream: method(str()), + }, +}); + +/** The remote view a client holds: the store's document as it is. */ +const View = remote("bench.view", { + version: 1, + params: { name: str() }, + hostAddress: () => Server(), + variants: { owner: { state: { doc: Shape } } }, +}); + +/** A file service: a call with a small argument, and one with bytes. */ +const Files = remote("bench.files", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + owner: { methods: { ping: method(obj({ n: int() }), int()), put: method(obj({ data: bytes() }), int()) } }, + }, +}); + +const Doc = defineState("doc", { schema: Shape, version: 1, initial: () => ({ entries: [] }) }); + +/** The `n` of every `ping` in the order the provider ran them. */ +const pings: number[] = []; + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Doc); + const change = ( + fn: (draft: { entries: { id: number; text: string; author: string; at: number }[] }) => void, + ctx: Context, + ) => doc.change((draft) => void fn(draft), ctx).then(() => null); + return { + doc, + append: (entry, ctx) => change((d) => void d.entries.push(entry), ctx), + edit: ({ index, text }, ctx) => + change((d) => { + d.entries[index]!.text = text; + }, ctx), + remove: (index, ctx) => change((d) => void d.entries.splice(index, 1), ctx), + stream: (token, ctx) => + change((d) => { + d.entries[d.entries.length - 1]!.text += token; + }, ctx), + }; + }); + host.serve(View, ({ name }, _principal, self) => ({ + variant: "owner", + members: { doc: self.get(Store, { name }).doc }, + })); + host.serve(Files, () => ({ + variant: "owner", + members: { + ping: async ({ n }) => { + pings.push(n); + return n + 1; + }, + put: async ({ data }) => data.byteLength, + }, + })); +}); + +/** A link end counting what it sends and keeping the last message. */ +function counted(link: Link): Link & { bytes: number; messages: number; last: Uint8Array | undefined } { + const counting = { + bytes: 0, + messages: 0, + last: undefined as Uint8Array | undefined, + send(message: Uint8Array) { + counting.bytes += message.byteLength; + counting.messages++; + counting.last = message; + link.send(message); + }, + onMessage: (listener: (message: Uint8Array) => void) => link.onMessage(listener), + close: () => link.close(), + onClose: (listener: () => void) => link.onClose(listener), + }; + return counting; +} + +const telemetry = new InMemoryTelemetryContext(); +const span = await telemetry.startSpan({ name: "bench" }, (s) => s); +const counting = withTelemetry(span, BACKGROUND_CONTEXT); + +/** The encodes counted so far (the in-memory adapter records each add). */ +const encodes = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.link.encodes") + .reduce((sum, { value }) => sum + value, 0); + +type Entry = { id: number; text: string; author: string; at: number }; + +function entry(id: number, length: number): Entry { + return { id, text: "x".repeat(length), author: "mario", at: 1_700_000_000_000 + id }; +} + +const notes = (length = 1_000): Entry[] => Array.from({ length }, (_, id) => entry(id, 80)); + +const document = (): Entry[] => Array.from({ length: 500 }, (_, id) => entry(id, 960)); + +/** + * A server with `entries` in store `name`, a client connected over a counted link in `codec`. `metered: false` leaves + * telemetry out (the in-memory adapter keeps every record, which a memory measurement would count). + */ +async function setup(name: string, entries: Entry[], codec: Codec, metered = true) { + const context = metered ? counting : BACKGROUND_CONTEXT; + const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + telemetry: metered ? telemetry : undefined, + onError: console.error, + }); + await server.load([{ name: "@bench/app", facet }], context); + const store = await server.get(Store, { name }, context); + for (const item of entries) await store.append(item, context); + const [a, b] = memoryLinkPair(); + const down = counted(a); + const up = counted(b); + server.accept(down, { principal: { kind: "owner" }, codec }, context); + const client = createHost

({ onError: console.error }); + client.connect(up, { principal: { kind: "server" }, codec }, context); + await client.load([], context); + const close = async () => { + await client.close(context); + await server.close(context); + }; + return { server, client, store, down, up, close, context }; +} + +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Heap and `ArrayBuffer` bytes: the message bytes themselves live outside the heap. */ +type Memory = { readonly heap: number; readonly arrayBuffers: number }; + +/** Waits until queued deliveries ran and collects garbage, repeatedly, then returns the memory in use. */ +async function settledMemory(): Promise { + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + const { heapUsed, arrayBuffers } = process.memoryUsage(); + return { heap: heapUsed, arrayBuffers }; +} + +/** Heap bytes `work` allocated: the growth plus what the collections it caused freed. */ +function allocated(work: () => void): number { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + work(); + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + return after - before + freed; +} + +/** + * `ArrayBuffer` bytes one encoded message keeps alive (its backing store, which may be larger than the message), over + * `runs` messages kept at once. Freed buffers are not reported reliably during a loop, so this measures what is kept. + */ +async function retainedBuffers(codec: Codec, message: Message, runs: number): Promise { + const before = (await settledMemory()).arrayBuffers; + const kept = Array.from({ length: runs }, () => codec.encode(message)); + const after = (await settledMemory()).arrayBuffers; + return kept.length === runs ? (after - before) / runs : 0; +} + +/** + * One message through a codec, `runs` times: wire bytes, encode and decode time per message, and heap allocated per + * message while encoding and decoding. + */ +function codecRun(codec: Codec, message: Message, runs: number) { + const bytes = codec.encode(message); + for (let i = 0; i < 100; i++) codec.decode(codec.encode(message)); + // The fastest of five loops: one loop of a microsecond-sized message is at the mercy of the collector and the JIT. + let encode = Number.POSITIVE_INFINITY; + let decode = Number.POSITIVE_INFINITY; + for (let repeat = 0; repeat < 5; repeat++) { + const t0 = performance.now(); + for (let i = 0; i < runs; i++) codec.encode(message); + const t1 = performance.now(); + for (let i = 0; i < runs; i++) codec.decode(bytes); + const t2 = performance.now(); + encode = Math.min(encode, (t1 - t0) / runs); + decode = Math.min(decode, (t2 - t1) / runs); + } + const heap = allocated(() => { + for (let i = 0; i < runs; i++) codec.decode(codec.encode(message)); + }); + return { bytes: bytes.byteLength, encode, decode, heap: heap / runs }; +} + +const CODECS = [binaryCodec, textCodec]; + +/** Results by row label, one value per codec. */ +const table = new Map(); + +const record = (label: string, codec: Codec, value: string): void => { + const row = table.get(label) ?? []; + row[CODECS.indexOf(codec)] = value; + table.set(label, row); +}; + +const ms = (value: number) => `${value.toFixed(3)} ms`; +const us = (value: number) => `${(value * 1000).toFixed(1)} µs`; +const kb = (value: number) => `${(value / 1024).toFixed(1)} KiB`; +const b = (value: number) => (value >= 10_240 ? kb(value) : `${value.toFixed(0)} B`); + +/** Records a captured message's codec run under `label`. */ +async function codecRows(label: string, codec: Codec, message: Message, runs: number): Promise { + const run = codecRun(codec, message, runs); + record(`${label}: wire bytes`, codec, b(run.bytes)); + record(`${label}: encode / decode`, codec, `${us(run.encode)} / ${us(run.decode)}`); + record(`${label}: heap per encode + decode`, codec, b(run.heap)); + record(`${label}: array buffer kept per encoded message`, codec, b(await retainedBuffers(codec, message, 50))); +} + +/** A snapshot over a link: hold to live, then the captured `bound` through the codec alone. */ +async function snapshot(label: string, codec: Codec, entries: Entry[]): Promise { + const { client, down, close } = await setup(label, entries, codec); + let start = performance.now(); + // The first hold also activates the view; it stays held, so later holds measure the snapshot path alone. + await client.get(View, { name: label }, counting); + record(`${label}: first hold to live (activation, one sample)`, codec, ms(performance.now() - start)); + const runs = 100; + start = performance.now(); + for (let i = 0; i < runs; i++) (await client.get(View, { name: label }, counting)).release(); + record(`${label}: hold to live (two hops, mean of ${runs})`, codec, ms((performance.now() - start) / runs)); + await codecRows(`${label}: snapshot`, codec, codec.decode(down.last!), 1_000); + await close(); +} + +/** + * Untimed commits before the timed ones. V8 sizes the young generation from the bytes surviving its collections so + * far; with these, the timed loop runs nearer its steady size (its medians match a run with a pinned young generation), + * so a scavenge's place and cost depend less on the setup (slice 7b fix round 3, `benchmarks.md`). + */ +const WARM_UP_COMMITS = 1_000; + +/** + * One commit per run, after `WARM_UP_COMMITS` untimed ones: commit-to-frame as the mean, median and p90 of the runs + * (the mean carries the scavenges that fall in the loop; the median and p90 do not), bytes and messages per commit, + * encodes per hold; then its message alone. `extra` notes more at the start, which the warm-up's writes take (a + * removal). + */ +async function commits( + label: string, + codec: Codec, + write: (store: Awaited>["store"], i: number) => Promise, + extra = 0, +): Promise { + const { client, store, down, close } = await setup(label, notes(1_000 + extra), codec); + const view = await client.get(View, { name: label }, counting); + const watch = await view.doc.watch(counting); + let frame: () => void = () => {}; + watch.start(async () => frame()); + const runs = 1_000; + /** One commit and its frame; its time in milliseconds. */ + const commit = async (i: number): Promise => { + const arrived = new Promise((resolve) => { + frame = resolve; + }); + const start = performance.now(); + await write(store, i); + await arrived; + return performance.now() - start; + }; + for (let i = 0; i < WARM_UP_COMMITS; i++) await commit(runs + i); + const [messages0, encodes0] = [down.messages, encodes()]; + const times: number[] = []; + for (let i = 0; i < runs; i++) times.push(await commit(i)); + const mean = times.reduce((sum, time) => sum + time, 0) / runs; + times.sort((a, b) => a - b); + const [median, p90] = [times[runs / 2]!, times[Math.floor(runs * 0.9)]!]; + record(`${label}: commit to frame (mean, median, p90)`, codec, `${ms(mean)}, ${ms(median)}, ${ms(p90)}`); + record( + `${label}: messages per commit, encodes per hold`, + codec, + `${(down.messages - messages0) / runs}, ${(encodes() - encodes0) / runs}`, + ); + await codecRows(`${label}: state message`, codec, codec.decode(down.last!), 10_000); + await close(); +} + +/** A streamed reply: 2,000 small text appends to one entry. */ +async function stream(codec: Codec): Promise { + const { client, store, down, close } = await setup("stream", [entry(0, 0)], codec); + const view = await client.get(View, { name: "stream" }, counting); + const watch = await view.doc.watch(counting); + let frames = 0; + const done = Promise.withResolvers(); + watch.start(async () => { + if (++frames === 2_000) done.resolve(); + }); + const bytes0 = down.bytes; + const start = performance.now(); + for (let i = 0; i < 2_000; i++) await store.stream(" tok", counting); + await done.promise; + record("streamed reply, 2,000 appends: time per append", codec, ms((performance.now() - start) / 2_000)); + record( + "streamed reply: bytes per append, total", + codec, + `${((down.bytes - bytes0) / 2_000).toFixed(0)} B, ${kb(down.bytes - bytes0)}`, + ); + await codecRows("streamed token", codec, codec.decode(down.last!), 10_000); + await close(); +} + +/** One trial: the memory retained per hold and the messages of one commit, with `holds` holds on one link. */ +async function holdTrial( + codec: Codec, + holds: number, +): Promise<{ readonly perHold: Memory; readonly messages: number }> { + const name = `holds ${holds}`; + const { client, store, down, close } = await setup(name, notes(), codec, false); + const before = await settledMemory(); + const views = []; + for (let i = 0; i < holds; i++) views.push(await client.get(View, { name }, BACKGROUND_CONTEXT)); + await store.edit({ index: 1, text: "warm" }, BACKGROUND_CONTEXT); + const after = await settledMemory(); + const messages0 = down.messages; + await store.edit({ index: 2, text: "changed" }, BACKGROUND_CONTEXT); + await new Promise((resolve) => setTimeout(resolve, 0)); + const messages = down.messages - messages0; + await close(); + const perHold = { + heap: (after.heap - before.heap) / holds, + arrayBuffers: (after.arrayBuffers - before.arrayBuffers) / holds, + }; + return { perHold, messages }; +} + +const BIG_CALL_BURSTS = 20; + +/** + * Calls: a small argument, and a 1 MiB `bytes()` argument; round trips, then the call and result messages alone. The + * 1 MiB round trip is the median and p90 of `BIG_CALL_BURSTS` bursts of 50 calls, each from a collected heap: the + * arguments fill memory outside the heap, and a full collection (about 0.25 ms per call over a burst) lands in one + * burst and not the next, so a single burst measured the collector, not the call. + */ +async function calls(codec: Codec): Promise { + const { client, down, up, close } = await setup("calls", [], codec); + const files = await client.get(Files, {}, counting); + let start = performance.now(); + for (let i = 0; i < 1_000; i++) await files.ping({ n: i }, counting); + record("call, small argument: round trip", codec, ms((performance.now() - start) / 1_000)); + await codecRows("call, small argument: call message", codec, codec.decode(up.last!), 10_000); + await codecRows("call, small argument: result message", codec, codec.decode(down.last!), 10_000); + const data = new Uint8Array(1024 * 1024); + const bursts: number[] = []; + for (let burst = 0; burst < BIG_CALL_BURSTS; burst++) { + gc(); + start = performance.now(); + for (let i = 0; i < 50; i++) await files.put({ data }, counting); + bursts.push((performance.now() - start) / 50); + } + bursts.sort((a, b) => a - b); + const [median, p90] = [bursts[BIG_CALL_BURSTS / 2]!, bursts[Math.floor(BIG_CALL_BURSTS * 0.9)]!]; + record("call, 1 MiB bytes argument: round trip (median, p90)", codec, `${ms(median)}, ${ms(p90)}`); + await codecRows("call, 1 MiB bytes argument: call message", codec, codec.decode(up.last!), 1_000); + await close(); +} + +/** + * Drops the client's parent link (the server's end closes it) and waits until the client saw the close; then connects + * a new counted link and returns it with the time it was connected. + */ +async function relink(server: Host

, client: Host

, down: Link, codec: Codec) { + down.close(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const [a, b] = memoryLinkPair(); + const next = { down: counted(a), up: counted(b) }; + const start = performance.now(); + server.accept(next.down, { principal: { kind: "owner" }, codec }, BACKGROUND_CONTEXT); + client.connect(next.up, { principal: { kind: "server" }, codec }, BACKGROUND_CONTEXT); + return { ...next, start }; +} + +/** Resolves when `state` is live. */ +async function live(state: NativeWatchable): Promise { + if (state.value.kind === "live") return; + const watch = await state.watch(BACKGROUND_CONTEXT); + const done = Promise.withResolvers(); + watch.start(async (call) => { + if (call.kind === "value" && call.value.kind === "live") done.resolve(); + }); + await done.promise; + await watch.stop(); +} + +/** + * Reconnects with `references` held references of the 1,000-item view (one provider object, a hold each): time from + * `connect` to every reference live again, and the bytes of the re-holds both ways; the mean of `runs` reconnects. + */ +async function reconnects(codec: Codec, references: number, runs: number): Promise { + const name = `reconnect ${references}`; + const { server, client, down, close } = await setup(name, notes(), codec, false); + const views = []; + for (let i = 0; i < references; i++) views.push(await client.get(View, { name }, BACKGROUND_CONTEXT)); + let link = down; + let time = 0; + let bytes = { down: 0, up: 0 }; + for (let run = 0; run < runs; run++) { + const next = await relink(server, client, link, codec); + await Promise.all(views.map((view) => live(view.state))); + time += performance.now() - next.start; + bytes = { down: next.down.bytes, up: next.up.bytes }; + link = next.down; + } + const label = `reconnect, ${references} reference(s) of 1,000 items`; + record(`${label}: time to all live (mean of ${runs})`, codec, ms(time / runs)); + record(`${label}: bytes down / up`, codec, `${b(bytes.down)} / ${b(bytes.up)}`); + await close(); +} + +/** 1,000 calls made while the link is down, flushed on reconnect: time from `connect` to the last result, and order. */ +async function parkedCalls(codec: Codec): Promise { + const { server, client, down, close } = await setup("parked", [], codec, false); + const files = await client.get(Files, {}, BACKGROUND_CONTEXT); + let link = down; + let time = 0; + let ordered = true; + const runs = 10; + for (let run = 0; run < runs; run++) { + pings.length = 0; + const next = await relink(server, client, link, codec); + // The link is connected but not yet live: every call below parks until the re-hold is bound. + const results = await Promise.all(Array.from({ length: 1_000 }, (_, n) => files.ping({ n }, BACKGROUND_CONTEXT))); + time += performance.now() - next.start; + ordered &&= pings.every((n, index) => n === index) && results.every((result, index) => result === index + 1); + link = next.down; + } + record("1,000 calls parked while down, flushed on reconnect: time to the last result", codec, ms(time / runs)); + record("1,000 parked calls: run in the order they were made", codec, ordered ? "yes" : "no"); + await close(); +} + +for (const codec of CODECS) { + // Warm-up: the first run pays for compilation. + await snapshot("warm-up", codec, notes()); + await snapshot("notes list of 1,000 items", codec, notes()); + await snapshot("500 KB document", codec, document()); + await commits("one appended item", codec, (store, i) => store.append(entry(10_000 + i, 80), counting)); + await commits("one changed field", codec, (store, i) => store.edit({ index: 500, text: `edit ${i}` }, counting)); + await commits("one removed item", codec, (store) => store.remove(0, counting), WARM_UP_COMMITS); + await stream(codec); + for (const holds of [1, 10]) { + // The median of seven trials: a single hold is small next to the collector's noise. + const trials: Awaited>[] = []; + for (let i = 0; i < 7; i++) trials.push(await holdTrial(codec, holds)); + const median = (of: (memory: Memory) => number) => + trials.map(({ perHold }) => of(perHold)).sort((a, b) => a - b)[3]!; + const retained = `${kb(median(({ heap }) => heap))} / ${b(median(({ arrayBuffers }) => arrayBuffers))}`; + record(`${holds} hold(s) of 1,000 items: heap / array buffers retained per hold (both ends)`, codec, retained); + record(`${holds} hold(s): messages per commit`, codec, `${trials[0]!.messages}`); + } + await calls(codec); + await reconnects(codec, 1, 20); + await reconnects(codec, 100, 5); + await parkedCalls(codec); +} + +const rows = [...table].filter(([label]) => !label.startsWith("warm-up")); +const width = Math.max(...rows.map(([label]) => label.length)); +console.log(`| ${"Case".padEnd(width)} | binary | text |`); +console.log(`| ${"-".repeat(width)} | --- | --- |`); +for (const [label, [binary, text]] of rows) console.log(`| ${label.padEnd(width)} | ${binary} | ${text} |`); diff --git a/packages/chord/test/next/bench/plugins.ts b/packages/chord/test/next/bench/plugins.ts new file mode 100644 index 000000000..65416b2f0 --- /dev/null +++ b/packages/chord/test/next/bench/plugins.ts @@ -0,0 +1,151 @@ +// Benchmark of the plugins component (slice 7c §7, slice 12): a cold build of a 20-plugin set (one worker facet each, +// its own files, a copied contract, the app's API and kinds checked), a rebuild of one facet after an edit, the +// evaluation of a facet (`vm.Script`), and the heap after 100 reloads of one plugin on a host (old facets collected, +// V8's compilation cache off). Medians over runs. +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/plugins.ts +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { + type BuiltScript, + buildFacet, + createPlugins, + discoverPlugins, + evaluateFacet, +} from "../../../src/next-node/plugins/index.ts"; + +const PLUGINS = 20; +const RUNS = 7; +const RELOADS = 100; + +const gc = (globalThis as { gc?: () => void }).gc!; +// The app's API (its v1 is Chord's surface here): every import is checked by version and kind, as in an app. +const HOST_PACKAGES = { "@bench/app/v1": chordNext }; +const API = { name: "@bench/app", versions: ["v1"] } as const; +const KINDS = ["server", "worker"]; +const NODE = { platform: "node", hostPackages: Object.keys(HOST_PACKAGES), api: API, kinds: KINDS } as const; + +/** A plugin's worker facet: a store over a document, a derive, a method, a helper module and the shared contract. */ +function worker(index: number, label: string): string { + return [ + 'import { constant, defineFacet, defineState, int, local, method, obj, str } from "@bench/app/v1";', + 'import { Shared } from "../contracts/contract.ts";', + 'import { format } from "./format.ts";', + `const Store = local("bench.store${index}", { version: 1, params: {}, members: { label: str(), n: int(), bump: method(undefined, int()) } });`, + 'const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) });', + "export default defineFacet((host) => host.serve(Store, (_, self) => {", + " const doc = self.state(Count);", + ` return { label: constant(format(${JSON.stringify(label)}, Shared.name)), n: self.derive([doc], ([d]) => d.n),`, + " bump: async (ctx) => { await doc.change((d) => { d.n += 1; }, ctx); return doc.value.n; } };", + "}));", + ].join("\n"); +} + +/** Writes the 20-plugin set (and the contracts plugin) under a new root. */ +async function pluginSet(): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-bench-plugins-")); + const write = async (dir: string, files: Record) => { + await mkdir(join(root, dir), { recursive: true }); + for (const [file, text] of Object.entries(files)) await writeFile(join(root, dir, file), text); + }; + await write("contracts", { + "package.json": JSON.stringify({ name: "@bench/contracts", type: "module" }), + "contract.ts": + 'import { local, str } from "@bench/app/v1";\nexport const Shared = local("bench.shared", { version: 1, params: {}, members: { name: str() } });', + }); + for (let index = 0; index < PLUGINS; index++) { + await write(`p${index}`, { + "package.json": JSON.stringify({ name: `@bench/p${index}`, type: "module" }), + "worker.ts": worker(index, "v1"), + "format.ts": + 'export function format(label: string, name: string): string {\n return label + " (" + name + ")";\n}', + }); + } + return root; +} + +/** Median of `values`. */ +function median(values: number[]): number { + return values.sort((a, b) => a - b)[Math.floor(values.length / 2)]!; +} + +const root = await pluginSet(); +const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); +const options = { ...NODE, plugins } as const; +const facets = plugins.filter((plugin) => plugin.facets.worker !== undefined); + +// A cold build of every worker facet (each its own esbuild build, all at once as the component runs them). +const cold: number[] = []; +let scripts: BuiltScript[] = []; +for (let run = 0; run < RUNS; run++) { + const started = performance.now(); + const built = await Promise.all(facets.map((plugin) => buildFacet(plugin, "worker", options, context))); + cold.push(performance.now() - started); + scripts = built.filter((each): each is BuiltScript => each.kind === "script"); +} +const bytes = scripts.reduce((sum, each) => sum + each.source.length, 0); +console.log( + `cold build of ${facets.length} worker facets: ${median(cold).toFixed(1)} ms (median of ${RUNS}), ${(bytes / 1024).toFixed(1)} KiB of scripts`, +); + +// A rebuild of one facet after an edit. +const rebuild: number[] = []; +for (let run = 0; run < RUNS; run++) { + await writeFile(join(root, "p0", "worker.ts"), worker(0, `edit ${run}`)); + const started = performance.now(); + await buildFacet(facets[0]!, "worker", options, context); + rebuild.push(performance.now() - started); +} +console.log(`rebuild of one facet: ${median(rebuild).toFixed(1)} ms (median of ${RUNS})`); + +// The evaluation of a facet: compile and run its script. +const evaluations: number[] = []; +for (let run = 0; run < RUNS; run++) { + const started = performance.now(); + for (let index = 0; index < scripts.length; index++) { + evaluateFacet(scripts[index]!, { hostPackages: HOST_PACKAGES, plugin: facets[index]! }); + } + evaluations.push((performance.now() - started) / scripts.length); +} +console.log(`evaluation per facet: ${median(evaluations).toFixed(2)} ms (median of ${RUNS})`); + +// The heap after 100 reloads of one plugin on a host, against after a warm-up (two builds alternating). +const KIND = { platform: "node", hostPackages: Object.keys(HOST_PACKAGES) } as const; +const component = createPlugins( + { roots: [root], api: API, kinds: { server: KIND, worker: KIND }, onError: () => {} }, + context, +); +const host = createHost({ onError: () => {} }); +await component.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); +await writeFile(join(root, "p0", "worker.ts"), worker(0, "a")); +const first = await component.build(["@bench/p0"], context); +await writeFile(join(root, "p0", "worker.ts"), worker(0, "b")); +const second = await component.build(["@bench/p0"], context); +const settled = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + return process.memoryUsage().heapUsed; +}; +for (let reload = 0; reload < 10; reload++) await component.reload(host, reload % 2 === 0 ? second : first, context); +const warm = await settled(); +const reloadTimes: number[] = []; +for (let reload = 0; reload < RELOADS; reload++) { + const started = performance.now(); + await component.reload(host, reload % 2 === 0 ? second : first, context); + reloadTimes.push(performance.now() - started); +} +const after = await settled(); +console.log( + `reload of one plugin (evaluation and host reload): ${median(reloadTimes).toFixed(2)} ms (median of ${RELOADS})`, +); +console.log(`heap after ${RELOADS} reloads: ${((after - warm) / 1024).toFixed(0)} KiB against after the warm-up`); +await host.close(context); +component.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/bench/registries.ts b/packages/chord/test/next/bench/registries.ts new file mode 100644 index 000000000..5eb75504b --- /dev/null +++ b/packages/chord/test/next/bench/registries.ts @@ -0,0 +1,108 @@ +// Benchmark of slice 8's registries and writables: a `writable.set` delivered to one watch, a set of the same value +// (one comparison), and one contribution of N (10, 100) changed while the registry is watched (one table build and one +// compose per set, over N entries). Median µs per operation over bursts, heap allocated per operation, and the +// registry's compose count per set (the cost guard: one). +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/registries.ts +import { performance } from "node:perf_hooks"; +import { GCProfiler } from "node:v8"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; + +const BURSTS = 9; + +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Median µs per operation over the bursts (after one warm-up burst), and heap bytes allocated per operation. */ +async function measure(runs: number, operation: (i: number) => void): Promise { + const times: number[] = []; + let bytes = 0; + let counted = 0; + for (let burst = -1; burst < BURSTS; burst++) { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + const started = performance.now(); + for (let i = 0; i < runs; i++) operation(i); + // The watches' listeners run on microtasks: their work counts too. + await new Promise((resolve) => setImmediate(resolve)); + const elapsed = performance.now() - started; + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + if (burst < 0) continue; + times.push((elapsed * 1000) / runs); + bytes += after - before + freed; + counted += runs; + } + times.sort((a, b) => a - b); + return `${times[Math.floor(times.length / 2)]!.toFixed(3)} µs, ${Math.round(bytes / counted)} B allocated`; +} + +const Sum = defineRegistry("bench.sum", { version: 1 }); + +/** A host with `contributors` host-lifetime objects each contributing a writable number to `Sum`, watched. */ +async function registryOf(contributors: number) { + const telemetry = new InMemoryTelemetryContext(); + const writables: Writable[] = []; + const types = Array.from({ length: contributors }, (_, i) => + local(`bench.contributor-${i}`, { version: 1, params: {}, members: { n: int() } }), + ); + const host = createHost({ telemetry, onError: (error) => console.log("onError", error.kind) }); + const facet = defineFacet((h) => { + h.serve(Sum, { compose: (table) => table.reduce((sum, { value }) => sum + value, 0) }); + for (const Type of types) { + h.serve(Type, (_, self) => { + const value = self.writable(0); + writables.push(value); + self.contribute(Sum, value); + return { n: self.derive([value], ([n]) => n) }; + }); + } + }); + await host.load([{ name: "@bench/registries", facet }], context); + const sum = await host.registry(Sum, context); + const watch = await sum.watch(context); + watch.start(async () => {}); + const composes = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.registry.composes") + .reduce((n, { value }) => n + value, 0); + return { host, writables, watch, composes }; +} + +const one = await registryOf(1); +const writable = one.writables[0]!; +const watch = await writable.watch(context); +watch.start(async () => {}); +console.log("writable.set, one watch:", await measure(20_000, (i) => writable.set(i + 1))); +// The listeners' backlogs dropped first: what follows measures the set alone. +await watch.stop(); +await one.watch.stop(); +writable.set(-1); +console.log("writable.set of the same value:", await measure(1_000_000, () => writable.set(-1))); +console.log("an empty operation (the loop's own cost):", await measure(1_000_000, () => {})); +await one.host.close(context); + +for (const contributors of [10, 100]) { + const { host, writables, watch, composes } = await registryOf(contributors); + const before = composes(); + const target = writables[Math.floor(contributors / 2)]!; + const sets = 5_000; + const result = await measure(sets, (i) => target.set(i + 1)); + console.log( + `one of ${contributors} contributions set, registry watched:`, + result, + `${((composes() - before) / (sets * (BURSTS + 1))).toFixed(2)} composes per set`, + ); + await watch.stop(); + await host.close(context); +} diff --git a/packages/chord/test/next/bench/relay.ts b/packages/chord/test/next/bench/relay.ts new file mode 100644 index 000000000..c699e9282 --- /dev/null +++ b/packages/chord/test/next/bench/relay.ts @@ -0,0 +1,619 @@ +// Light benchmarks of links and relays (slice 4a): commit-to-frame and wire bytes through a relay (tab ↔ server ↔ +// worker) over memory links, a real IPC link to a forked worker and a loopback WebSocket to the tab; hold-to-live +// through a relay including the `resolve` of a worker; a tab throttled to 64 KiB/s with stalls of 1, 10 and 60 s +// receiving a streamed reply into a 500 KB view, directly and through a relay (the worker folds either way: its window +// is per origin), in virtual time; 10,000 calls from one tab with another tab's latency through the relay. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc --no-flush-bytecode test/next/bench/relay.ts +// (`--no-flush-bytecode`: V8 otherwise drops the bytecode of functions idle during a stall, which reads as freed heap.) +// The same file is the forked worker when its first argument is `worker`. +import { type ChildProcess, fork } from "node:child_process"; +import type { AddressInfo } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type IpcPort, ipcLink, type Link, memoryLinkPair, webSocketLink } from "../../../src/next/link/index.ts"; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); +const Shape = obj({ entries: arr(Entry) }); + +type Entry = { id: number; text: string; author: string; at: number }; + +/** A worker's own store: one document of entries, and the write the benchmark makes (a streamed token). */ +const Store = local("bench.store", { + version: 1, + params: {}, + members: { doc: Shape, stream: method(str()) }, +}); + +/** What a tab holds: its session's store as it is, placed on the session's worker, and a small call. */ +const View = remote("bench.view", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { owner: { state: { doc: Shape }, methods: { stream: method(str()), ping: method(int(), int()) } } }, +}); + +/** The start data a worker gets in its start message: its session and the payload its store begins with. */ +type Start = { readonly id: string; readonly entries: "notes" | "document" }; + +function entry(id: number, length: number): Entry { + return { id, text: "x".repeat(length), author: "mario", at: 1_700_000_000_000 + id }; +} + +/** The notes list (1,000 entries of 80 characters, 134 KiB) or a 500 KB conversation document (500 of 960). */ +function entriesOf(kind: Start["entries"]): Entry[] { + return kind === "notes" + ? Array.from({ length: 1_000 }, (_, id) => entry(id, 80)) + : Array.from({ length: 500 }, (_, id) => entry(id, 960)); +} + +/** A worker's facet: its store starts with the start message's entries (the §18 core facet pattern). */ +function workerFacet(start: Start) { + const Doc = defineState("doc", { + schema: Shape, + version: 1, + initial: () => ({ entries: entriesOf(start.entries) }), + }); + return defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Doc); + return { + doc, + stream: (token, ctx) => + doc + .change((draft) => { + draft.entries[draft.entries.length - 1]!.text += token; + }, ctx) + .then(() => null), + }; + }); + host.serve(View, (_params, _principal, self) => { + const store = self.get(Store, {}); + return { + variant: "owner", + members: { doc: store.doc, stream: (token, ctx) => store.stream(token, ctx), ping: async (n) => n + 1 }, + }; + }); + }); +} + +/** A worker host for `start`, not loaded yet. */ +function workerHost(start: Start): Host

{ + return createHost

({ + address: Session({ id: start.id }), + principal: { kind: "worker", session: start.id }, + onError: console.error, + }); +} + +/** A worker on `link`, its parent link: connect first, then load (W6). */ +function startWorker(start: Start, link: Link): Host

{ + const worker = workerHost(start); + worker.connect(link, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@bench/worker", facet: workerFacet(start) }], context); + return worker; +} + +/** A link end counting the bytes it sends and receives. */ +type Counted = Link & { readonly sent: number; readonly received: number }; + +function counted(link: Link): Counted { + let sent = 0; + let received = 0; + return { + get sent() { + return sent; + }, + get received() { + return received; + }, + send(message) { + sent += message.byteLength; + link.send(message); + }, + onMessage: (listener) => + link.onMessage((message) => { + received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** A link end timing how long its host handles each received message. */ +type Timed = Link & { longest: number }; + +function timed(link: Link): Timed { + const timing: Timed = { + longest: 0, + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + const start = performance.now(); + listener(message); + timing.longest = Math.max(timing.longest, performance.now() - start); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; + return timing; +} + +/** How a server reaches its workers: in this process, or forked over IPC. */ +type WorkerKind = "memory" | "ipc"; + +/** The children forked in this run, ended with it. */ +const children: ChildProcess[] = []; + +/** A server routing `Session` to workers of `kind` whose stores start with `entries`. */ +async function server(kind: WorkerKind, entries: Start["entries"]) { + const host = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); + /** Each worker link's server end, counted; the in-process workers and their ends, timed. */ + const workerLinks: Counted[] = []; + const workers: { readonly host: Host

; readonly link: Timed }[] = []; + host.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const start = { id: address.params.id, entries }; + const principal = { kind: "worker", session: start.id } as const; + if (kind === "memory") { + const [a, b] = memoryLinkPair(); + const end = timed(b); + workers.push({ host: startWorker(start, end), link: end }); + const link = counted(a); + workerLinks.push(link); + return { link, principal, trusted: true }; + } + const child = fork(fileURLToPath(import.meta.url), ["worker"], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + children.push(child); + const link = counted(ipcLink(child)); + // The app's own start message; the link skips it. + child.send(start); + workerLinks.push(link); + return { link, principal, trusted: true }; + }, + }, + context, + ); + await host.load([], context); + return { host, workerLinks, workers }; +} + +/** A tab: a client host connected to `link`. */ +async function tab(link: Link): Promise> { + const host = createHost

({ onError: console.error }); + host.connect(link, { principal: { kind: "server" } }, context); + await host.load([], context); + return host; +} + +/** A tab on `on` over a memory link, the server's end counted. */ +async function memoryTab(on: Host

): Promise<{ readonly tab: Host

; readonly link: Counted }> { + const [a, b] = memoryLinkPair(); + const link = counted(a); + on.accept(link, { principal: { kind: "owner" } }, context); + return { tab: await tab(b), link }; +} + +/** A tab on `on` over a loopback WebSocket (`ws` serving, Node's own `WebSocket` in the tab), the server's end counted. */ +async function webSocketTab(on: Host

) { + const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await new Promise((resolve) => wss.once("listening", resolve)); + const accepted = Promise.withResolvers(); + wss.on("connection", (socket) => { + const link = counted(webSocketLink(socket)); + on.accept(link, { principal: { kind: "owner" } }, context); + accepted.resolve(link); + }); + const { port } = wss.address() as AddressInfo; + const host = await tab(webSocketLink(new WebSocket(`ws://127.0.0.1:${port}`))); + const close = () => new Promise((resolve) => wss.close(resolve)); + return { tab: host, link: await accepted.promise, close }; +} + +/** A worker with `entries` served directly to a tab over one memory link (no relay), the worker's end counted. */ +async function direct(entries: Start["entries"], wrap: (link: Link) => Link = (link) => link) { + const worker = workerHost({ id: "D", entries }); + await worker.load([{ name: "@bench/worker", facet: workerFacet({ id: "D", entries }) }], context); + const [a, b] = memoryLinkPair(); + const link = counted(a); + const end = timed(link); + worker.accept(end, { principal: { kind: "owner" } }, context); + return { worker, link, end, tab: await tab(wrap(b)) }; +} + +const ms = (value: number) => `${value.toFixed(3)} ms`; +const kb = (value: number) => `${(value / 1024).toFixed(1)} KiB`; +const b = (value: number) => (value >= 10_240 ? kb(value) : `${value.toFixed(0)} B`); +const s = (value: number) => `${value.toFixed(2)} s`; +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Prints a markdown table. */ +function print(title: string, head: readonly string[], rows: readonly (readonly string[])[]): void { + console.log(`\n${title}\n`); + console.log(`| ${head.join(" | ")} |`); + console.log(`| ${head.map(() => "---").join(" | ")} |`); + for (const row of rows) console.log(`| ${row.join(" | ")} |`); +} + +/** + * Commits as a tab sees them: the tab calls `stream` on its view 1,000 times, each one commit on the worker; the time + * from the call to the tab's watch frame (the call's hops down, the frame's up), and the bytes per commit on the tab + * link (sent to the tab) and the worker link (received from the worker). + */ +async function commits(holder: Host

, session: string, tabLink: Counted, workerLinks: readonly Counted[]) { + const view = await holder.get(View, { session }, context); + // Made by the route's `resolve` for the first hold (none without a relay). + const workerLink = workerLinks[0]; + const watch = await view.doc.watch(context); + let frame: () => void = () => {}; + watch.start(async () => frame()); + // Warm-up. + for (let i = 0; i < 100; i++) await view.stream(" w", context); + const runs = 1_000; + const tab0 = tabLink.sent; + const worker0 = workerLink?.received ?? 0; + let total = 0; + for (let i = 0; i < runs; i++) { + const arrived = new Promise((resolve) => { + frame = resolve; + }); + const start = performance.now(); + await Promise.all([view.stream(" tok", context), arrived]); + total += performance.now() - start; + } + await watch.stop(); + view.release(); + const workerBytes = workerLink === undefined ? "-" : b((workerLink.received - worker0) / runs); + return [ms(total / runs), `${b((tabLink.sent - tab0) / runs)} / ${workerBytes}`]; +} + +/** Commit-to-frame through a relay over each transport, and directly (one memory link, no relay) as the baseline. */ +async function transports(): Promise { + const rows: string[][] = []; + { + const { host, workerLinks } = await server("memory", "notes"); + const { tab: t, link } = await memoryTab(host); + rows.push(["memory tab, in-process worker (memory)", ...(await commits(t, "C", link, workerLinks))]); + await t.close(context); + await host.close(context); + } + { + const { host, workerLinks } = await server("ipc", "notes"); + const { tab: t, link } = await memoryTab(host); + const start = performance.now(); + (await t.get(View, { session: "C" }, context)).release(); + const first = performance.now() - start; + rows.push(["memory tab, forked worker (IPC)", ...(await commits(t, "C", link, workerLinks))]); + rows.push([" first hold to live, the fork included (one sample)", ms(first), "-"]); + await t.close(context); + await host.close(context); + } + { + const { host, workerLinks } = await server("ipc", "notes"); + const { tab: t, link, close } = await webSocketTab(host); + rows.push(["loopback WebSocket tab, forked worker (IPC)", ...(await commits(t, "C", link, workerLinks))]); + await t.close(context); + await host.close(context); + await close(); + } + { + const { worker, link, tab: t } = await direct("notes"); + rows.push(["no relay: memory tab on the worker (baseline)", ...(await commits(t, "D", link, []))]); + await t.close(context); + await worker.close(context); + } + print( + "Commits through a relay (1,000 streamed tokens into the 1,000-item notes list view)", + ["Topology", "call to frame", "bytes per commit: to the tab / from the worker"], + rows, + ); +} + +/** + * Hold to live through a relay: a first hold of each of 20 sessions (the route's `resolve` starts an in-process worker, + * which loads and activates the view: one hop more than a later hold), then 100 holds of a resolved session, each + * released; and the same 100 holds directly on a worker. + */ +async function holds(): Promise { + const rows: string[][] = []; + const { host } = await server("memory", "notes"); + const { tab: t } = await memoryTab(host); + let start = performance.now(); + for (let i = 0; i < 20; i++) await t.get(View, { session: `H${i}` }, context); + rows.push([ + "first hold, `resolve` of an in-process worker included (mean of 20)", + ms((performance.now() - start) / 20), + ]); + start = performance.now(); + for (let i = 0; i < 100; i++) (await t.get(View, { session: "H0" }, context)).release(); + rows.push([ + "hold of a resolved session through the relay (four hops, mean of 100)", + ms((performance.now() - start) / 100), + ]); + await t.close(context); + await host.close(context); + const { worker, tab: d } = await direct("notes"); + await d.get(View, { session: "D" }, context); + start = performance.now(); + for (let i = 0; i < 100; i++) (await d.get(View, { session: "D" }, context)).release(); + rows.push(["no relay: hold on the worker (two hops, mean of 100, baseline)", ms((performance.now() - start) / 100)]); + await d.close(context); + await worker.close(context); + print("Hold to live, the 1,000-item notes list view (133.7 KiB snapshot)", ["Case", "time"], rows); +} + +/** + * A tab's end of a link that delivers what arrives at `rate` bytes per virtual second, whole messages in order, and + * nothing while stalled; virtual time advances by `step`. Idle time banks no budget. + */ +class Throttled implements Link { + stalled = false; + delivered = 0; + readonly #link: Link; + readonly #rate: number; + readonly #queue: Uint8Array[] = []; + #budget = 0; + #listener: (message: Uint8Array) => void = () => {}; + + constructor(link: Link, rate: number) { + this.#link = link; + this.#rate = rate; + link.onMessage((message) => this.#queue.push(message)); + } + + send(message: Uint8Array): void { + this.#link.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#listener = listener; + } + + close(): void { + this.#link.close(); + } + + onClose(listener: () => void): void { + this.#link.onClose(listener); + } + + /** `milliseconds` of virtual time pass. */ + step(milliseconds: number): void { + if (this.stalled) return; + const budget = (this.#rate * milliseconds) / 1000; + this.#budget += budget; + while (this.#queue.length > 0 && this.#queue[0]!.byteLength <= this.#budget) { + const message = this.#queue.shift()!; + this.#budget -= message.byteLength; + this.delivered += message.byteLength; + this.#listener(message); + } + if (this.#queue.length === 0) this.#budget = Math.min(this.#budget, budget); + } +} + +/** The heap in use once queued work ran and garbage was collected, repeatedly. */ +async function settledHeap(): Promise { + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + return process.memoryUsage().heapUsed; +} + +/** Lets every host handle what the last step delivered (memory links deliver in microtasks). */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +const STEP = 10; + +/** Tokens per virtual second: a fast model's streamed reply. */ +const TOKENS = 100; + +/** + * A pause of the throttled tab: a stall (it receives nothing and so acknowledges nothing), or the same time without one + * (the control: what the streaming alone costs). + */ +type Pause = { readonly kind: "stall" | "control"; readonly seconds: number }; + +/** + * A tab throttled to 64 KiB/s holding the 500 KB document view of a worker that streams a reply at 100 tokens/s, in + * virtual time (10 ms steps): 2 s of streaming, the pause, 10 s more of streaming, then until the tab caught up. + * `relayed`: through a server (it forwards at once, the worker folds the tab origin's ops); `direct`: on the worker (it + * folds the ops in its pending queues). + */ +async function throttled(topology: "direct" | "relayed", pause: Pause): Promise { + const relayed = topology === "relayed"; + let throttle: Throttled | undefined; + const throttling = (link: Link) => { + throttle = new Throttled(link, 64 * 1024); + return throttle; + }; + let t: Host

; + let producer: () => { readonly host: Host

; readonly end: Timed }; + let close: () => Promise; + if (relayed) { + const { host, workers } = await server("memory", "document"); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" } }, context); + t = await tab(throttling(b)); + producer = () => ({ host: workers[0]!.host, end: workers[0]!.link }); + close = async () => { + await t.close(context); + await host.close(context); + }; + } else { + const made = await direct("document", throttling); + t = made.tab; + producer = () => ({ host: made.worker, end: made.end }); + close = async () => { + await t.close(context); + await made.worker.close(context); + }; + } + // The first snapshot crosses the throttle in virtual time too. + const getting = t.get(View, { session: relayed ? "T" : "D" }, context); + let bound = false; + void getting.then(() => { + bound = true; + }); + while (!bound) { + throttle!.step(STEP); + await settle(); + } + const view = await getting; + const { host: worker, end: providerEnd } = producer(); + const store = await worker.get(Store, {}, context); + const last = (value: { readonly entries: readonly Entry[] }) => value.entries.at(-1)!.text.length; + const caughtUp = () => last(view.doc.value) === last(store.doc.value); + let clock = 0; + const commitTimes = { outside: 0, outsideCount: 0, during: 0, duringCount: 0 }; + /** One virtual step: a token when streaming (its commit timed), delivery, the hosts' turn. */ + const step = async (streaming: boolean, phase: "outside" | "during") => { + clock += STEP; + if (streaming && clock % (1000 / TOKENS) === 0) { + const start = performance.now(); + await store.stream(" tok", context); + commitTimes[phase] += performance.now() - start; + commitTimes[phase === "outside" ? "outsideCount" : "duringCount"]++; + } + throttle!.step(STEP); + await settle(); + }; + for (let i = 0; i < 2000 / STEP; i++) await step(true, "outside"); + while (!caughtUp()) await step(false, "outside"); + const heap0 = await settledHeap(); + throttle!.stalled = pause.kind === "stall"; + for (let i = 0; i < (pause.seconds * 1000) / STEP; i++) await step(true, "during"); + const heap = (await settledHeap()) - heap0; + throttle!.stalled = false; + const stallEnd = clock; + const delivered0 = throttle!.delivered; + providerEnd.longest = 0; + let recovered: number | undefined; + for (let i = 0; recovered === undefined && i < 120_000 / STEP; i++) { + await step(clock - stallEnd < 10_000, "outside"); + if (caughtUp()) recovered = clock - stallEnd; + } + const row = [ + pause.kind === "stall" + ? `${topology}, ${pause.seconds} s stall` + : `${topology}, ${pause.seconds} s without a stall (control)`, + recovered === undefined ? "> 120 s" : s(recovered / 1000), + b(throttle!.delivered - delivered0), + kb(heap), + `${ms(commitTimes.outside / commitTimes.outsideCount)} / ${ms(commitTimes.during / commitTimes.duringCount)}`, + ms(providerEnd.longest), + ]; + store.release(); + view.release(); + await close(); + return row; +} + +async function slowTabs(): Promise { + const rows: string[][] = []; + for (const topology of ["direct", "relayed"] as const) { + for (const seconds of [1, 10, 60]) rows.push(await throttled(topology, { kind: "stall", seconds })); + rows.push(await throttled(topology, { kind: "control", seconds: 60 })); + } + print( + "A tab throttled to 64 KiB/s, a reply streamed at 100 tokens/s into the 500 KB view (virtual time)", + [ + "Case", + "caught up after the stall", + "bytes to the tab until then", + "heap growth during the stall", + "commit at the worker, outside / during the pause", + "longest message handling at the provider after the stall", + ], + rows, + ); +} + +/** 10,000 calls from tab A through the relay at once, and tab B's sequential calls meanwhile; B alone as the baseline. */ +async function calls(): Promise { + const { host } = await server("memory", "notes"); + const a = await memoryTab(host); + const bTab = await memoryTab(host); + const viewA = await a.tab.get(View, { session: "K" }, context); + const viewB = await bTab.tab.get(View, { session: "K" }, context); + const latencies = async (until: () => boolean, max: number) => { + const times: number[] = []; + while (!until() && times.length < max) { + const start = performance.now(); + await viewB.ping(times.length, context); + times.push(performance.now() - start); + } + return times.sort((x, y) => x - y); + }; + const percentile = (times: readonly number[], p: number) => + times[Math.min(times.length - 1, Math.floor(times.length * p))]!; + for (let i = 0; i < 1_000; i++) await viewB.ping(i, context); + const alone = await latencies(() => false, 1_000); + let done = false; + const start = performance.now(); + const burst = Promise.all(Array.from({ length: 10_000 }, (_, i) => viewA.ping(i, context))).then(() => { + done = true; + }); + const during = await latencies(() => done, 100_000); + await burst; + const total = performance.now() - start; + print( + "Calls through the relay (in-process worker, memory links, N = 256 per link)", + ["Case", "p50", "p95", "max", "calls"], + [ + ["tab B alone", ms(percentile(alone, 0.5)), ms(percentile(alone, 0.95)), ms(alone.at(-1)!), `${alone.length}`], + [ + "tab B while tab A has 10,000 calls in flight or queued", + ms(percentile(during, 0.5)), + ms(percentile(during, 0.95)), + ms(during.at(-1)!), + `${during.length}`, + ], + ["tab A: 10,000 calls at once, until the last result", ms(total), "-", "-", "10000"], + ], + ); + await a.tab.close(context); + await bTab.tab.close(context); + await host.close(context); +} + +// The forked worker: its link first (it queues what arrives before the start message), then the start message. +if (process.argv[2] === "worker") { + // A forked child's `process` is an IPC port. + const link = ipcLink(process as unknown as IpcPort); + const start = await new Promise((resolve) => { + const listener = (message: unknown) => { + if (message instanceof Uint8Array) return; + process.off("message", listener); + // The app's own message, sent by `resolve` above. + resolve(message as Start); + }; + process.on("message", listener); + }); + startWorker(start, link); + link.onClose(() => process.exit(0)); +} else { + await transports(); + await holds(); + await slowTabs(); + await calls(); + for (const child of children) child.kill(); +} diff --git a/packages/chord/test/next/bench/reload.ts b/packages/chord/test/next/bench/reload.ts new file mode 100644 index 000000000..cb9e73f6e --- /dev/null +++ b/packages/chord/test/next/bench/reload.ts @@ -0,0 +1,118 @@ +// Benchmark of `host.reload` (slice 7b §11): one plugin of 20 reloaded on a host with 1,000 live objects, as a pure +// swap and as a native restart. Per case: the reload's time until it resolves (staging, commit, every fate started and +// every swap checked), the time until every object of the plugin is live again, and the heap retained after 20 +// reloads against after the first (old attempts and facets collected). Medians over the reloads after a warm-up. +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/reload.ts +import { performance } from "node:perf_hooks"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Facet, type Host, type Reference } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const PLUGINS = 20; +const OBJECTS = 1_000; +const RELOADS = 20; + +const gc = (globalThis as { gc?: () => void }).gc!; + +const Counter = defineState("bench.counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +/** The plugins' types: one per plugin, objects with params (each lives while referenced). */ +const types = Array.from({ length: PLUGINS }, (_, index) => + local(`bench.item${index}`, { + version: 1, + params: { id: int() }, + members: { n: int(), label: int(), bump: method(undefined, int()) }, + }), +); + +/** Plugin `index`'s facet at code version `code`; with `native`, each object opens a native. */ +function facet(index: number, code: number, native: boolean): Facet { + return defineFacet((h) => + h.serve(types[index]!, (_, self) => { + const counter = self.state(Counter); + if (native) + self.open( + async () => code, + () => {}, + ); + return { + n: self.derive([counter], ([value]) => value.n), + label: constant(code), + bump: async (ctx: Context) => { + await counter.change((draft) => { + draft.n++; + }, ctx); + return counter.value.n; + }, + }; + }), + ); +} + +/** Whole-heap use after full collections. */ +async function heap(): Promise { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + return process.memoryUsage().heapUsed; +} + +/** The median of `values`. */ +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +/** + * One case: `perPlugin(index)` objects per plugin (1,000 in all), plugin 0 reloaded `RELOADS` times; `native` makes + * plugin 0 a native object (restarts) instead of a pure one (swaps). + */ +async function run(name: string, counts: readonly number[], native: boolean): Promise { + const host: Host = createHost({ onError: (error) => console.log("onError", error.kind) }); + const facets = Array.from({ length: PLUGINS }, (_, index) => facet(index, 0, native && index === 0)); + await host.load( + facets.map((each, index) => ({ name: `@bench/p${index}`, facet: each as Facet as never })), + context, + ); + const refs: Reference<(typeof types)[number]>[] = []; + for (const [index, count] of counts.entries()) { + for (let id = 0; id < count; id++) refs.push(await host.get(types[index]!, { id }, context)); + } + const reloaded = refs.slice(0, counts[0]); + const reloads: number[] = []; + const live: number[] = []; + let afterFirst = 0; + for (let round = 1; round <= RELOADS; round++) { + facets[0] = facet(0, round, native); + const set = facets.map((each, index) => ({ name: `@bench/p${index}`, facet: each as never })); + const started = performance.now(); + await host.reload(set, context); + const resolved = performance.now(); + while (reloaded.some((ref) => ref.state.value.kind !== "live" || ref.label.value !== round)) { + await new Promise((resolve) => setImmediate(resolve)); + } + const allLive = performance.now(); + if (round > 2) { + reloads.push(resolved - started); + live.push(allLive - started); + } + if (round === 1) afterFirst = await heap(); + } + const afterAll = await heap(); + console.log( + `${name}: reload resolves in ${median(reloads).toFixed(2)} ms, all ${counts[0]} live again in ${median(live).toFixed(2)} ms; ` + + `heap after ${RELOADS} reloads ${((afterAll - afterFirst) / 1024).toFixed(0)} KiB above after the first`, + ); + await host.close(context); +} + +const spread = Array.from({ length: PLUGINS }, () => OBJECTS / PLUGINS); +const concentrated = [OBJECTS, ...Array.from({ length: PLUGINS - 1 }, () => 0)]; +await run("swap, 50 of 1,000 objects in the reloaded plugin", spread, false); +await run("swap, all 1,000 objects in the reloaded plugin", concentrated, false); +await run("restart, 50 of 1,000 objects in the reloaded plugin", spread, true); +await run("restart, all 1,000 objects in the reloaded plugin", concentrated, true); diff --git a/packages/chord/test/next/bench/store.ts b/packages/chord/test/next/bench/store.ts new file mode 100644 index 000000000..9259e5ec8 --- /dev/null +++ b/packages/chord/test/next/bench/store.ts @@ -0,0 +1,451 @@ +// Benchmarks of the store (slice 6a §14, decision "The commit-step benchmark"): the commit step against Durable's +// `SqliteStorage` on the same SQLite file settings, (a) one document delta per commit, (b) one table put per commit, +// (c) 1,000 whole transactions back to back (a host's `change` against a Durable Session's `commit`), with and without +// a watched document, and (d) transactions of one in-memory document only (no store); as bursts (median and p95 per +// commit over several runs) and the heap after GC; the outbox's extra cost apart. Then (e) the other costs: queue wait +// against step time, query pages, a 10k-row `updateWhere`'s step, long text per row, and migration chunks (the first +// and the late ones). +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/store.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { defineDoc } from "../../../../durable/src/documents.ts"; +import { createSession } from "../../../../durable/src/session/session.ts"; +import { SqliteStorage } from "../../../../durable/src/storage/sqlite/storage.ts"; +import { commitWrites } from "../../../../durable/src/testing/commit-writes.ts"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument } from "../../../src/next/store/engine.ts"; +import { lt } from "../../../src/next/store/expression.ts"; +import { query } from "../../../src/next/store/query.ts"; +import { TableSql, tableSqlName } from "../../../src/next/store/sql.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { stagePut, stageTable } from "../../../src/next/store/table-writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const COMMITS = 1_000; +const RUNS = 7; + +/** One commit as a benchmark makes it, given its index. */ +type Commit = (index: number) => Promise; + +/** Per-commit times (ms) of `RUNS` bursts of `COMMITS` commits each, after one warm-up burst. */ +async function bursts(setup: () => Promise<{ readonly commit: Commit; readonly close: () => Promise }>) { + const times: number[] = []; + const perRun: number[] = []; + for (let run = -1; run < RUNS; run++) { + const { commit, close } = await setup(); + const started = performance.now(); + for (let index = 0; index < COMMITS; index++) { + const before = performance.now(); + await commit(index); + if (run >= 0) times.push(performance.now() - before); + } + if (run >= 0) perRun.push((performance.now() - started) / COMMITS); + await close(); + } + times.sort((a, b) => a - b); + perRun.sort((a, b) => a - b); + return { + median: times[Math.floor(times.length / 2)]!, + p95: times[Math.floor(times.length * 0.95)]!, + burst: perRun[Math.floor(perRun.length / 2)]!, + }; +} + +/** A fresh SQLite file in its own directory. */ +async function file(): Promise<{ readonly path: string; readonly remove: () => Promise }> { + const directory = await mkdtemp(join(tmpdir(), "chord-store-bench-")); + return { path: join(directory, "bench.sqlite"), remove: () => rm(directory, { recursive: true, force: true }) }; +} + +const VALUE = { title: "a note", body: "x".repeat(200), n: 0 }; + +/** Chord's store: one document created, then one delta per commit. */ +function chordDeltas(outbox: boolean) { + return async () => { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { + outbox, + })) as ChordStore; + const address = { namespace: "chord", owner: "bench.store", kind: "note", key: "" }; + const base = { version: 1, fingerprint: "0123456789abcdef", value: VALUE }; + let id = 0; + const fence = { + admit: () => true, + adopt: (_seq: number, ids: readonly number[]) => { + id = ids[0] ?? id; + }, + }; + await store.commit([createDocument(address, base, JSON.stringify(VALUE))], fence, context); + return { + commit: (index: number) => { + const ops = [["s", ["n"], index + 1]] as const; + return store.commit( + [changeDocument(id, index + 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + fence, + context, + ); + }, + close: async () => { + await store.close(context); + await remove(); + }, + }; + }; +} + +/** Durable's `SqliteStorage`: one session document created, then one delta per commit. */ +async function durableDeltas() { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + const id = await storage.mintId(); + const record = { id, kind: "note", scope: { kind: "session" } } as never; + await commitWrites( + storage, + [{ type: "document.create", record, content: { kind: "base", version: 1, value: VALUE } }], + context, + ); + return { + commit: (index: number) => + commitWrites( + storage, + [ + { type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], index + 1]] } }, + ] as never, + context, + ), + close: async () => { + await storage.close(context); + await remove(); + }, + }; +} + +const EDIT_COLUMNS = { id: str(), path: str(), at: int(), body: str() }; +const Edits = defineTable("edits", { version: 1, columns: EDIT_COLUMNS, key: ["id"] }); +/** `Edits` with an index, for pages in a non-key order. */ +const IndexedEdits = defineTable("edits", { + version: 1, + columns: EDIT_COLUMNS, + key: ["id"], + indexes: { by_at: ["at"] }, +}); + +/** Chord's store: one table created, then one `put` per commit (the row checked and encoded in the commit). */ +function chordPuts(outbox: boolean) { + return async () => { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { + outbox, + })) as ChordStore; + const table = tableSqlName("bench.store", Edits.name); + const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); + const fence = { admit: () => true, adopt: () => {} }; + const { version, record: columns, key, fingerprint } = Edits; + await store.commit( + [stageTable({ kind: "table", table, version, state: "ready", columns, key, fingerprint })], + fence, + context, + ); + return { + commit: (index: number) => { + const row = Edits.rows.put( + { id: `e${index}`, path: `src/file${index % 50}.ts`, at: index, body: VALUE.body }, + version, + ); + return store.commit([stagePut(sql.put, table, row, version)], fence, context); + }, + close: async () => { + await store.close(context); + await remove(); + }, + }; + }; +} + +/** Durable's `SqliteStorage`: one conversation created, then one entry appended per commit. */ +async function durableEntries() { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + await commitWrites(storage, [{ type: "conversation", value: { id: 1 } }] as never, context); + return { + commit: async (index: number) => { + const id = await storage.mintId(); + const data = { path: `src/file${index % 50}.ts`, at: index, body: VALUE.body }; + return commitWrites( + storage, + [{ type: "entry", value: { id, conversationId: 1, kind: "edit", data } }] as never, + context, + ); + }, + close: async () => { + await storage.close(context); + await remove(); + }, + }; +} + +/** The heap after GC, in KiB. */ +const NoteSchema = obj({ title: str(), body: str(), n: int() }); + +const InMemoryNote = defineState("bench.note", { schema: NoteSchema, version: 1, initial: () => ({ ...VALUE }) }); + +const PersistedNote = defineState("bench.note", { + schema: NoteSchema, + version: 1, + initial: () => ({ ...VALUE }), + persist: true, + checkpointWhen: () => false, +}); + +const Counter = local("bench.counter", { version: 1, params: {}, members: { touch: method() } }); + +/** Whole Chord transactions on a host: one `change` of one document per commit, persisted or in memory, watched or not. */ +function chordTransactions(options: { + readonly persist: boolean; + readonly watched: boolean; + readonly outbox: boolean; +}) { + return async () => { + const { path, remove } = await file(); + const store = options.persist + ? await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: options.outbox }) + : undefined; + let note: SingletonHandle | undefined; + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Counter, (_, self) => { + note = self.state(options.persist ? PersistedNote : InMemoryNote); + return { touch: async () => null }; + }), + ); + await host.load([{ name: "@bench/store", facet }], context); + const reference = await host.get(Counter, {}, context); + await note!.change((draft) => { + draft.n = -1; + }, context); + const watch = options.watched ? await note!.watch(context) : undefined; + watch?.start(async () => {}); + return { + commit: (index: number) => + note!.change((draft) => { + draft.n = index + 1; + }, context), + close: async () => { + await watch?.stop(); + reference.release(); + await host.close(context); + await store?.close(context); + await remove(); + }, + }; + }; +} + +const DurableNote = defineDoc({ + kind: "bench.note", + version: 1, + scope: "session", + initial: () => ({ ...VALUE }), + checkpointWhen: () => false, +}); + +/** Whole Durable transactions on a Session over `SqliteStorage`: one document change per commit, watched or not. */ +function durableTransactions(watched: boolean) { + return async () => { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + const session = createSession(storage); + await session.commit(async (tx) => { + (await tx.doc(DurableNote)).n = -1; + }, context); + const watch = watched ? await session.watchDoc(DurableNote, context) : undefined; + watch?.start(async () => {}); + return { + commit: (index: number) => + session.commit(async (tx) => { + (await tx.doc(DurableNote)).n = index + 1; + }, context), + close: async () => { + await watch?.stop(); + await session.close(context); + await remove(); + }, + }; + }; +} + +function heap(): number { + globalThis.gc?.(); + return Math.round(process.memoryUsage().heapUsed / 1024); +} + +const format = ({ median, p95, burst }: { median: number; p95: number; burst: number }) => + `median ${(median * 1000).toFixed(1)} µs, p95 ${(p95 * 1000).toFixed(1)} µs, burst ${(burst * 1000).toFixed(1)} µs/commit`; + +const before = heap(); +const durable = await bursts(durableDeltas); +const chord = await bursts(chordDeltas(false)); +const outbox = await bursts(chordDeltas(true)); +console.log(`(a, c) one document delta per commit, ${RUNS} bursts of ${COMMITS}:`); +console.log(` Durable SqliteStorage: ${format(durable)}`); +console.log(` Chord store: ${format(chord)}`); +console.log(` Chord store + outbox: ${format(outbox)}`); +const durablePuts = await bursts(durableEntries); +const chordTable = await bursts(chordPuts(false)); +const tableOutbox = await bursts(chordPuts(true)); +console.log(`(b) one table put (Durable: one entry append) per commit, ${RUNS} bursts of ${COMMITS}:`); +console.log(` Durable SqliteStorage: ${format(durablePuts)}`); +console.log(` Chord store: ${format(chordTable)}`); +console.log(` Chord store + outbox: ${format(tableOutbox)}`); +for (const watched of [false, true]) { + const durableTx = await bursts(durableTransactions(watched)); + const chordTx = await bursts(chordTransactions({ persist: true, watched, outbox: false })); + const chordTxOutbox = await bursts(chordTransactions({ persist: true, watched, outbox: true })); + console.log(`(c) whole transactions, ${watched ? "a watched document" : "no watch"}, ${RUNS} bursts of ${COMMITS}:`); + console.log(` Durable Session: ${format(durableTx)}`); + console.log(` Chord host: ${format(chordTx)}`); + console.log(` Chord host + outbox: ${format(chordTxOutbox)}`); +} +const memory = await bursts(chordTransactions({ persist: false, watched: false, outbox: false })); +const memoryWatched = await bursts(chordTransactions({ persist: false, watched: true, outbox: false })); +console.log(`(d) one in-memory document per transaction (no store), ${RUNS} bursts of ${COMMITS}:`); +console.log(` Chord host: ${format(memory)}`); +console.log(` Chord host, watched: ${format(memoryWatched)}`); +console.log(`heap after GC: ${heap() - before} KiB above the start`); + +/** A file store with `count` rows of `Edits` at version 1 (one commit), as a table handle. */ +async function filledEdits(count: number, body = VALUE.body, definition: typeof Edits = Edits) { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + const table = tableSqlName("bench.store", Edits.name); + const fence = { admit: () => true, adopt: () => {} }; + await prepareTables([{ definition, table }], store, fence, "bench", context); + const handle = new TableHandle(definition, table, store, { + label: "bench", + scope: {}, + assertLive: () => {}, + fail: () => {}, + }); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + for (let i = 0; i < count; i++) + changes.put({ id: `e${String(i).padStart(6, "0")}`, path: `src/f${i % 50}.ts`, at: i, body }); + await store.commit(staged, fence, context); + return { + store, + table, + handle, + fence, + close: async () => { + await store.close(context); + await remove(); + }, + }; +} + +/** The median of `values`. */ +function median(values: number[]): number { + values.sort((a, b) => a - b); + return values[Math.floor(values.length / 2)]!; +} + +console.log("(e) other costs:"); +{ + // 100 commits submitted at once: the time each waits in the queue, apart from its own step. + const { store, table, fence, close } = await filledEdits(0); + const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); + const waits: number[] = []; + const steps: number[] = []; + await Promise.all( + Array.from({ length: 100 }, async (_, index) => { + const row = Edits.rows.put({ id: `q${index}`, path: "p", at: index, body: VALUE.body }, 1); + const put = stagePut(sql.put, table, row, 1); + const submitted = performance.now(); + let started = 0; + const timed: StagedWrite = { + order: put.order, + apply: (step) => { + started = performance.now(); + return put.apply(step); + }, + }; + await store.commit([timed], fence, context); + waits.push(started - submitted); + steps.push(performance.now() - started); + }), + ); + console.log( + ` queue: 100 commits at once, median wait ${(median(waits) * 1000).toFixed(0)} µs, step ${(median(steps) * 1000).toFixed(1)} µs`, + ); + await close(); +} +{ + const { handle, close } = await filledEdits(10_000, VALUE.body, IndexedEdits); + const q = query(IndexedEdits).orderBy((e) => e.at, "desc"); + let after: string | undefined; + const started = performance.now(); + for (let page = 0; page < 100; page++) { + const result = await handle.page(q, 100, context, after === undefined ? {} : { after }); + after = result.next; + } + console.log( + ` query pages: 100 rows in an index's order, ${((performance.now() - started) * 10).toFixed(0)} µs per page`, + ); + await close(); +} +{ + const { store, handle, fence, close } = await filledEdits(10_000); + const staged: StagedWrite[] = []; + new TableChanges(handle, staged).updateWhere((e: { at: never }) => lt(e.at, 10_000 as never), { path: "moved" }); + const started = performance.now(); + await store.commit(staged, fence, context); + console.log( + ` updateWhere over 10,000 rows: one step of ${(performance.now() - started).toFixed(1)} ms (every owner's commits wait)`, + ); + await close(); +} +for (const length of [10, 10_000]) { + const body = "ü".repeat(length); + const started = performance.now(); + const { close } = await filledEdits(2_000, body); + console.log( + ` text: 2,000 puts with ${length}-character cells, ${(((performance.now() - started) / 2_000) * 1000).toFixed(1)} µs per row`, + ); + await close(); +} +{ + const { store, table, fence, close } = await filledEdits(10_000); + const EditsV2 = evolveTable(Edits, (t) => + t.add("dir", str({ default: "" })).rows((row) => ({ dir: row.path.slice(0, 3) })), + ); + const chunks: number[] = []; + const commit = store.commit.bind(store); + store.commit = async (writes, commitFence, commitContext) => { + const started = performance.now(); + const seq = await commit(writes, commitFence, commitContext); + chunks.push(performance.now() - started); + return seq; + }; + const started = performance.now(); + await prepareTables([{ definition: EditsV2, table }], store, fence, "bench", context); + const total = performance.now() - started; + const rows = chunks.slice(1, -1); + console.log( + ` migration (rows) of 10,000 rows: ${total.toFixed(0)} ms, ${((total / 10_000) * 1000).toFixed(1)} µs per row; ` + + `chunk commits: first ${rows[0]!.toFixed(2)} ms, last ${rows.at(-1)!.toFixed(2)} ms, median ${median(rows).toFixed(2)} ms`, + ); + await close(); +} diff --git a/packages/chord/test/next/bench/workers.ts b/packages/chord/test/next/bench/workers.ts new file mode 100644 index 000000000..077b6df10 --- /dev/null +++ b/packages/chord/test/next/bench/workers.ts @@ -0,0 +1,107 @@ +// Benchmark of the workers component (slice 10 §7): a worker's cold start with shipped scripts (fork to `started`, fork +// to `ready`, medians over runs) and the server's memory per idle worker (heap after a GC, 20 workers running idle). +// Run by hand from packages/chord (not in CI), pinned: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/workers.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { hostKind, str } from "../../../src/next/contracts/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { createPlugins } from "../../../src/next-node/plugins/index.ts"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { createWorkers } from "../../../src/next-node/workers/index.ts"; +import { WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import { Control } from "../../next-node/fixtures/workers/control.ts"; + +const RUNS = 7; +const WORKERS = 20; +const ENTRY = fileURLToPath(new URL("../../next-node/fixtures/workers/worker.ts", import.meta.url)); +const ROOT = fileURLToPath(new URL("../../next-node/fixtures/workers/plugins/", import.meta.url)); +const EXEC_ARGV = ["--conditions=source", "--experimental-strip-types", "--no-warnings"]; + +const gc = (globalThis as { gc?: () => void }).gc!; +const Session = hostKind("session", { id: str() }); +const modules = { "@earendil-works/chord/next": chordNext }; +const plugins = createPlugins( + { roots: [ROOT], kinds: { worker: { platform: "node", hostPackages: Object.keys(modules) } }, onError: () => {} }, + context, +); +const scripts = await plugins.scripts("worker", context); + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +// Cold starts: a fork with the shipped scripts, timed to `started` and to `ready`. +const started: number[] = []; +const ready: number[] = []; +for (let run = 0; run < RUNS; run++) { + const forked = performance.now(); + const times: { started?: number; ready?: number } = {}; + const done = Promise.withResolvers(); + const child = new WorkerProcess( + { entry: ENTRY, execArgv: EXEC_ARGV, onOutput: () => {} }, + { + control: (message: FromWorker) => { + if (message.kind === "started") times.started = performance.now() - forked; + if (message.kind === "ready") { + times.ready = performance.now() - forked; + done.resolve(); + } + }, + linkLost: () => {}, + linkClosed: () => {}, + exited: () => {}, + }, + ); + child.send({ + kind: "start", + address: Session({ id: `R${run}` }), + start: {}, + scripts, + lock: `bench:${process.pid}:${run}`, + }); + await done.promise; + started.push(times.started!); + ready.push(times.ready!); + child.kill(); + await child.exited; +} +console.log( + `cold start (${scripts.length} shipped script): fork to started ${median(started).toFixed(0)} ms, to ready ${median(ready).toFixed(0)} ms (median of ${RUNS})`, +); + +// Memory per idle worker on the server: its records, process handle, link and remote references. +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: () => {} }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: ENTRY, + execArgv: EXEC_ARGV, + authorize: () => true, + principal: () => ({ kind: "worker" }), + start: () => ({}), + lock: (address) => `bench:${process.pid}:${address.key}`, + plugins, + onOutput: () => {}, + onError: () => {}, + }, + context, +); +await server.load([], context); +gc(); +const before = process.memoryUsage().heapUsed; +const held = []; +for (let i = 0; i < WORKERS; i++) held.push(await server.get(Control, { id: `W${i}` }, context)); +gc(); +const after = process.memoryUsage().heapUsed; +console.log( + `server heap per idle worker: ${((after - before) / WORKERS / 1024).toFixed(1)} KiB (${WORKERS} workers, one reference each)`, +); +for (const ref of held) ref.release(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/contracts/accepts.test.ts b/packages/chord/test/next/contracts/accepts.test.ts new file mode 100644 index 000000000..c561df3d0 --- /dev/null +++ b/packages/chord/test/next/contracts/accepts.test.ts @@ -0,0 +1,269 @@ +import { describe, expect, it, vi } from "vitest"; +import { + type AcceptsTable, + arr, + evolve, + hostKind, + int, + keyed, + local, + method, + nul, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; +import { checkAccepts } from "../../../src/next/contracts/internal.ts"; + +const Server = hostKind("server"); + +const Note = obj({ text: str() }); +const NoteV2 = obj({ text: str(), tag: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, + }, +}); + +/** Calls `checkAccepts` through an untyped door, as a JavaScript caller would. */ +const checkUntyped = checkAccepts as (contract: unknown, accepts: unknown) => unknown; + +const toV1 = (note: { text: string; tag: string }) => ({ text: note.text }); + +describe("state mappings", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2))); + + it("a changed member maps the current member to the old one", () => { + const accepts: AcceptsTable = { 1: { state: { editor: { notes: (current) => current.map(toV1) } } } }; + expect(checkAccepts(v2, accepts)).toEqual({ versions: [{ version: 1, fingerprint: NotesV1.fingerprint }] }); + // @ts-expect-error: the changed member needs a mapping + expect(() => checkAccepts(v2, { 1: { state: { editor: {} } } })).toThrow( + "notes accepts[1].state.editor.notes: missing mapping", + ); + // @ts-expect-error: "refuse" is only for removed members + expect(() => checkAccepts(v2, { 1: { state: { editor: { notes: "refuse" } } } })).toThrow( + "notes accepts[1].state.editor.notes: expected a function", + ); + // @ts-expect-error: the mapping returns the old shape, not the current one + checkAccepts(v2, { 1: { state: { editor: { notes: (current) => current.map((note) => note.tag) } } } }); + }); + + it("a removed member maps the variant's whole current state, or refuses", () => { + const v3 = evolve(v2, (e) => e.removeState("editor", "count")); + const accepts: AcceptsTable = { + 1: { state: { editor: { notes: (current) => current.map(toV1), count: (state) => state.notes.length } } }, + 2: { state: { editor: { count: "refuse" } } }, + }; + expect(checkAccepts(v3, accepts).versions.map(({ version }) => version)).toEqual([1, 2]); + expect(() => checkUntyped(v3, { 2: { state: { editor: { count: 1 } } } })).toThrow( + 'notes accepts[2].state.editor.count: expected a function or "refuse"', + ); + }); + + it("an absent section is allowed only when nothing is needed there", () => { + // @ts-expect-error: the state section is needed + expect(() => checkAccepts(v2, { 1: {} })).toThrow("notes accepts[1].state.editor: missing mapping"); + const v3 = evolve(v2, (e) => e.addState("reader", "count", int())); + expect(() => checkAccepts(v3, { 2: {} })).not.toThrow(); + expect(() => checkAccepts(v3, { 2: { state: {}, methods: {} } })).not.toThrow(); + }); + + it("unnecessary mappings are refused", () => { + expect(() => + // @ts-expect-error: reader.notes did not change + checkAccepts(v2, { 1: { state: { editor: { notes: (c) => c.map(toV1) }, reader: { notes: (c) => c } } } }), + ).toThrow("notes accepts[1].state.reader: unnecessary mapping"); + const v3 = evolve(v2, (e) => e.addState("reader", "count", int())); + // @ts-expect-error: added members need nothing + expect(() => checkAccepts(v3, { 2: { state: { reader: { count: () => 0 } } } })).toThrow( + "notes accepts[2].state.reader: unnecessary mapping", + ); + }); + + it("is checked by edit, not by schema: a no-op edit and a reverted change still need mappings", () => { + const noop = evolve(NotesV1, (e) => e.state("reader", "notes", arr(Note))); + // @ts-expect-error: the edit needs an (identity) mapping + expect(() => checkAccepts(noop, { 1: {} })).toThrow("notes accepts[1].state.reader: missing mapping"); + expect(() => checkAccepts(noop, { 1: { state: { reader: { notes: (current) => current } } } })).not.toThrow(); + const reverted = evolve(v2, (e) => e.state("editor", "notes", arr(Note))); + expect(() => checkAccepts(reverted, { 1: { state: { editor: { notes: (current) => current } } } })).not.toThrow(); + const readded = evolve(NotesV1, (e) => e.removeState("editor", "count").addState("editor", "count", int())); + expect(() => checkAccepts(readded, { 1: { state: { editor: { count: (current) => current } } } })).not.toThrow(); + }); +}); + +describe("method mappings", () => { + it("changed args map old to current; a changed result maps current to old; both need both", () => { + const args = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => + checkAccepts(args, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }) } } } }), + ).not.toThrow(); + expect(() => + // @ts-expect-error: only args changed + checkAccepts(args, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (r) => r } } } }), + ).toThrow("notes accepts[1].methods.add.result: unnecessary mapping"); + // @ts-expect-error: args must produce the current shape + checkAccepts(args, { 1: { methods: { add: { args: (old) => old } } } }); + + const result = evolve(NotesV1, (e) => e.result("add", int())); + expect(() => + checkAccepts(result, { 1: { methods: { add: { result: (current) => String(current) } } } }), + ).not.toThrow(); + // @ts-expect-error: the result mapping is missing + expect(() => checkAccepts(result, { 1: { methods: { add: {} } } })).toThrow( + "notes accepts[1].methods.add.result: missing mapping", + ); + + const both = evolve(NotesV1, (e) => e.method("add", NoteV2, int())); + const accepts: AcceptsTable = { + 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (current) => String(current) } } }, + }; + expect(() => checkAccepts(both, accepts)).not.toThrow(); + // @ts-expect-error: both sides changed + expect(() => checkAccepts(both, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }) } } } })).toThrow( + "notes accepts[1].methods.add.result: missing mapping", + ); + }); + + it("a removed method takes a call replacement or refuses", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("clear")); + const call = async (args: null) => args; + expect(() => checkAccepts(v2, { 1: { methods: { clear: { call } } } })).not.toThrow(); + expect(() => checkAccepts(v2, { 1: { methods: { clear: "refuse" } } })).not.toThrow(); + // @ts-expect-error: a removed method has no args mapping + expect(() => checkAccepts(v2, { 1: { methods: { clear: { args: (old) => old } } } })).toThrow( + 'notes accepts[1].methods.clear: expected { call } or "refuse"', + ); + // @ts-expect-error: the removed method needs a replacement or a refusal + expect(() => checkAccepts(v2, { 1: {} })).toThrow("notes accepts[1].methods.clear: missing mapping"); + }); + + it("a method removed from one variant that had it counts as removed", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("add").addMethod("editor", "add", Note, str())); + // @ts-expect-error: reader lost `add` + expect(() => checkAccepts(v2, { 1: { methods: { add: { args: (old) => old, result: (r) => r } } } })).toThrow( + 'notes accepts[1].methods.add: expected { call } or "refuse"', + ); + expect(() => checkAccepts(v2, { 1: { methods: { add: "refuse" } } })).not.toThrow(); + }); + + it("call and refuse are also allowed for changed methods", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => checkAccepts(v2, { 1: { methods: { add: { call: async (old) => old.text } } } })).not.toThrow(); + expect(() => checkAccepts(v2, { 1: { methods: { add: "refuse" } } })).not.toThrow(); + expect(() => + // @ts-expect-error: call cannot be combined with args + checkAccepts(v2, { 1: { methods: { add: { call: async () => "", args: (old) => ({ ...old, tag: "" }) } } } }), + ).toThrow('notes accepts[1].methods.add: expected { call } or "refuse"'); + // @ts-expect-error: call returns the old result type + checkAccepts(v2, { 1: { methods: { add: { call: async () => 1 } } } }); + }); + + it("added methods and methods added to a new variant need nothing", () => { + const v2 = evolve(NotesV1, (e) => e.addMethod("reader", "clear", nul(), nul())); + expect(checkAccepts(v2, { 1: {} }).versions).toHaveLength(1); + // @ts-expect-error: unnecessary mapping + expect(() => checkAccepts(v2, { 1: { methods: { clear: "refuse" } } })).toThrow( + "notes accepts[1].methods.clear: unnecessary mapping", + ); + }); + + it("refuses wrong shapes", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: 1 } } } })).toThrow( + "notes accepts[1].methods.add.args: expected a function", + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0, extra: 1 } } } })).toThrow( + 'notes accepts[1].methods.add: unknown key "extra"', + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0 } } }, other: 1 })).toThrow( + "notes accepts[other]: not an older version", + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0 } }, extra: {} } })).toThrow( + 'notes accepts[1]: unknown key "extra"', + ); + expect(() => checkUntyped(v2, null)).toThrow("notes accepts: expected an object"); + }); +}); + +describe("keyed mappings (slice 7b)", () => { + const Store = local("store", { version: 1, params: {}, members: { notes: keyed(Note), drafts: keyed(Note) } }); + const V2 = evolve(Store, (e) => e.keyed("notes", NoteV2).removeKeyed("drafts")); + + it("needs a function or refuse for a changed keyed member, refuse for a removed one", () => { + expect(checkAccepts(V2, { 1: { keyed: { notes: toV1, drafts: "refuse" } } }).versions).toHaveLength(1); + expect(checkAccepts(V2, { 1: { keyed: { notes: "refuse", drafts: "refuse" } } }).versions).toHaveLength(1); + expect(() => checkUntyped(V2, { 1: { keyed: { notes: toV1 } } })).toThrow( + "accepts[1].keyed.drafts: missing mapping", + ); + expect(() => checkUntyped(V2, { 1: { keyed: { notes: toV1, drafts: toV1 } } })).toThrow( + 'accepts[1].keyed.drafts: expected "refuse"', + ); + expect(() => checkUntyped(Store, {})).not.toThrow(); + }); +}); + +describe("versions", () => { + it("refuses unknown versions and the current version", () => { + const v2 = evolve(NotesV1, (e) => e); + // @ts-expect-error: version 3 does not exist + expect(() => checkAccepts(v2, { 3: {} })).toThrow("notes accepts[3]: not an older version"); + // @ts-expect-error: version 2 is current + expect(() => checkAccepts(v2, { 2: {} })).toThrow("notes accepts[2]: not an older version"); + expect(checkAccepts(v2, {})).toEqual({ versions: [] }); + }); + + it("maps directly over several versions: every edit after the accepted version counts", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + const v3 = evolve(v2, (e) => e.result("add", int()).removeState("editor", "count")); + const accepts: AcceptsTable = { + 1: { + state: { editor: { count: "refuse" } }, + methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (current) => String(current) } }, + }, + 2: { + state: { editor: { count: (state) => state.notes.length } }, + methods: { add: { result: (r) => `${r}` } }, + }, + }; + expect(checkAccepts(v3, accepts).versions).toEqual([ + { version: 1, fingerprint: NotesV1.fingerprint }, + { version: 2, fingerprint: v2.fingerprint }, + ]); + const fromV1 = { + state: { editor: { count: "refuse" } }, + methods: { add: { result: (r: number) => `${r}` } }, + } as const; + // @ts-expect-error: from v1, args changed too + expect(() => checkAccepts(v3, { 1: fromV1 })).toThrow("notes accepts[1].methods.add.args: missing mapping"); + }); + + it("never calls a mapping", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2).removeMethod("clear").state("editor", "count", str())); + const mappings = { args: vi.fn(), call: vi.fn(), count: vi.fn() }; + checkUntyped(v2, { + 1: { + state: { editor: { count: mappings.count } }, + methods: { add: { args: mappings.args }, clear: { call: mappings.call } }, + }, + }); + expect(Object.values(mappings).map((mapping) => mapping.mock.calls.length)).toEqual([0, 0, 0]); + }); + + it("local tables are flat: state without a variant level", () => { + const v1 = local("info", { version: 1, params: {}, members: { title: str(), save: method(str()) } }); + const v2 = evolve(v1, (e) => e.state("title", int()).args("save", int())); + const accepts: AcceptsTable = { + 1: { state: { title: (current) => String(current) }, methods: { save: { args: (old) => old.length } } }, + }; + expect(() => checkAccepts(v2, accepts)).not.toThrow(); + expect(() => + checkUntyped(v2, { 1: { state: { members: { title: String } }, methods: { save: "refuse" } } }), + ).toThrow("info accepts[1].state.members: unnecessary mapping"); + }); +}); diff --git a/packages/chord/test/next/contracts/contract.test.ts b/packages/chord/test/next/contracts/contract.test.ts new file mode 100644 index 000000000..e7c409022 --- /dev/null +++ b/packages/chord/test/next/contracts/contract.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { type Contract, contractData } from "../../../src/next/contracts/contract.ts"; +import { + arr, + bytes, + evolve, + type Implementation, + int, + local, + method, + nul, + obj, + type ParamsOf, + remote, + str, + union, +} from "../../../src/next/contracts/index.ts"; +import { checkAccepts, currentVariants, LOCAL, takesNoArgs } from "../../../src/next/contracts/internal.ts"; +import { bindProvider, checkParams } from "../../../src/next/data/internal.ts"; +import type { Watchable } from "../../../src/next/state/index.ts"; +import { expectCollected, Server } from "../support.ts"; + +const Notes = remote("notes", { + version: 1, + params: { session: str(), page: int() }, + hostAddress: () => Server(), + variants: { + editor: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() }), str()), clear: method() } }, + reader: { state: { notes: arr(str()) } }, + }, +}); + +describe("contract handles", () => { + it("carry kind, name, version and fingerprint only", () => { + expect(Object.keys(Notes)).toEqual(["kind", "name", "version", "fingerprint"]); + }); + + it("are branded: spread and hand-built copies are refused", () => { + const copy = { ...Notes }; + const refusal = "notes: not a declared contract (spread and hand-built copies are refused)"; + expect(() => contractData(copy)).toThrow(refusal); + expect(() => evolve(copy, (e) => e)).toThrow(refusal); + expect(() => checkAccepts(copy, {})).toThrow(refusal); + expect(() => bindProvider(copy, {})).toThrow(refusal); + expect(() => checkParams(copy, { session: "s", page: 1 }, BACKGROUND_CONTEXT)).toThrow(refusal); + }); + + it("are collected when dropped, with their data", async () => { + const refs = (() => { + const contract = local("gc", { version: 1, params: {}, members: { s: str() } }); + return [new WeakRef(contract), new WeakRef(contractData(contract))]; + })(); + await expectCollected(...refs); + }); +}); + +/** The members of one variant of a remote implementation. */ +type Members = Extract, { readonly variant: V }>["members"]; + +describe("derived types", () => { + it("Implementation: one variant with its members, state watchables and methods taking args and a context", () => { + expectTypeOf["variant"]>().toEqualTypeOf<"editor" | "reader">(); + type Editor = Members; + expectTypeOf().toEqualTypeOf>(); + expectTypeOf().toEqualTypeOf<(args: { text: string }, context: Context) => Promise>(); + // A method without arguments omits them. + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + expectTypeOf>().toEqualTypeOf<"notes">(); + }); + + it("Implementation of a local contract is flat", () => { + const Info = local("info", { + version: 1, + params: {}, + members: { title: str(), blob: bytes(), save: method(str()) }, + }); + type Info = Implementation; + expectTypeOf().toEqualTypeOf<"title" | "blob" | "save">(); + expectTypeOf().toEqualTypeOf<(args: string, context: Context) => Promise>(); + const Counts = local("counts", { + version: 1, + params: {}, + members: { reset: method(), total: method(undefined, int()), maybe: method(union([nul(), str()])) }, + }); + type Counts = Implementation; + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + // Only a `null` argument schema is omitted; a union with null is an argument. + expectTypeOf().toEqualTypeOf<(args: string | null, context: Context) => Promise>(); + }); + + it("takesNoArgs: exactly the methods whose argument schema is null", () => { + const Counts = local("counts", { + version: 1, + params: {}, + members: { + reset: method(), + total: method(undefined, int()), + set: method(int()), + maybe: method(union([nul(), str()])), + }, + }); + const methods = [...currentVariants(Counts).get(LOCAL)!.methods].map(([name, data]) => [name, takesNoArgs(data)]); + expect(methods).toEqual([ + ["reset", true], + ["total", true], + ["set", false], + ["maybe", false], + ]); + }); + + it("ParamsOf: the params object", () => { + expectTypeOf>().toEqualTypeOf<{ readonly session: string; readonly page: number }>(); + }); +}); diff --git a/packages/chord/test/next/contracts/declare.test.ts b/packages/chord/test/next/contracts/declare.test.ts new file mode 100644 index 000000000..9f17d301c --- /dev/null +++ b/packages/chord/test/next/contracts/declare.test.ts @@ -0,0 +1,299 @@ +import { describe, expect, it } from "vitest"; +import { + arr, + evolve, + hostKind, + int, + keyed, + local, + method, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; +import { currentVariants, LOCAL } from "../../../src/next/contracts/internal.ts"; + +const Note = obj({ text: str() }); + +const Server = hostKind("server"); + +const hostAddress = () => Server(); + +/** Declares through an untyped door, as a JavaScript caller would. */ +const untyped = { + remote: remote as (name: unknown, declaration: unknown) => unknown, + local: local as (name: unknown, declaration: unknown) => unknown, +}; + +describe("declaration checks", () => { + it("accept a remote and a local declaration", () => { + const Notes = remote("notes", { + version: 1, + params: { session: str() }, + hostAddress, + variants: { + editor: { state: { notes: arr(Note) }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) } }, + }, + }); + expect(Notes).toMatchObject({ kind: "remote", name: "notes", version: 1 }); + expect(Notes.fingerprint).toMatch(/^[0-9a-f]{16}$/); + const Info = local("app.session-info", { + version: 3, + params: {}, + members: { init: obj({ title: str() }), clear: method() }, + }); + expect(Info).toMatchObject({ kind: "local", name: "app.session-info", version: 3 }); + }); + + // Each case declares a valid `hostAddress` unless it checks that one, so the check it names is the one reached. + it.each([ + [ + "empty name", + "", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + 'A contract name "": expected lowercase', + ], + [ + "name not a string", + 1, + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + "A contract name 1: expected lowercase", + ], + [ + "an uppercase name", + "Notes", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + "expected lowercase", + ], + [ + "a name with a colon", + "a:b", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + "expected lowercase", + ], + [ + "version zero", + "n", + { version: 0, params: {}, hostAddress, variants: { a: {} } }, + "n.version: expected a positive integer", + ], + [ + "fractional version", + "n", + { version: 1.5, params: {}, hostAddress, variants: { a: {} } }, + "n.version: expected a positive integer", + ], + [ + "unsafe version", + "n", + { version: 2 ** 53, params: {}, hostAddress, variants: { a: {} } }, + "n.version: expected a positive integer", + ], + [ + "unknown declaration key", + "n", + { version: 1, params: {}, hostAddress, variants: { a: {} }, place: "x" }, + 'n: unknown key "place"', + ], + [ + "a host address that is no function", + "n", + { version: 1, params: {}, variants: { a: {} }, hostAddress: "x" }, + "n.hostAddress: expected a function of the params", + ], + [ + "no host address", + "n", + { version: 1, params: {}, variants: { a: {} } }, + "n.hostAddress: expected a function of the params", + ], + [ + "no variants", + "n", + { version: 1, params: {}, hostAddress, variants: {} }, + "n.variants: at least one variant is required", + ], + ["missing params", "n", { version: 1, hostAddress, variants: { a: {} } }, "n.params: expected an object"], + [ + "unknown variant key", + "n", + { version: 1, params: {}, hostAddress, variants: { a: { members: {} } } }, + 'n.a: unknown key "members"', + ], + [ + "a schema under methods", + "n", + { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: str() } } } }, + "n.a.methods.m: expected method(...)", + ], + [ + "a method under state", + "n", + { version: 1, params: {}, hostAddress, variants: { a: { state: { s: method() } } } }, + "n.a.state.s: a method belongs in methods", + ], + [ + "bad state schema", + "n", + { version: 1, params: {}, hostAddress, variants: { a: { state: { s: { not: {} } } } } }, + 'n.a.state.s: unsupported keyword "not"', + ], + [ + "open arguments", + "n", + { + version: 1, + params: {}, + hostAddress, + variants: { a: { methods: { m: method({ type: "object", properties: {} }) } } }, + }, + "n.a.methods.m.args: argument objects must be closed", + ], + [ + "bad result", + "n", + { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: method(undefined, { not: {} }) } } } }, + 'n.a.methods.m.result: unsupported keyword "not"', + ], + ])("remote refuses %s", (_case, name, declaration, message) => { + expect(() => untyped.remote(name, declaration)).toThrow(message); + }); + + it.each([ + ["unknown key", { version: 1, params: {}, members: {}, variants: {} }, 'i: unknown key "variants"'], + ["a host address", { version: 1, params: {}, members: {}, hostAddress }, 'i: unknown key "hostAddress"'], + [ + "bad member schema", + { version: 1, params: {}, members: { s: { not: {} } } }, + 'i.members.s: unsupported keyword "not"', + ], + [ + "open arguments", + { version: 1, params: {}, members: { m: method({ type: "object", properties: {} }) } }, + "i.members.m.args: argument objects must be closed", + ], + ])("local refuses %s", (_case, declaration, message) => { + expect(() => untyped.local("i", declaration)).toThrow(message); + }); + + it("refuse a remote declaration without a host address at the type level", () => { + expect(() => + // @ts-expect-error: every remote type declares its host address + remote("n", { version: 1, params: {}, variants: { a: {} } }), + ).toThrow("n.hostAddress: expected a function of the params"); + }); + + it("does not check state for closed objects", () => { + expect(() => + local("i", { + version: 1, + params: {}, + members: { s: { type: "object", properties: { a: str() }, required: ["a"] } }, + }), + ).not.toThrow(); + }); +}); + +describe("D11: members across variants", () => { + it("refuses a name that is state in one variant and a method in another", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + hostAddress, + // @ts-expect-error: `x` is a method in variant b + variants: { a: { state: { x: str() } }, b: { methods: { x: method() } } }, + }), + ).toThrow('n.b.methods.x: "x" is a state member; its kind never changes'); + }); + + it("refuses one name as state and method within a variant", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + hostAddress, + // @ts-expect-error: `x` is both state and a method + variants: { a: { state: { x: str() }, methods: { x: method() } } }, + }), + ).toThrow('n.a.methods.x: "x" is a state member'); + }); + + it("refuses a method with different signatures in two variants", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + hostAddress, + // @ts-expect-error: `m` takes a string in a and an integer in b + variants: { a: { methods: { m: method(str()) } }, b: { methods: { m: method(int()) } } }, + }), + ).toThrow("n.b.methods.m: a method has the same signature in every variant"); + expect(() => + remote("n", { + version: 1, + params: {}, + hostAddress, + variants: { + a: { methods: { m: method(str(), str()) } }, + b: { methods: { m: method(str({ max: 3 }), str()) } }, + }, + }), + ).toThrow("n.b.methods.m: a method has the same signature in every variant"); + }); + + it("allows state schemas to differ per variant (redaction)", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + hostAddress, + variants: { + editor: { state: { notes: arr(obj({ text: str(), author: str() })) }, methods: { m: method(str()) } }, + reader: { state: { notes: arr(obj({ text: str() })) }, methods: { m: method(str()) } }, + }, + }), + ).not.toThrow(); + }); +}); + +describe("reserved member names", () => { + it.each(["then", "release", "variant", "state", "retry"])("refuses %s in remote, local and evolve", (name) => { + expect(() => + remote("n", { version: 1, params: {}, hostAddress, variants: { v: { state: { [name]: str() } } } }), + ).toThrow(`n.v.state.${name}: "${name}" is reserved: a reference has it`); + expect(() => local("n", { version: 1, params: {}, members: { [name]: method() } })).toThrow( + `n.members.${name}: "${name}" is reserved: a reference has it`, + ); + const base = local("n", { version: 1, params: {}, members: {} }); + expect(() => evolve(base, (e) => e.addState(name, str()))).toThrow( + `n v2.members.${name}: "${name}" is reserved: a reference has it`, + ); + }); +}); + +describe("keyed members (slice 6a §12)", () => { + it("declare a keyed member in a local contract: its own kind and a node of its own in the fingerprint", () => { + const Keyed = local("app.store", { version: 1, params: {}, members: { notes: keyed(Note), add: method() } }); + const State = local("app.store", { version: 1, params: {}, members: { notes: Note, add: method() } }); + const variant = currentVariants(Keyed).get(LOCAL)!; + expect([[...variant.keyed.keys()], [...variant.state.keys()]]).toEqual([["notes"], []]); + expect(Keyed.fingerprint).not.toBe(State.fingerprint); + // Evolving the contract carries its keyed members over unchanged (their versions are slice 7's). + const next = evolve(Keyed, (e) => e.addMethod("clear", str(), str())); + expect(currentVariants(next).get(LOCAL)!.keyed.has("notes")).toBe(true); + }); + + it("refuse a keyed member in a remote contract, at the type level and at declaration", () => { + expect(() => + remote("app.notes", { + version: 1, + params: {}, + hostAddress, + // @ts-expect-error keyed members are only in local contracts + variants: { editor: { state: { notes: keyed(Note) } } }, + }), + ).toThrow("app.notes.editor.state.notes: keyed members are only in local contracts"); + }); +}); diff --git a/packages/chord/test/next/contracts/evolve.test.ts b/packages/chord/test/next/contracts/evolve.test.ts new file mode 100644 index 000000000..0c4a17354 --- /dev/null +++ b/packages/chord/test/next/contracts/evolve.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import type { Context } from "../../../src/context/index.ts"; +import { type Contract, contractData, type ShapeOf } from "../../../src/next/contracts/contract.ts"; +import type { CurrentStep } from "../../../src/next/contracts/history.ts"; +import { + arr, + bool, + evolve, + hostKind, + type Implementation, + int, + keyed, + local, + method, + nul, + obj, + remote, + type Static, + str, +} from "../../../src/next/contracts/index.ts"; + +const Server = hostKind("server"); + +const Note = obj({ text: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, + }, +}); + +const InfoV1 = local("info", { version: 1, params: {}, members: { title: str(), clear: method() } }); + +/** The current members of a contract's variant, as Chord holds them. */ +function current(contract: Contract, variant: string) { + const data = contractData(contract); + const members = data.versions.at(-1)!.variants.get(variant)!; + return { state: [...members.state.keys()], methods: [...members.methods.keys()], data }; +} + +/** The current version of a contract's type-level history. */ +type VersionOf = CurrentStep["history"]>["version"]; + +/** The members of one variant of a remote implementation. */ +type Members = Extract, { readonly variant: V }>["members"]; + +describe("edits", () => { + it("state changes one state member of one variant", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "count", str())); + expect(v2.version).toBe(2); + expectTypeOf["count"]["value"]>().toEqualTypeOf(); + expectTypeOf["notes"]["value"]>().toEqualTypeOf<{ text: string }[]>(); + // @ts-expect-error: `missing` is not a state member of editor + expect(() => evolve(NotesV1, (e) => e.state("editor", "missing", str()))).toThrow( + "notes v2.editor.state.missing: no such state member", + ); + }); + + it("addState adds a state member", () => { + const v2 = evolve(NotesV1, (e) => e.addState("reader", "count", int())); + expect(current(v2, "reader").state).toEqual(["notes", "count"]); + expectTypeOf["count"]["value"]>().toEqualTypeOf(); + expect(() => evolve(NotesV1, (e) => e.addState("reader", "notes", int()))).toThrow( + "notes v2.reader.state.notes: already declared", + ); + }); + + it("removeState removes a state member", () => { + const v2 = evolve(NotesV1, (e) => e.removeState("editor", "count")); + expect(current(v2, "editor").state).toEqual(["notes"]); + expectTypeOf>().toEqualTypeOf<"notes" | "add" | "clear">(); + }); + + it("args and result change one side of a method in every variant that has it", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", obj({ text: str(), tag: str() })).result("add", int())); + const { data } = current(v2, "editor"); + const editor = data.versions[1]!.variants.get("editor")!.methods.get("add")!; + const reader = data.versions[1]!.variants.get("reader")!.methods.get("add")!; + expect(editor).toEqual(reader); + expect(data.versions[1]!.changes).toEqual([ + ["editor", "add", "args"], + ["reader", "add", "args"], + ["editor", "add", "result"], + ["reader", "add", "result"], + ]); + type Add = Members["add"]; + expectTypeOf[0]>().toEqualTypeOf<{ text: string; tag: string }>(); + expectTypeOf>>().toEqualTypeOf(); + }); + + it("method replaces both sides", () => { + const v2 = evolve(NotesV1, (e) => e.method("clear", obj({ all: bool() }), bool())); + type Clear = Members["clear"]; + expectTypeOf[0]>().toEqualTypeOf<{ all: boolean }>(); + expectTypeOf>>().toEqualTypeOf(); + // @ts-expect-error: no method `missing` + expect(() => evolve(NotesV1, (e) => e.method("missing", nul(), nul()))).toThrow('notes v2: no method "missing"'); + }); + + it("addMethod adds a method; to another variant only with the same signature", () => { + const v2 = evolve(NotesV1, (e) => e.addMethod("reader", "clear", nul(), nul())); + expect(current(v2, "reader").methods).toEqual(["add", "clear"]); + expect(() => evolve(NotesV1, (e) => e.addMethod("reader", "clear", str(), nul()))).toThrow( + "notes v2.reader.methods.clear: a method has the same signature in every variant", + ); + expect(() => evolve(NotesV1, (e) => e.addMethod("reader", "add", nul(), nul()))).toThrow( + "notes v2.reader.methods.add: already declared", + ); + }); + + it("removeMethod removes a method from every variant", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("add")); + expect(current(v2, "editor").methods).toEqual(["clear"]); + expect(current(v2, "reader").methods).toEqual([]); + expectTypeOf>().toEqualTypeOf<"notes">(); + }); + + it("local edits take no variant", () => { + const v2 = evolve(InfoV1, (e) => + e + .state("title", int()) + .addState("body", str()) + .addMethod("save", obj({ force: bool() }), nul()) + .removeMethod("clear"), + ); + expect(current(v2, "members")).toMatchObject({ state: ["title", "body"], methods: ["save"] }); + expectTypeOf>().toEqualTypeOf<"title" | "body" | "save">(); + const v3 = evolve(v2, (e) => e.removeState("body").args("save", nul())); + // Arguments changed to `null`: the method omits them now. + expectTypeOf["save"]>>().toEqualTypeOf<[context: Context]>(); + }); + + it("checks every edit: closed arguments and the vocabulary", () => { + expect(() => evolve(NotesV1, (e) => e.args("add", { type: "object", properties: { text: str() } }))).toThrow( + "notes v2.editor.methods.add.args: argument objects must be closed", + ); + expect(() => evolve(InfoV1, (e) => e.addMethod("m", { type: "object", properties: {} }, nul()))).toThrow( + "info v2.members.m.args: argument objects must be closed", + ); + expect(() => evolve(NotesV1, (e) => e.state("editor", "count", { not: {} } as never))).toThrow( + 'notes v2.editor.state.count: unsupported keyword "not"', + ); + expect(() => + evolve(NotesV1, (e) => e.state("editor", "count", { type: "object", properties: {} })), + ).not.toThrow(); + }); +}); + +describe("keyed edits (slice 7b)", () => { + const Store = local("store", { version: 1, params: {}, members: { notes: keyed(Note), n: int() } }); + + it("changes, adds and removes a local contract's keyed members, each version fingerprinted", () => { + const V2 = evolve(Store, (e) => e.keyed("notes", obj({ text: str(), tag: str() })).addKeyed("drafts", Note)); + const V3 = evolve(V2, (e) => e.removeKeyed("drafts")); + const versions = contractData(V3).versions; + expect(versions.map(({ variants }) => [...variants.get("members")!.keyed.keys()])).toEqual([ + ["notes"], + ["notes", "drafts"], + ["notes"], + ]); + expect(versions[1]!.changes).toEqual([ + ["members", "notes", "keyed"], + ["members", "drafts", "keyed"], + ]); + expect(new Set(versions.map(({ fingerprint }) => fingerprint)).size).toBe(3); + // The same edits elsewhere give the same fingerprint (an old plugin's copy). + const again = evolve(Store, (e) => e.keyed("notes", obj({ text: str(), tag: str() })).addKeyed("drafts", Note)); + expect(contractData(again).versions[1]!.fingerprint).toBe(versions[1]!.fingerprint); + }); + + it("refuses an unknown keyed member, one declared already, and a kind change", () => { + expect(() => evolve(Store, (e) => e.keyed("missing" as "notes", Note))).toThrow("no such keyed member"); + expect(() => evolve(Store, (e) => e.addKeyed("n", Note))).toThrow("already declared"); + expect(() => evolve(Store, (e) => e.removeKeyed("missing" as "notes"))).toThrow("no such keyed member"); + }); +}); + +describe("kinds and fixed parts", () => { + it("a member's kind never changes, also not after a removal and re-add", () => { + expect(() => + evolve(NotesV1, (e) => e.removeState("editor", "count").addMethod("editor", "count", nul(), nul())), + ).toThrow('notes v2.editor.methods.count: "count" is a state member; its kind never changes'); + const v2 = evolve(NotesV1, (e) => e.removeMethod("clear")); + const v3 = evolve(v2, (e) => e.state("editor", "count", str())); + expect(() => evolve(v3, (e) => e.addState("reader", "clear", str()))).toThrow( + 'notes v4.reader.state.clear: "clear" is a method member; its kind never changes', + ); + }); + + it("the variant set never changes", () => { + // @ts-expect-error: no variant `admin` + expect(() => evolve(NotesV1, (e) => e.addState("admin", "x", str()))).toThrow( + "notes v2.admin: unknown variant (the variant set never changes)", + ); + }); + + it("a new base is a full declaration with an explicit version and no older versions", () => { + const base = remote("notes", { + version: 5, + params: { session: str(), tenant: str() }, + hostAddress: () => Server(), + variants: { editor: {} }, + }); + expect(contractData(base).versions.map(({ version }) => version)).toEqual([5]); + expect(evolve(base, (e) => e).version).toBe(6); + }); + + it("evolve must return the editor it was given", () => { + const other = evolve(NotesV1, (e) => e); + expect(other.version).toBe(2); + let foreign: unknown; + evolve(InfoV1, (e) => { + foreign = e; + return e; + }); + expect(() => evolve(NotesV1, () => foreign as never)).toThrow( + "notes v2: evolve must return the editor it was given", + ); + }); + + it("leaves the previous contract untouched", () => { + evolve(NotesV1, (e) => e.removeState("editor", "count").removeMethod("add")); + expect(current(NotesV1, "editor")).toMatchObject({ state: ["notes", "count"], methods: ["add", "clear"] }); + }); +}); + +describe("long histories", () => { + it("count versions beyond 9 and 99, at runtime and in the types", () => { + let contract: Contract = InfoV1; + for (let step = 0; step < 120; step++) contract = evolve(contract, (e) => e); + expect(contract.version).toBe(121); + expect(contractData(contract).versions).toHaveLength(121); + + const v9 = local("i", { version: 9, params: {}, members: { s: str() } }); + const v10 = evolve(v9, (e) => e); + const v99 = local("i", { version: 99, params: {}, members: { s: str() } }); + const v100 = evolve(v99, (e) => e); + const v101 = evolve(v100, (e) => e.state("s", int())); + expectTypeOf>().toEqualTypeOf<"10">(); + expectTypeOf>().toEqualTypeOf<"100">(); + expectTypeOf>().toEqualTypeOf<"101">(); + }); + + it("the types track every step with its changes", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "count", str()).addMethod("reader", "clear", nul(), nul())); + type History = ShapeOf["history"]; + expectTypeOf().toEqualTypeOf<2>(); + expectTypeOf().toEqualTypeOf< + ["editor", "count", "state"] | ["reader", "clear", "args" | "result"] + >(); + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/packages/chord/test/next/contracts/fingerprint.test.ts b/packages/chord/test/next/contracts/fingerprint.test.ts new file mode 100644 index 000000000..a31434198 --- /dev/null +++ b/packages/chord/test/next/contracts/fingerprint.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { + evolve, + hostKind, + int, + local, + method, + obj, + remote, + type Schema, + str, + union, +} from "../../../src/next/contracts/index.ts"; +import { NORMAL_FORM_VERSION } from "../../../src/next/contracts/internal.ts"; +import type { Params } from "../../../src/next/contracts/params.ts"; + +const Server = hostKind("server"); + +/** Fingerprint of a one-variant remote contract with state `s` and params `params`. */ +function fingerprintOf(state: Schema, params: Params = {}): string { + return remote("n", { version: 1, params, hostAddress: () => Server(), variants: { a: { state: { s: state } } } }) + .fingerprint; +} + +describe("fingerprints", () => { + it("pin the normal form: a fixed contract has a fixed hash", () => { + expect(NORMAL_FORM_VERSION).toBe(2); + const Notes = remote("notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { state: { notes: obj({ text: str() }) }, methods: { add: method(obj({ text: str() }), str()) } }, + reader: { state: { notes: obj({ text: str() }) } }, + }, + }); + expect(Notes.fingerprint).toBe("b3eda86070a7e319"); + }); + + it("are stable across key order and annotations", () => { + const plain = { + type: "object", + properties: { a: { type: "string" }, b: { type: "integer" } }, + required: ["a", "b"], + }; + const reordered = { + required: ["b", "a"], + title: "T", + properties: { + b: { type: "integer", description: "d" }, + a: { $comment: "c", type: "string", examples: ["x"] }, + }, + type: "object", + deprecated: true, + }; + expect(fingerprintOf(reordered)).toBe(fingerprintOf(plain)); + }); + + it.each([ + ["defaults", str(), str({ default: "x" })], + ["bounds", str(), str({ max: 3 })], + ["formats", str(), { type: "string", format: "email" }], + ["union order", union([str(), int()]), union([int(), str()])], + ["additionalProperties", { type: "object", properties: {} }, obj({})], + ["readOnly", str(), { type: "string", readOnly: true }], + ])("are sensitive to %s", (_case, a, b) => { + expect(fingerprintOf(a)).not.toBe(fingerprintOf(b)); + }); + + it("are sensitive to params order", () => { + expect(fingerprintOf(str(), { a: str(), b: int() })).not.toBe(fingerprintOf(str(), { b: int(), a: str() })); + }); + + it("give the same v1 from a separate declaration, and evolve keeps v1's", () => { + const declare = () => + remote("n", { version: 1, params: {}, hostAddress: () => Server(), variants: { a: { state: { s: str() } } } }); + const v1 = declare(); + const v2 = evolve(v1, (e) => e.state("a", "s", int())); + expect(declare().fingerprint).toBe(v1.fingerprint); + expect(v2.fingerprint).not.toBe(v1.fingerprint); + expect(evolve(declare(), (e) => e.state("a", "s", int())).fingerprint).toBe(v2.fingerprint); + }); + + it("differ between local and remote", () => { + const asRemote = remote("n", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { members: { state: { s: str() } } }, + }); + const asLocal = local("n", { version: 1, params: {}, members: { s: str() } }); + expect(asLocal.fingerprint).not.toBe(asRemote.fingerprint); + }); + + it("do not depend on the name or version", () => { + const a = remote("a", { version: 1, params: {}, hostAddress: () => Server(), variants: { v: {} } }); + const b = remote("b", { version: 7, params: {}, hostAddress: () => Server(), variants: { v: {} } }); + expect(a.fingerprint).toBe(b.fingerprint); + }); +}); diff --git a/packages/chord/test/next/contracts/history.test.ts b/packages/chord/test/next/contracts/history.test.ts new file mode 100644 index 000000000..9d7ab7ce2 --- /dev/null +++ b/packages/chord/test/next/contracts/history.test.ts @@ -0,0 +1,37 @@ +import { describe, expectTypeOf, it } from "vitest"; +import type { + ChangesAfter, + CurrentStep, + Increment, + OlderVersions, + StepAt, +} from "../../../src/next/contracts/history.ts"; + +type Empty = { readonly a: { readonly state: Record; readonly methods: Record } }; +type History = [ + { version: "9"; variants: Empty; changes: never }, + { version: "10"; variants: Empty; changes: ["a", "x", "state"] }, + { version: "11"; variants: Empty; changes: ["a", "m", "args"] }, +]; + +describe("type-level history", () => { + it("increments decimal versions", () => { + expectTypeOf>().toEqualTypeOf<"2">(); + expectTypeOf>().toEqualTypeOf<"10">(); + expectTypeOf>().toEqualTypeOf<"20">(); + expectTypeOf>().toEqualTypeOf<"100">(); + expectTypeOf>().toEqualTypeOf<"1100">(); + }); + + it("finds the current step, older versions and steps by version", () => { + expectTypeOf["version"]>().toEqualTypeOf<"11">(); + expectTypeOf>().toEqualTypeOf<"9" | "10">(); + expectTypeOf["changes"]>().toEqualTypeOf<["a", "x", "state"]>(); + }); + + it("collects every change after a version", () => { + expectTypeOf>().toEqualTypeOf<["a", "x", "state"] | ["a", "m", "args"]>(); + expectTypeOf>().toEqualTypeOf<["a", "m", "args"]>(); + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/packages/chord/test/next/contracts/host-kind.test.ts b/packages/chord/test/next/contracts/host-kind.test.ts new file mode 100644 index 000000000..93026d54d --- /dev/null +++ b/packages/chord/test/next/contracts/host-kind.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { evolve, hostKind, int, oneOf, opt, remote, str } from "../../../src/next/contracts/index.ts"; +import { contractData, kindParams } from "../../../src/next/contracts/internal.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { checkAddress } from "../../../src/next/data/internal.ts"; +import { expectCollected } from "../support.ts"; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); +const Shard = hostKind("shard", { region: oneOf("eu", "us"), n: int({ min: 0 }) }); + +describe("host kinds", () => { + it("make addresses: the kind, the params and a key built like an identity key", () => { + expect(Server()).toEqual({ kind: "server", params: {}, key: "server" }); + expect(Session({ id: "S1" })).toEqual({ kind: "session", params: { id: "S1" }, key: "session|S1" }); + expect(Shard({ region: "eu", n: 3 }).key).toBe("shard|eu|3"); + // Escaped like identity keys: two different param lists never make one key. + expect(Session({ id: "a|b" }).key).not.toBe(Shard({ region: "a" as "eu", n: "b" as unknown as number }).key); + expect(Session({ id: "a|b" }).key).toBe("session|a\\|b"); + }); + + it("follow the params policy of contracts at declaration", () => { + // @ts-expect-error: params fields are required + expect(() => hostKind("k", { id: opt(str()) })).toThrow("k.params.id: a param is"); + expect(() => hostKind("")).toThrow(`A host kind's name "": expected lowercase [a-z0-9@/._~-]`); + expect(() => hostKind("Session")).toThrow("expected lowercase"); + expect(() => hostKind("a:b")).toThrow("expected lowercase"); + expect(() => hostKind("client")).toThrow('"client" is reserved for hosts without an address'); + expect(hostKind("@pi/session.v2_~-").kind).toBe("@pi/session.v2_~-"); + }); + + it("check an address's params where it is routed, with the kind's compiled check: invalid/host_address", () => { + const params = kindParams(Session); + expect(checkAddress("session", params, { id: "S1" }, context).key).toBe("session|S1"); + for (const bad of [{ id: 1 }, {}, { id: "S1", extra: 1 }, null]) { + let thrown: unknown; + try { + checkAddress("session", params, bad, context); + } catch (error) { + thrown = error; + } + expect(isChordError(thrown, "invalid") && thrown.reason).toBe("host_address"); + } + }); + + it("keep their params data only while the kind is referenced", async () => { + const ref = (() => { + const Temp = hostKind("temp", { id: str() }); + kindParams(Temp); + return new WeakRef(Temp); + })(); + await expectCollected(ref); + }); +}); + +describe("hostAddress", () => { + const variants = { v: { state: { n: int() } } }; + + it("is not part of the fingerprint and is carried by evolve", () => { + const hostAddress = ({ session }: { session: string }) => Session({ id: session }); + const onSession = remote("app.placed", { version: 1, params: { session: str() }, hostAddress, variants }); + const onServer = remote("app.placed", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants, + }); + expect(onSession.fingerprint).toBe(onServer.fingerprint); + expect(contractData(onSession).hostAddress).toBe(hostAddress); + const next = evolve(onSession, (v) => v.addState("v", "m", int())); + expect(contractData(next).hostAddress).toBe(hostAddress); + }); + + it("is typed by the contract's params", () => { + remote("app.typed", { + version: 1, + params: { session: str() }, + // @ts-expect-error: the params have no `id` + hostAddress: ({ id }) => Session({ id }), + variants, + }); + }); +}); diff --git a/packages/chord/test/next/contracts/params.test.ts b/packages/chord/test/next/contracts/params.test.ts new file mode 100644 index 000000000..e760a9c58 --- /dev/null +++ b/packages/chord/test/next/contracts/params.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { + arr, + bool, + bytes, + hostKind, + int, + num, + obj, + oneOf, + opt, + remote, + str, +} from "../../../src/next/contracts/index.ts"; + +const Server = hostKind("server"); + +const variants = { a: { state: { s: str() } } }; + +describe("params policy", () => { + it("accepts required str, int, bool and oneOf of strings, with constraints", () => { + expect(() => + remote("n", { + version: 1, + params: { + a: str({ min: 1, max: 9, pattern: "^x" }), + b: int({ min: 0 }), + c: bool(), + d: oneOf("chat", "agent"), + }, + hostAddress: () => Server(), + variants, + }), + ).not.toThrow(); + }); + + it("refuses optional fields, in the types and at declaration", () => { + expect(() => + // @ts-expect-error: params fields are required + remote("n", { version: 1, params: { a: opt(str()) }, hostAddress: () => Server(), variants }), + ).toThrow("n.params.a: a param is a required str(), int(), bool() or oneOf(...) of strings"); + }); + + it.each([ + ["a number", num()], + ["nested objects", obj({ a: str() })], + ["arrays", arr(str())], + ["bytes", bytes()], + ["numeric literals", oneOf(1, 2)], + ["defaults", str({ default: "x" })], + ])("refuses %s, in the types and at declaration", (_case, schema) => { + // @ts-expect-error: not a params field schema + expect(() => remote("n", { version: 1, params: { p: schema }, hostAddress: () => Server(), variants })).toThrow( + "n.params.p: a param is", + ); + }); +}); diff --git a/packages/chord/test/next/contracts/schema.test.ts b/packages/chord/test/next/contracts/schema.test.ts new file mode 100644 index 000000000..ee6a05691 --- /dev/null +++ b/packages/chord/test/next/contracts/schema.test.ts @@ -0,0 +1,106 @@ +// Tests of the schema helpers: the JSON Schema they build over pi-schema's builders, their types, `bytes()`, `method`. + +import { compile } from "@earendil-works/pi-schema"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + arr, + bool, + bytes, + int, + lit, + map, + method, + nul, + num, + obj, + oneOf, + opt, + type Static, + str, + union, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { Method } from "../../../src/next/contracts/internal.ts"; +import type { Equal } from "../support.ts"; + +describe("schema helpers", () => { + it("produce plain JSON Schema with bounds under their JSON Schema names", () => { + expect(str({ min: 1, max: 3, pattern: "^a" })).toEqual({ + type: "string", + minLength: 1, + maxLength: 3, + pattern: "^a", + }); + expect(num({ min: 0 })).toEqual({ type: "number", minimum: 0 }); + expect(int({ max: 9 })).toEqual({ type: "integer", maximum: 9 }); + expect(arr(bool(), { min: 1, max: 2 })).toEqual({ + type: "array", + items: { type: "boolean" }, + minItems: 1, + maxItems: 2, + }); + expect(nul()).toEqual({ type: "null" }); + expect(lit(3)).toEqual({ type: "number", const: 3 }); + expect(oneOf("a", "b")).toEqual({ anyOf: [lit("a"), lit("b")] }); + expect(union([str(), nul()])).toEqual({ anyOf: [str(), nul()] }); + expect(map(int())).toEqual({ type: "object", patternProperties: { "^.*$": int() } }); + expect(unknown()).toEqual({}); + }); + + it("leave absent bounds out instead of writing undefined", () => { + expect(Object.keys(str())).toEqual(["type"]); + expect(Object.keys(bytes())).toEqual(["type"]); + }); + + it("obj() is always closed and opt() makes a field optional", () => { + const schema = obj({ a: str(), b: opt(int()) }); + expect(schema).toEqual({ + type: "object", + required: ["a"], + properties: { a: str(), b: int() }, + additionalProperties: false, + }); + expect(compile(schema).check({ a: "x" })).toBe(true); + expect(compile(schema).check({ a: "x", c: 1 })).toBe(false); + }); + + it("Static gives the values' type; pi-schema's compiler checks them", () => { + const schema = obj({ kind: oneOf("chat", "agent"), n: opt(int()), tags: arr(str()), data: bytes() }); + type Expected = { kind: "chat" | "agent"; n?: number; tags: string[]; data: Uint8Array }; + expectTypeOf, Expected>>().toEqualTypeOf(); + expect(compile(schema).check({ kind: "chat", tags: [], data: new Uint8Array(1) })).toBe(true); + }); +}); + +describe("bytes", () => { + it("is plain data: the marker and nothing else", () => { + const schema = bytes({ min: 1, max: 2 }); + expect(Reflect.ownKeys(schema)).toEqual(["type", "minLength", "maxLength"]); + expect(schema).toEqual({ type: "bytes", minLength: 1, maxLength: 2 }); + }); + + it("checks instanceof Uint8Array and the length bounds", () => { + const validator = compile(bytes({ min: 1, max: 2 })); + expect(validator.check(new Uint8Array(1))).toBe(true); + expect(validator.check(new Uint8Array(2))).toBe(true); + expect(validator.check(new Uint8Array(0))).toBe(false); + expect(validator.check(new Uint8Array(3))).toBe(false); + expect(validator.check([1])).toBe(false); + expect(validator.check("ab")).toBe(false); + }); + + it("checks inside a record (slices.md L1 carry-over)", () => { + const validator = compile(map(bytes({ max: 1 }))); + expect(validator.check({ a: new Uint8Array(1) })).toBe(true); + expect(validator.check({ a: new Uint8Array(2) })).toBe(false); + }); +}); + +describe("method", () => { + it("an omitted side is null", () => { + expect(method()).toEqual(new Method(nul(), nul())); + expect(method(str())).toEqual(new Method(str(), nul())); + expect(method(undefined, int())).toEqual(new Method(nul(), int())); + expectTypeOf(method(undefined, int())).toEqualTypeOf, ReturnType>>(); + }); +}); diff --git a/packages/chord/test/next/contracts/snapshot.test.ts b/packages/chord/test/next/contracts/snapshot.test.ts new file mode 100644 index 000000000..e3d6d76b8 --- /dev/null +++ b/packages/chord/test/next/contracts/snapshot.test.ts @@ -0,0 +1,194 @@ +// Tests of schema snapshots: the vocabulary, closed arguments, copies and their normal forms, bytes leaves. + +import { compile } from "@earendil-works/pi-schema"; +import { describe, expect, it } from "vitest"; +import { arr, bytes, int, map, obj, opt, str, union } from "../../../src/next/contracts/schema.ts"; +import { snapshotArguments, snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; + +describe("vocabulary", () => { + it("accepts primitives, arrays, tuples, objects, records, unions, intersections, $defs/$ref, constraints", () => { + const recursive = { + $defs: { + Node: { + type: "object", + properties: { value: { type: "number" }, next: { anyOf: [{ type: "null" }, { $ref: "#/$defs/Node" }] } }, + required: ["value", "next"], + }, + }, + $ref: "#/$defs/Node", + }; + const schemas = [ + { type: "array", prefixItems: [{ type: "string" }, { type: "integer" }], items: false }, + { allOf: [obj({ a: str() }), obj({ b: str() })] }, + { enum: ["a", "b"] }, + recursive, + str({ min: 1, pattern: "^x" }), + { type: "string", format: "email", default: "a@b.c", readOnly: true, title: "t", description: "d" }, + { type: "number", multipleOf: 2, exclusiveMinimum: 0 }, + { type: "array", items: { type: "string" }, uniqueItems: true }, + { type: "object", minProperties: 1, maxProperties: 2, $comment: "c", deprecated: true, examples: [{}] }, + map(int()), + bytes({ max: 4 }), + map(bytes()), + ]; + for (const schema of schemas) expect(() => snapshotSchema(schema, "s")).not.toThrow(); + }); + + it.each([ + ["conditional", { if: { type: "string" }, else: { minLength: 1 } }, 'unsupported keyword "else"'], + ["dependent", { type: "object", dependentSchemas: { a: {} } }, 'unsupported keyword "dependentSchemas"'], + [ + "dependentRequired", + { type: "object", dependentRequired: { a: ["b"] } }, + 'unsupported keyword "dependentRequired"', + ], + ["not", { not: { type: "string" } }, 'unsupported keyword "not"'], + ["contains", { type: "array", contains: { type: "string" } }, 'unsupported keyword "contains"'], + ["propertyNames", { type: "object", propertyNames: { pattern: "^a" } }, 'unsupported keyword "propertyNames"'], + ["unevaluatedProperties", { unevaluatedProperties: false }, 'unsupported keyword "unevaluatedProperties"'], + ["unevaluatedItems", { unevaluatedItems: false }, 'unsupported keyword "unevaluatedItems"'], + ["contentSchema", { type: "string", contentSchema: {} }, 'unsupported keyword "contentSchema"'], + ["definitions", { definitions: {} }, 'unsupported keyword "definitions"'], + ["oneOf", { oneOf: [{ type: "string" }] }, 'unsupported keyword "oneOf"'], + ["unknown keyword", { type: "string", flavour: "x" }, 'unsupported keyword "flavour"'], + ["non-JSON type", { type: "bigint" }, 'unsupported type "bigint"'], + ["undefined", { type: "undefined" }, 'unsupported type "undefined"'], + ["function", { type: "function", parameters: [], returnType: { type: "null" } }, "s: unsupported"], + [ + "non-enumerable key (a TypeBox refinement)", + Object.defineProperty({ type: "string" }, "~refine", { value: [{ check: () => true }] }), + "s.~refine: non-enumerable keys are not supported", + ], + ["opt(...) outside an object", opt(str()), "s: expected a schema"], + ["non-JSON data", { type: "number", default: Number.NaN }, "s.default: expected JSON data"], + ["undefined data", { type: "number", maximum: undefined }, "s.maximum: expected JSON data"], + ["foreign prototype", new (class Schema {})(), "s: expected a schema"], + ["array as schema", [], "s: expected a schema"], + ])("refuses %s", (_name, schema, message) => { + expect(() => snapshotSchema(schema, "s")).toThrow(message); + }); + + it("refuses symbol keys, accessors and cycles, naming the path", () => { + expect(() => snapshotSchema({ type: "string", [Symbol("x")]: 1 }, "s")).toThrow( + "s: symbol keys are not supported", + ); + const accessor = Object.defineProperty({ type: "object" }, "properties", { get: () => ({}), enumerable: true }); + expect(() => snapshotSchema(accessor, "s")).toThrow("s.properties: accessors are not supported"); + const cyclic: { type: string; items?: object } = { type: "array" }; + cyclic.items = cyclic; + expect(() => snapshotSchema(cyclic, "s")).toThrow("s.items: cyclic schema"); + expect(() => snapshotSchema({ type: "object", properties: { a: { type: 1 } } }, "s")).toThrow( + "s.properties.a.type: unsupported type 1", + ); + }); + + it("allows a schema object shared by two places (not a cycle)", () => { + const shared = str(); + expect(() => snapshotSchema(obj({ a: shared, b: shared }), "s")).not.toThrow(); + }); +}); + +describe("closed arguments", () => { + it("requires every object, nested ones included, to be closed", () => { + expect(() => snapshotArguments(obj({ a: obj({ b: str() }) }), "m.args")).not.toThrow(); + const open = (properties: object) => ({ type: "object", properties }); + expect(() => snapshotArguments(obj({ a: open({ b: str() }) }), "m.args")).toThrow( + "m.args.properties.a: argument objects must be closed (additionalProperties: false)", + ); + expect(() => snapshotArguments(open({}), "m.args")).toThrow("m.args: argument objects must be closed"); + expect(() => snapshotArguments(obj({ list: arr(open({})) }), "m.args")).toThrow( + "m.args.properties.list.items: argument objects must be closed", + ); + expect(() => snapshotArguments({ type: "object" }, "m.args")).toThrow("argument objects must be closed"); + }); + + it("accepts a raw closed object schema and records", () => { + const closed = { type: "object", properties: { a: str() }, additionalProperties: false }; + expect(() => snapshotArguments(closed, "m")).not.toThrow(); + expect(() => snapshotArguments(obj({ tags: map(str()) }), "m")).not.toThrow(); + }); + + it("checks objects inside $defs, unions and tuples", () => { + const open = { type: "object", properties: { a: str() } }; + expect(() => snapshotArguments(union([open, str()]), "m")).toThrow("m.anyOf[0]: argument objects"); + expect(() => snapshotArguments({ type: "array", items: [open] }, "m")).toThrow("m.items[0]: argument objects"); + expect(() => snapshotArguments({ $defs: { A: open }, $ref: "A" }, "m")).toThrow("m.$defs.A: argument objects"); + }); + + it("does not apply to state or results", () => { + expect(() => snapshotSchema({ type: "object", properties: { a: str() } }, "state")).not.toThrow(); + }); +}); + +describe("snapshots", () => { + it("are copies, independent of later changes to the app's schema objects", () => { + const app = obj({ text: str(), tags: arr(str()) }) as { properties: { text: object }; required: string[] }; + const { schema } = snapshotSchema(app, "s"); + (app.properties.text as { minLength?: number }).minLength = 5; + app.required.push("x"); + expect(schema).not.toBe(app); + expect(compile(schema).check({ text: "", tags: [] })).toBe(true); + }); + + it("copy data values (defaults, consts)", () => { + const fallback = { a: [1] }; + const { schema } = snapshotSchema({ type: "object", default: fallback, additionalProperties: false }, "s"); + fallback.a.push(2); + const copied = (schema as { default: { a: number[] } }).default; + expect(copied).toEqual({ a: [1] }); + }); + + it("copy a bytes leaf as data: the compiled check comes from its keywords", () => { + const { schema } = snapshotSchema(bytes({ max: 2 }), "s"); + expect(schema).toEqual({ type: "bytes", maxLength: 2 }); + expect(compile(schema).check(new Uint8Array(2))).toBe(true); + expect(compile(schema).check(new Uint8Array(3))).toBe(false); + // A record of bytes (slices.md L1 carry-over: refused while bytes carried a TypeBox refinement). + const record = snapshotSchema(map(bytes({ min: 1 })), "s").schema; + expect(compile(record).check({ a: new Uint8Array(1) })).toBe(true); + expect(compile(record).check({ a: new Uint8Array(0) })).toBe(false); + }); + + it("refuse a bytes marker with other keywords or bad bounds", () => { + expect(() => snapshotSchema({ type: "bytes", items: {} }, "s")).toThrow('s: unsupported keyword "items"'); + expect(() => snapshotSchema({ type: "bytes", maxLength: -1 }, "s")).toThrow( + "s: bytes bounds must be non-negative integers", + ); + }); + + it("normal form: sorted keys, sorted required and enum, annotations dropped on schema nodes only", () => { + const { normal } = snapshotSchema( + { + title: "T", + type: "object", + required: ["b", "a"], + properties: { b: { enum: ["y", "x"] }, a: { type: "string", default: { title: "kept", z: 1, a: 2 } } }, + additionalProperties: false, + }, + "s", + ); + expect(JSON.stringify(normal)).toBe( + JSON.stringify({ + additionalProperties: false, + properties: { a: { default: { a: 2, title: "kept", z: 1 }, type: "string" }, b: { enum: ["x", "y"] } }, + required: ["a", "b"], + type: "object", + }), + ); + }); + + it("normal form keeps union and tuple order", () => { + const ab = snapshotSchema(union([str(), int()]), "s").normal; + const ba = snapshotSchema(union([int(), str()]), "s").normal; + expect(ab).not.toEqual(ba); + }); +}); + +describe("bytes", () => { + it("mark a snapshot with a bytes leaf anywhere inside", () => { + expect(snapshotSchema(str(), "s").bytes).toBe(false); + expect(snapshotSchema(obj({ a: arr(obj({ b: bytes() })) }), "s").bytes).toBe(true); + expect(snapshotSchema(union([str(), bytes()]), "s").bytes).toBe(true); + expect(snapshotSchema({ type: "object", default: { type: "bytes" } }, "s").bytes).toBe(false); + }); +}); diff --git a/packages/chord/test/next/data/args.test.ts b/packages/chord/test/next/data/args.test.ts new file mode 100644 index 000000000..bdd26d32b --- /dev/null +++ b/packages/chord/test/next/data/args.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { + type AcceptsTable, + arr, + bytes, + evolve, + int, + local, + method, + nul, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { bindProvider, type CheckedArgs, checkArgs, type Value } from "../../../src/next/data/internal.ts"; +import { counted, expectCollected, Server } from "../support.ts"; + +const Note = obj({ text: str() }); +const NoteV2 = obj({ text: str(), tag: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { + state: { notes: arr(Note) }, + methods: { add: method(Note, str()), clear: method(), attach: method(obj({ data: bytes({ max: 4 }) })) }, + }, + reader: { state: { notes: arr(Note) } }, + }, +}); +const NotesV2 = evolve(NotesV1, (e) => e.args("add", NoteV2).result("add", int())); +const NotesV3 = evolve(NotesV2, (e) => e.removeMethod("attach")); + +const toV2 = vi.fn((old: { text: string }) => ({ ...old, tag: "v1" })); +const resultToOld = (current: number) => String(current); +const attach = async () => null; +const accepts: AcceptsTable = { + 1: { methods: { add: { args: toV2, result: resultToOld }, attach: { call: attach } } }, + 2: { methods: { attach: "refuse" } }, +}; +const provider = bindProvider(NotesV3, accepts); + +/** The error a refused check throws, as JSON. */ +function refusal(run: () => unknown): unknown { + try { + run(); + } catch (error) { + if (isChordError(error)) return error.toJSON(); + throw error; + } + throw new Error("expected a refusal"); +} + +describe("bindProvider", () => { + it("checks the accepts table once and lists the accepted versions", () => { + expect(provider.contract).toBe(NotesV3); + expect(provider.accepted).toEqual([ + { version: 1, fingerprint: NotesV1.fingerprint }, + { version: 2, fingerprint: NotesV2.fingerprint }, + ]); + expect(() => bindProvider(NotesV3, { 1: {} } as never)).toThrow("notes accepts[1].methods.add: missing mapping"); + }); + + it("compiles nothing and calls no mapping", async () => { + const counts = await counted(() => bindProvider(NotesV3, accepts)); + expect(counts).toEqual({ walks: 0, validations: 0, compilations: 0 }); + expect(toV2).not.toHaveBeenCalled(); + }); +}); + +describe("checkArgs", () => { + it("returns current-version arguments as passed: no copy, never mapped", () => { + const raw = { text: "t", tag: "x" }; + const checked = checkArgs(provider, "add", raw, 3, BACKGROUND_CONTEXT); + expect(checked).toEqual({ kind: "method", args: raw, result: undefined }); + expect(checked.args).toBe(raw); + expect(toV2).not.toHaveBeenCalled(); + expectTypeOf(checked).toEqualTypeOf>(); + if (checked.kind === "method") expectTypeOf(checked.args).toEqualTypeOf<{ text: string; tag: string }>(); + }); + + it("checks an old version against the old schema, then maps directly to the current shape", () => { + const checked = checkArgs(provider, "add", { text: "t" }, 1, BACKGROUND_CONTEXT); + expect(checked).toEqual({ kind: "method", args: { text: "t", tag: "v1" }, result: resultToOld }); + expect(refusal(() => checkArgs(provider, "add", { text: "t", tag: "x" }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "args", + details: { path: "/tag" }, + }); + // Version 2 already has the current args: checked with that schema, not mapped. + expect(checkArgs(provider, "add", { text: "t", tag: "x" }, 2, BACKGROUND_CONTEXT)).toEqual({ + kind: "method", + args: { text: "t", tag: "x" }, + result: undefined, + }); + }); + + it("returns the old arguments with the call replacement; never invokes it", () => { + const raw = { data: new Uint8Array(2) }; + expect(checkArgs(provider, "attach", raw, 1, BACKGROUND_CONTEXT)).toEqual({ + kind: "call", + args: raw, + call: attach, + }); + }); + + it("refuses unknown fields, nested ones too, and checks bytes by instanceof and length", () => { + expect( + refusal(() => checkArgs(provider, "add", { text: "t", tag: "x", extra: 1 }, 3, BACKGROUND_CONTEXT)), + ).toMatchObject({ + details: { path: "/extra" }, + }); + expect(refusal(() => checkArgs(provider, "attach", { data: [1, 2] }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + details: { path: "/data" }, + }); + expect( + refusal(() => checkArgs(provider, "attach", { data: new Uint8Array(5) }, 1, BACKGROUND_CONTEXT)), + ).toMatchObject({ + details: { path: "/data" }, + }); + const Nested = local("nested", { version: 1, params: {}, members: { m: method(obj({ a: obj({ b: str() }) })) } }); + const nested = bindProvider(Nested, {}); + expect(refusal(() => checkArgs(nested, "m", { a: { b: "x", c: 1 } }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + details: { path: "/a/c" }, + }); + }); + + it("takes null for no-argument methods", () => { + expect(checkArgs(provider, "clear", null, 3, BACKGROUND_CONTEXT)).toEqual({ + kind: "method", + args: null, + result: undefined, + }); + expect(refusal(() => checkArgs(provider, "clear", {}, 3, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "args", + details: { path: "/" }, + }); + }); + + it("checks local callers too", () => { + const Info = local("info", { version: 1, params: {}, members: { save: method(obj({ title: str() }), nul()) } }); + const local1 = bindProvider(Info, {}); + expect(checkArgs(local1, "save", { title: "t" }, 1, BACKGROUND_CONTEXT).args).toEqual({ title: "t" }); + expect(refusal(() => checkArgs(local1, "save", { title: 1 }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + message: "args/title: expected string, got integer", + }); + }); + + it("treats `format` as an annotation: a contract with format email accepts any string", () => { + const Mail = local("mail", { + version: 1, + params: {}, + members: { send: method(obj({ to: { type: "string", format: "email" } }), nul()) }, + }); + const mail = bindProvider(Mail, {}); + expect(checkArgs(mail, "send", { to: "nope" }, 1, BACKGROUND_CONTEXT).args).toEqual({ to: "nope" }); + }); + + it("refuses an unknown method, an unaccepted version and a refused method", () => { + expect(refusal(() => checkArgs(provider, "missing", null, 3, BACKGROUND_CONTEXT))).toEqual({ + code: "invalid", + reason: "member", + message: 'notes v3 has no method "missing"', + }); + expect(refusal(() => checkArgs(provider, "attach", null, 3, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "member", + }); + expect(refusal(() => checkArgs(provider, "add", { text: "t" }, 4, BACKGROUND_CONTEXT))).toEqual({ + code: "contract_mismatch", + message: "notes v4 is not accepted", + }); + expect( + refusal(() => checkArgs(bindProvider(NotesV3, {}), "add", { text: "t" }, 1, BACKGROUND_CONTEXT)), + ).toMatchObject({ + code: "contract_mismatch", + }); + expect(refusal(() => checkArgs(provider, "attach", { data: new Uint8Array() }, 2, BACKGROUND_CONTEXT))).toEqual({ + code: "contract_mismatch", + message: 'notes v2 refuses "attach"', + }); + }); + + it("compiles lazily, once per accepted version and method; equal schema objects share a validator", async () => { + const v1 = local("lazy", { version: 1, params: {}, members: { m: method(obj({ a: str() })), clear: method() } }); + const v2 = evolve(v1, (e) => e.args("m", obj({ a: str(), b: str() }))); + const fresh = bindProvider(v2, { 1: { methods: { m: { args: (old) => ({ ...old, b: "" }) } } } }); + const calls = [ + (context: Context) => checkArgs(fresh, "clear", null, 2, context), + (context: Context) => checkArgs(fresh, "clear", null, 2, context), + // Version 1 shares the unchanged `clear` schema object with version 2. + (context: Context) => checkArgs(fresh, "clear", null, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x" }, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x" }, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x", b: "y" }, 2, context), + ]; + const compilations = []; + for (const call of calls) compilations.push((await counted(call)).compilations); + expect(compilations).toEqual([1, 0, 0, 1, 0, 1]); + }); + + it("counts one validation per call and no walk", async () => { + const counts = await counted((context) => checkArgs(provider, "add", { text: "t" }, 1, context)); + expect(counts).toEqual({ walks: 0, validations: 1, compilations: 0 }); + }); + + it("types a call replacement's arguments as values", () => { + expectTypeOf, { kind: "call" }>["call"]>().toEqualTypeOf< + (args: Value, context: Context) => Promise + >(); + }); +}); + +describe("memory", () => { + it("a dropped binding releases its dispatch table, mappings, schemas and validators after the last call", async () => { + const refs = await (async () => { + const contract = evolve( + local("gc", { version: 1, params: {}, members: { m: method(obj({ a: str() })) } }), + (e) => e.args("m", obj({ a: str(), b: str() })), + ); + const mapping = (old: { a: string }) => ({ ...old, b: "" }); + const bound = bindProvider(contract, { 1: { methods: { m: { args: mapping } } } }); + const pending = Promise.resolve(checkArgs(bound, "m", { a: "x" }, 1, BACKGROUND_CONTEXT)); + await pending; + return [new WeakRef(contract), new WeakRef(bound), new WeakRef(mapping)]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/data/errors.test.ts b/packages/chord/test/next/data/errors.test.ts new file mode 100644 index 000000000..bebc4e8e4 --- /dev/null +++ b/packages/chord/test/next/data/errors.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { ChordError, type ChordErrorReason, isChordError, PluginError } from "../../../src/next/data/index.ts"; +import { + asChordError, + CHORD_ERROR_REASONS, + chordErrorFromJson, + invalid, + PROTOCOL_VERSION, +} from "../../../src/next/data/internal.ts"; + +describe("ChordError", () => { + it("is a tagged union: a wrong pairing is a type error", () => { + new ChordError({ code: "invalid", reason: "args", message: "m" }); + new ChordError({ code: "unavailable", message: "m" }); + new ChordError({ code: "unavailable", reason: "locked", message: "m" }); + // @ts-expect-error: invalid requires a reason + new ChordError({ code: "invalid", message: "m" }); + // @ts-expect-error: gone carries no reason + new ChordError({ code: "gone", reason: "args", message: "m" }); + // @ts-expect-error: part belongs to too_large + new ChordError({ code: "unavailable", reason: "part", message: "m" }); + }); + + it("narrowing on code narrows reason", () => { + const error = new ChordError({ code: "too_large", reason: "part", message: "m" }) as ChordError; + if (error.code === "invalid") expectTypeOf(error.reason).toEqualTypeOf>(); + if (error.code === "unavailable") + expectTypeOf(error.reason).toEqualTypeOf< + "locked" | "park_timeout" | "busy" | "storage_failed" | "undeclared" | undefined + >(); + if (error.code === "gone") expectTypeOf(error.reason).toEqualTypeOf(); + expect(error).toBeInstanceOf(Error); + expect(error.name).toBe("ChordError"); + }); + + it("round-trips through JSON without stack or cause", () => { + const details = { path: "/text" }; + const error = new ChordError( + { code: "invalid", reason: "args", message: "args/text: bad", details }, + { cause: new Error("x") }, + ); + const json = JSON.parse(JSON.stringify(error)); + expect(json).toEqual({ code: "invalid", reason: "args", message: "args/text: bad", details }); + const back = chordErrorFromJson(json); + expect(back).toBeInstanceOf(ChordError); + expect(back.toJSON()).toEqual(json); + expect(new ChordError({ code: "gone", message: "m" }).toJSON()).toEqual({ code: "gone", message: "m" }); + }); + + it("keeps details by reference", () => { + const details = { large: [1, 2, 3] }; + expect(chordErrorFromJson({ code: "gone", message: "m", details }).details).toBe(details); + }); + + it.each([ + ["not an object", "x", "not an object"], + ["null", null, "not an object"], + ["an array", [], "not an object"], + ["unknown key", { code: "gone", message: "m", stack: "s" }, 'unknown key "stack"'], + ["unknown code", { code: "teapot", message: "m" }, "unknown code"], + ["inherited name as code", { code: "toString", message: "m" }, "unknown code"], + ["code not a string", { code: 1, message: "m" }, "unknown code"], + ["invalid without reason", { code: "invalid", message: "m" }, "reason not allowed for invalid"], + ["reason not allowed", { code: "gone", reason: "args", message: "m" }, "reason not allowed for gone"], + [ + "reason of another code", + { code: "unavailable", reason: "part", message: "m" }, + "reason not allowed for unavailable", + ], + ["message not a string", { code: "gone", message: 1 }, "message is not a string"], + ])("chordErrorFromJson refuses %s", (_case, raw, problem) => { + expect(() => chordErrorFromJson(raw)).toThrow(`Invalid ChordError JSON: ${problem}`); + }); +}); + +describe("protocol table", () => { + it("is pinned together with PROTOCOL_VERSION", () => { + const pinned = { + version: 1, + reasons: { + invalid: ["args", "params", "member", "host_address", "reentrant", "read_after_bulk", "cycle"], + contract_mismatch: ["updating"], + forbidden: ["revoked"], + gone: [], + not_ready: [], + released: ["ended", "closed", "settled", "version", "discarded"], + too_large: ["part"], + unavailable: ["locked", "park_timeout", "busy", "storage_failed", "undeclared"], + aborted: [], + }, + }; + const changed = + PROTOCOL_VERSION !== pinned.version || JSON.stringify(CHORD_ERROR_REASONS) !== JSON.stringify(pinned.reasons); + expect(changed, "the error table changed: bump PROTOCOL_VERSION and update this pin").toBe(false); + }); +}); + +describe("helpers", () => { + it("isChordError checks the class and optionally the code", () => { + const error = invalid("member", "no method"); + expect(isChordError(error)).toBe(true); + expect(isChordError(error, "invalid")).toBe(true); + expect(isChordError(error, "gone")).toBe(false); + expect(isChordError(new Error("x"))).toBe(false); + expect(isChordError({ code: "invalid", reason: "member", message: "m" })).toBe(false); + }); + + it("invalid builds an invalid error with its reason and details", () => { + const error = invalid("params", "params/a: missing", { path: "/a" }); + expect(error.toJSON()).toEqual({ + code: "invalid", + reason: "params", + message: "params/a: missing", + details: { path: "/a" }, + }); + expectTypeOf(error.code).toEqualTypeOf<"invalid">(); + }); + + it("asChordError keeps a ChordError and wraps anything else as unavailable", () => { + const chord = invalid("cycle", "c"); + expect(asChordError(chord, "open")).toBe(chord); + const foreign = new TypeError("boom"); + const wrapped = asChordError(foreign, "open notes"); + expect(wrapped.toJSON()).toEqual({ code: "unavailable", message: "open notes: boom" }); + expect(wrapped.cause).toBe(foreign); + expect(asChordError("text", "open").message).toBe("open: text"); + }); +}); + +describe("PluginError", () => { + it("carries a plugin's code, message and JSON details", () => { + const error = new PluginError("quota", "too many", { limit: 3 }); + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ name: "PluginError", code: "quota", message: "too many", details: { limit: 3 } }); + expect(new PluginError("c", "m").details).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next/data/failure.test.ts b/packages/chord/test/next/data/failure.test.ts new file mode 100644 index 000000000..c7e88861e --- /dev/null +++ b/packages/chord/test/next/data/failure.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import { invalid, openFailure, sameCause, waitingFor } from "../../../src/next/data/internal.ts"; + +describe("failure causes", () => { + it("a value thrown by an open is transient unless marked broken", () => { + const error = new Error("network down"); + const transient = openFailure(error, "open notes"); + expect(transient.kind).toBe("transient"); + expect(transient.error.toJSON()).toEqual({ code: "unavailable", message: "open notes: network down" }); + expect(transient.error.cause).toBe(error); + }); + + it("broken() marks an open's failure as broken; its message is the cause's", () => { + const marked = broken(new Error("bad config")); + expect(marked.message).toBe("bad config"); + const cause = openFailure(marked, "open notes"); + expect(cause.kind).toBe("broken"); + expect(cause.error.message).toBe("open notes: bad config"); + }); + + it("the kind comes from the stage, never from the error code", () => { + const chord = invalid("host_address", "wrong host"); + expect(openFailure(chord, "open")).toEqual({ kind: "transient", error: chord }); + expect(openFailure(broken(chord), "open")).toEqual({ kind: "broken", error: chord }); + }); +}); + +describe("a dependent's waiting cause (dependency model §3)", () => { + const root = new ChordError({ code: "unavailable", message: "app.catalog is not served here" }); + + it("names the nearest blocker and the root cause, its blocker's error as the cause", () => { + const cause = waitingFor("app.chat(S1)", root); + expect(cause.kind).toBe("waiting"); + expect(cause.error.toJSON()).toEqual({ + code: "unavailable", + message: "waiting for app.chat(S1): app.catalog is not served here", + }); + expect(cause.error.cause).toBe(root); + // A chain names its nearest blocker and its root only (fix round 1, Opus F1). + const chain = waitingFor("app.panel", cause.error); + expect(chain.error.message).toBe("waiting for app.panel: app.catalog is not served here"); + }); + + it("reaches a fixed point when a ring of waiting attempts passes its causes round (Opus F1)", () => { + let b = waitingFor("X", new ChordError({ code: "unavailable", message: "X is not served here" })).error; + let rounds = 0; + for (; rounds < 10; rounds++) { + const next = waitingFor("A", waitingFor("B", b).error).error; + if (next.message === b.message) break; + b = next; + } + expect(rounds).toBeLessThanOrEqual(1); + expect(b.message).toBe("waiting for A: X is not served here"); + }); + + it("keeps the root cause within 500 characters: a long label is clipped first (Sol 8)", () => { + const original = new ChordError({ code: "unavailable", message: "ORIGINAL: missing provider" }); + const message = waitingFor(`sol.item(${"x".repeat(600)})`, original).error.message; + expect(message.length).toBeLessThanOrEqual(500); + expect(message).toContain(original.message); + expect(message.startsWith("waiting for sol.item(xxx")).toBe(true); + }); + + it("keeps a long chain's root cause, never repeated ellipses (Sol 8)", () => { + const original = new ChordError({ code: "unavailable", message: "ORIGINAL: missing provider" }); + let error = original; + for (let i = 0; i < 2000; i++) error = waitingFor(`level-${i}`, error).error; + expect(error.message).toBe("waiting for level-1999: ORIGINAL: missing provider"); + }); + + it("clips a long root past its share, and both when both are long", () => { + const long = new ChordError({ code: "unavailable", message: "y".repeat(600) }); + const short = waitingFor("app.a", long).error.message; + expect(short).toHaveLength(500); + expect(short.startsWith("waiting for app.a: yyy")).toBe(true); + const both = waitingFor("z".repeat(600), long).error.message; + expect(both).toHaveLength(500); + expect(both.endsWith("y…")).toBe(true); + }); + + it("is compared by value: kind, code, reason and message", () => { + expect(sameCause(waitingFor("a", root), waitingFor("a", root))).toBe(true); + expect(sameCause(waitingFor("a", root), waitingFor("b", root))).toBe(false); + expect(sameCause({ kind: "broken", error: root }, { kind: "transient", error: root })).toBe(false); + const other = new ChordError({ code: "unavailable", reason: "undeclared", message: root.message }); + expect(sameCause({ kind: "waiting", error: root }, { kind: "waiting", error: other })).toBe(false); + }); +}); diff --git a/packages/chord/test/next/data/params.test.ts b/packages/chord/test/next/data/params.test.ts new file mode 100644 index 000000000..20777e981 --- /dev/null +++ b/packages/chord/test/next/data/params.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { contractData } from "../../../src/next/contracts/contract.ts"; +import { bool, int, local, oneOf, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { checkParams, identityKey } from "../../../src/next/data/internal.ts"; +import { validatorFor } from "../../../src/next/data/validator.ts"; +import { counted, expectCollected, Server } from "../support.ts"; + +const Session = remote("session", { + version: 1, + params: { id: str({ min: 1 }), mode: oneOf("chat", "agent"), page: int({ min: 0 }), open: bool() }, + hostAddress: () => Server(), + variants: { owner: {} }, +}); + +const valid = { id: "S1", mode: "chat", page: 2, open: true } as const; + +/** The `invalid/params` error of a refused check. */ +function refusal(raw: unknown): unknown { + try { + checkParams(Session, raw, BACKGROUND_CONTEXT); + } catch (error) { + if (isChordError(error, "invalid")) + return { reason: error.reason, message: error.message, details: error.details }; + throw error; + } + throw new Error("expected a refusal"); +} + +describe("checkParams", () => { + it("returns the caller's own object and the key of the field values in declared order", () => { + const { params, key } = checkParams(Session, valid, BACKGROUND_CONTEXT); + expect(params).toBe(valid); + expect(key).toBe("S1|chat|2|true"); + expectTypeOf(params).toEqualTypeOf<{ + readonly id: string; + readonly mode: "chat" | "agent"; + readonly page: number; + readonly open: boolean; + }>(); + }); + + it("gives the same key whatever the property order", () => { + expect(checkParams(Session, { open: true, page: 2, mode: "chat", id: "S1" }, BACKGROUND_CONTEXT).key).toBe( + "S1|chat|2|true", + ); + }); + + it("refuses missing, extra and wrong fields at their paths", () => { + const { id: _, ...missing } = valid; + expect(refusal(missing)).toEqual({ + reason: "params", + message: expect.stringMatching(/^params\/id: /), + details: { path: "/id" }, + }); + expect(refusal({ ...valid, extra: 1 })).toMatchObject({ + message: expect.stringMatching(/^params\/extra: /), + details: { path: "/extra" }, + }); + expect(refusal({ ...valid, page: "2" })).toMatchObject({ + message: "params/page: expected integer, got string", + details: { path: "/page" }, + }); + expect(refusal("S1")).toMatchObject({ message: "params: expected object, got string", details: { path: "/" } }); + }); + + it("enforces literal membership and constraints", () => { + expect(refusal({ ...valid, mode: "other" })).toMatchObject({ details: { path: "/mode" } }); + expect(refusal({ ...valid, id: "" })).toMatchObject({ details: { path: "/id" } }); + expect(refusal({ ...valid, page: -1 })).toMatchObject({ details: { path: "/page" } }); + expect(refusal({ ...valid, page: 1.5 })).toMatchObject({ details: { path: "/page" } }); + }); + + it("escapes the separator: no two params join to one key", () => { + const Pair = local("pair", { version: 1, params: { a: str(), b: str() }, members: {} }); + const keys = [ + { a: "x|y", b: "z" }, + { a: "x", b: "y|z" }, + { a: "x\\", b: "|z" }, + { a: "x\\|", b: "z" }, + ].map((raw) => checkParams(Pair, raw, BACKGROUND_CONTEXT).key); + expect(new Set(keys).size).toBe(keys.length); + expect(keys[0]).toBe("x\\|y|z"); + }); + + it("counts one compilation per contract, then one validation per get; a failure adds one walk", async () => { + const Fresh = local("fresh", { version: 1, params: { id: str() }, members: {} }); + expect(await counted((context) => checkParams(Fresh, { id: "a" }, context))).toEqual({ + walks: 0, + validations: 1, + compilations: 1, + }); + expect(await counted((context) => checkParams(Fresh, { id: "b" }, context))).toEqual({ + walks: 0, + validations: 1, + compilations: 0, + }); + const failed = await counted((context) => { + expect(() => checkParams(Fresh, { id: 1 }, context)).toThrow(); + }); + expect(failed).toEqual({ walks: 1, validations: 1, compilations: 0 }); + }); + + it("does not retain a dropped contract's params validator", async () => { + const refs = (() => { + const contract = local("gc", { version: 1, params: { id: str() }, members: {} }); + checkParams(contract, { id: "a" }, BACKGROUND_CONTEXT); + const validator = validatorFor(contractData(contract).params, BACKGROUND_CONTEXT); + return [new WeakRef(contract), new WeakRef(validator)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("identityKey", () => { + it("joins name, key and principal key, each escaped", () => { + expect(identityKey(Session, "S1|chat|2|true", '{"kind":"user"}')).toBe( + 'session|S1\\|chat\\|2\\|true|{"kind":"user"}', + ); + }); + + it("local objects take no principal key, by type", () => { + const Info = local("info", { version: 1, params: {}, members: {} }); + expect(identityKey(Info, "")).toBe("info|"); + // @ts-expect-error: local has no principal + identityKey(Info, "", "p"); + // @ts-expect-error: remote needs a principal key + identityKey(Session, "k"); + }); +}); diff --git a/packages/chord/test/next/data/principal.test.ts b/packages/chord/test/next/data/principal.test.ts new file mode 100644 index 000000000..adca8418f --- /dev/null +++ b/packages/chord/test/next/data/principal.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { canonicalJson, principalKey } from "../../../src/next/data/internal.ts"; +import { counted, expectCollected } from "../support.ts"; + +describe("canonicalJson", () => { + it("sorts object keys and writes -0 as 0", () => { + expect(canonicalJson({ b: 1, a: [{ d: -0, c: null }] }, BACKGROUND_CONTEXT)).toBe( + '{"a":[{"c":null,"d":0}],"b":1}', + ); + expect(canonicalJson("x", BACKGROUND_CONTEXT)).toBe('"x"'); + }); + + it("counts one walk", async () => { + expect(await counted((context) => canonicalJson({ a: [1, 2] }, context))).toEqual({ + walks: 1, + validations: 0, + compilations: 0, + }); + }); +}); + +describe("principalKey", () => { + it("defaults to the canonical JSON, whatever the property order", () => { + expect(principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT)).toBe('{"id":"u1","kind":"user"}'); + expect(principalKey({ id: "u1", kind: "user" }, BACKGROUND_CONTEXT)).toBe( + principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT), + ); + }); + + it("uses a custom key function when given", () => { + const byId = (principal: { kind: string; id: string }) => `${principal.kind}:${principal.id}`; + expect(principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT, byId)).toBe("user:u1"); + }); + + it("caches per principal object: no second walk", async () => { + const principal = { kind: "user", id: "u1" }; + const first = await counted((context) => principalKey(principal, context)); + const second = await counted((context) => principalKey(principal, context)); + expect([first.walks, second.walks]).toEqual([1, 0]); + }); + + it("caches custom keys per function", () => { + const principal = { kind: "user", id: "u1" }; + let calls = 0; + const custom = () => `key${++calls}`; + expect([ + principalKey(principal, BACKGROUND_CONTEXT, custom), + principalKey(principal, BACKGROUND_CONTEXT, custom), + ]).toEqual(["key1", "key1"]); + expect(principalKey(principal, BACKGROUND_CONTEXT, () => "other")).toBe("other"); + expect(principalKey(principal, BACKGROUND_CONTEXT)).toBe('{"id":"u1","kind":"user"}'); + }); + + it("never retains a dropped custom key function while its principal lives", async () => { + const principal = { kind: "user", id: "u1" }; + const ref = (() => { + const custom = () => "k"; + principalKey(principal, BACKGROUND_CONTEXT, custom); + return new WeakRef(custom); + })(); + await expectCollected(ref); + expect(principalKey(principal, BACKGROUND_CONTEXT)).toBeTypeOf("string"); + }); + + it("does not retain dropped principals", async () => { + const ref = (() => { + const principal = { kind: "user", id: "gone" }; + principalKey(principal, BACKGROUND_CONTEXT); + principalKey(principal, BACKGROUND_CONTEXT, () => "k"); + return new WeakRef(principal); + })(); + await expectCollected(ref); + }); +}); diff --git a/packages/chord/test/next/data/validator.test.ts b/packages/chord/test/next/data/validator.test.ts new file mode 100644 index 000000000..d1c09ec4c --- /dev/null +++ b/packages/chord/test/next/data/validator.test.ts @@ -0,0 +1,110 @@ +// Tests of compiled checks: pi-schema validators of Chord-owned schemas and the `invalid` error of a failed check. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bytes, int, map, obj, str } from "../../../src/next/contracts/schema.ts"; +import { snapshotArguments } from "../../../src/next/contracts/snapshot.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { check, validatorFor } from "../../../src/next/data/validator.ts"; +import { counted, expectCollected } from "../support.ts"; + +const args = snapshotArguments( + obj({ text: str(), items: arr(obj({ n: int() })), tags: map(str()), blob: bytes({ max: 2 }) }), + "m.args", +); + +const valid = { text: "t", items: [{ n: 1 }], tags: {}, blob: new Uint8Array(1) }; + +/** The `{ message, details }` of the `invalid/args` error a failed check throws. */ +function failure(value: unknown): unknown { + try { + check(validatorFor(args, BACKGROUND_CONTEXT), value, "args", BACKGROUND_CONTEXT); + } catch (error) { + if (isChordError(error, "invalid")) + return { reason: error.reason, message: error.message, details: error.details }; + throw error; + } + return "passed"; +} + +describe("failed checks", () => { + it("pass a valid value", () => { + expect(failure(valid)).toBe("passed"); + }); + + it("report the first error with its JSON Pointer path", () => { + expect(failure({ ...valid, text: 1 })).toEqual({ + reason: "args", + message: "args/text: expected string, got integer", + details: { path: "/text" }, + }); + expect(failure({ ...valid, items: [{ n: "x" }] })).toMatchObject({ details: { path: "/items/0/n" } }); + expect(failure(null)).toEqual({ + reason: "args", + message: "args: expected object, got null", + details: { path: "/" }, + }); + }); + + it("report a missing field at its own path", () => { + const { text: _, ...missing } = valid; + expect(failure(missing)).toMatchObject({ + message: "args/text: missing", + details: { path: "/text" }, + }); + expect(failure({ ...valid, items: [{}] })).toMatchObject({ details: { path: "/items/0/n" } }); + }); + + it("report an unknown field at its own path, nested too", () => { + expect(failure({ ...valid, extra: 1 })).toEqual({ + reason: "args", + message: "args/extra: unknown field", + details: { path: "/extra" }, + }); + expect(failure({ ...valid, items: [{ n: 1, m: 2 }] })).toMatchObject({ details: { path: "/items/0/m" } }); + }); + + it("check bytes by instanceof and length", () => { + expect(failure({ ...valid, blob: [1] })).toMatchObject({ message: "args/blob: expected bytes, got array" }); + expect(failure({ ...valid, blob: new Uint8Array(3) })).toMatchObject({ + message: "args/blob: expected at most 2 bytes", + details: { path: "/blob" }, + }); + }); + + it("treat `format` as an annotation: no format is checked", () => { + const owner = snapshotArguments(obj({ mail: { type: "string", format: "email" } }), "m"); + const validator = validatorFor(owner, BACKGROUND_CONTEXT); + expect(validator.check({ mail: "a@b.c" })).toBe(true); + expect(validator.check({ mail: "nope" })).toBe(true); + }); + + it("escape `~` and `/` in keys (JSON Pointer), so a path names one place", () => { + expect(failure({ ...valid, tags: { "a/b": 1, "c~d": "x" } })).toMatchObject({ details: { path: "/tags/a~1b" } }); + }); +}); + +describe("validators", () => { + it("compile once per schema owner (counted), then validate (counted); a failure adds one walk", async () => { + const owner = snapshotArguments(obj({ a: str() }), "m"); + const first = await counted((context) => check(validatorFor(owner, context), { a: "x" }, "args", context)); + const second = await counted((context) => check(validatorFor(owner, context), { a: "y" }, "args", context)); + const failed = await counted((context) => { + expect(() => check(validatorFor(owner, context), {}, "args", context)).toThrow(); + }); + expect([first, second, failed]).toEqual([ + { walks: 0, validations: 1, compilations: 1 }, + { walks: 0, validations: 1, compilations: 0 }, + { walks: 1, validations: 1, compilations: 0 }, + ]); + }); + + it("are collected with their schema owner, the check function too", async () => { + const refs = (() => { + const owner = snapshotArguments(obj({ blob: bytes(), n: int() }), "m"); + const validator = validatorFor(owner, BACKGROUND_CONTEXT); + return [new WeakRef(owner), new WeakRef(validator), new WeakRef(validator.check)]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/documents/address-source.test.ts b/packages/chord/test/next/documents/address-source.test.ts new file mode 100644 index 000000000..ccd8788c6 --- /dev/null +++ b/packages/chord/test/next/documents/address-source.test.ts @@ -0,0 +1,111 @@ +// Tests of address sources (slice 6a §11; decision "One retirement mechanism"): the frame a watched document address +// gets for a change, a write over another base, a retirement, a recreation after it in the same batch, and a +// replacement outside a commit. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op, Prepared } from "../../../src/delta/index.ts"; +import { AddressSource } from "../../../src/next/documents/address-source.ts"; +import type { JsonObject } from "../../../src/next/documents/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { changesOf, delivered, expectCollected } from "../support.ts"; + +/** An address whose value the test sets. */ +class Address extends AddressSource { + current: JsonObject = { n: 0 }; + + get value(): JsonObject { + return this.current; + } +} + +/** A prepared write of `value` over `base` with `ops`. */ +function prepared(base: JsonObject, value: JsonObject, ops: readonly Op[]): Prepared { + return { base, value, ops, baseRevision: 0, abort: () => {} }; +} + +/** A watched address on its own batch, and the changes its watch got. */ +async function watched() { + const batch = new Batch(() => {}, undefined); + const address = new Address(batch); + const changes = changesOf(await address.watch(context)); + return { batch, address, changes }; +} + +describe("address frames", () => { + it("a write over the read value carries its ops; over another base, a root replacement", async () => { + const { batch, address, changes } = await watched(); + const first = address.current; + batch.commit(() => { + address.current = { n: 1 }; + address.written(first, prepared(first, address.current, [["s", ["n"], 1]])); + }, context); + await delivered(); + const second = address.current; + batch.commit(() => { + address.current = { n: 2 }; + address.written(second, prepared({ n: 7 }, address.current, [["s", ["n"], 2]])); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: false }, + { kind: "value", value: { n: 2 }, ops: [["r", { n: 2 }]], retired: false }, + ]); + }); + + it("a retirement reads initial() with the flag; a recreation in the same batch is one root replacement, flagged", async () => { + const { batch, address, changes } = await watched(); + batch.commit(() => { + address.current = { n: 0 }; + address.retired(address.current); + }, context); + await delivered(); + batch.commit(() => { + const initial = address.current; + address.retired(initial); + address.current = { n: 5 }; + address.written(initial, prepared(initial, address.current, [["s", ["n"], 5]])); + }, context); + await delivered(); + const recreated = address.current; + batch.commit(() => { + address.current = { n: 6 }; + address.written(recreated, prepared(recreated, address.current, [["s", ["n"], 6]])); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: true }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + // The flag was reset by the frame that carried it. + { kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }, + ]); + }); + + it("a replacement outside a commit (a load, a migration) is a root replacement", async () => { + const { batch, address, changes } = await watched(); + batch.commit(() => { + address.current = { n: 3 }; + address.replaced(); + }, context); + await delivered(); + expect(changes).toEqual([{ kind: "value", value: { n: 3 }, ops: [["r", { n: 3 }]], retired: false }]); + }); +}); + +describe("unwatched addresses", () => { + it("keep no ops of a write: no frame takes them", async () => { + const batch = new Batch(() => {}, undefined); + const address = new Address(batch); + let ops: Op[] | undefined = [["s", ["n"], 1]]; + const collected = new WeakRef(ops); + const base = address.current; + batch.commit(() => { + address.current = { n: 1 }; + address.written(base, prepared(base, address.current, ops!)); + }, context); + ops = undefined; + await expectCollected(collected); + // The address itself is still here. + expect(address.value).toEqual({ n: 1 }); + }); +}); diff --git a/packages/chord/test/next/documents/definition.test.ts b/packages/chord/test/next/documents/definition.test.ts new file mode 100644 index 000000000..9efbb1224 --- /dev/null +++ b/packages/chord/test/next/documents/definition.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type StateDefinition } from "../../../src/next/documents/index.ts"; +import { metered } from "../support.ts"; + +describe("defineState", () => { + it("types the value and whether it is keyed", () => { + const Notes = defineState("notes", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), + }); + const Drafts = defineState("drafts", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + }); + expectTypeOf(Notes).toEqualTypeOf>(); + expectTypeOf(Drafts).toEqualTypeOf>(); + expect([Notes.keyed, Drafts.keyed]).toEqual([false, true]); + }); + + it('takes a restricted name only: lowercase [a-z0-9@/._~-] (decision "Restricted names")', () => { + const options = { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }; + for (const name of ["@app/notes.v_1~x-y", "n"]) expect(defineState(name, options).name).toBe(name); + for (const name of ["", "Notes", "app:notes", "app notes", "a\0b", "a\uD800", "é"]) { + expect(() => defineState(name, options)).toThrow( + `A state name ${JSON.stringify(name)}: expected lowercase [a-z0-9@/._~-]`, + ); + } + // Plugin code, not type-checked. + expect(() => defineState(42 as never, options)).toThrow("A state name 42"); + }); +}); + +describe("initial() (R8)", () => { + it("is checked against the schema once, the first time it is used", async () => { + const telemetry = await metered(); + const initial = vi.fn(() => ({ n: 0 })); + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial }); + expect(Doc.initial(telemetry.context)).toEqual({ n: 0 }); + expect(Doc.initial(telemetry.context)).toEqual({ n: 0 }); + expect(initial).toHaveBeenCalledTimes(2); + expect(telemetry.counts()).toEqual({ "chord.data.compilations": 1, "chord.data.validations": 1 }); + }); + + it("refuses a value that does not match, every time until one does", () => { + let value: unknown = { n: "x" }; + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => value as { n: number } }); + expect(() => Doc.initial(context)).toThrow("doc: initial() does not match the schema"); + expect(() => Doc.initial(context)).toThrow("doc: initial() does not match the schema"); + value = { n: 1 }; + expect(Doc.initial(context)).toEqual({ n: 1 }); + }); +}); + +describe("fingerprints", () => { + it("are computed once from the schema's structure: descriptions ignored, a structural change seen", () => { + const make = (schema: Parameters[1]["schema"]) => + defineState("doc", { schema, version: 1, initial: () => ({ n: 0 }) as never }).fingerprint; + const plain = make(obj({ n: int() })); + expect(plain).toMatch(/^[0-9a-f]{16}$/); + expect(make(obj({ n: { ...int(), description: "a count" } }))).toBe(plain); + expect(make(obj({ n: str() }))).not.toBe(plain); + }); +}); + +describe("migrate (R8)", () => { + it("maps an older value and checks the result against the schema each time it produces one", async () => { + const telemetry = await metered(); + const Doc = defineState("doc", { + schema: obj({ n: int() }), + version: 2, + initial: () => ({ n: 0 }), + migrate: (old, from) => ({ n: (old.count as number) * from }), + }); + expect(Doc.migrate({ count: 3 }, 1, telemetry.context)).toEqual({ n: 3 }); + expect(() => Doc.migrate({ count: "x" }, 1, telemetry.context)).toThrow( + "doc: migrate() from version 1 does not match the schema", + ); + expect(telemetry.counts()["chord.data.validations"]).toBe(2); + }); + + it("refuses an older value without a migrate", () => { + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 2, initial: () => ({ n: 0 }) }); + expect(() => Doc.migrate({ n: 1 }, 1, context)).toThrow("doc: no migrate from version 1 to 2"); + }); +}); diff --git a/packages/chord/test/next/documents/documents.test.ts b/packages/chord/test/next/documents/documents.test.ts new file mode 100644 index 000000000..acf9252da --- /dev/null +++ b/packages/chord/test/next/documents/documents.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { delivered, framesOf, loaded, rejection } from "../support.ts"; + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Drafts = defineState("drafts", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, +}); +const Owner = local("app.owner", { + version: 1, + params: { id: int() }, + members: { list: obj({ items: arr(str()) }), touch: method() }, +}); + +/** Serves `Owner` and hands out its handles. */ +async function owner() { + const handles: { list?: SingletonHandle<{ items: string[] }>; drafts?: KeyedHandle<{ body: string }> } = {}; + const { host } = await loaded((h) => + h.serve(Owner, (_, self) => { + handles.list = self.state(List); + handles.drafts = self.state(Drafts); + return { list: handles.list, touch: async () => null }; + }), + ); + const reference = await host.get(Owner, { id: 1 }, context); + return { host, reference, list: handles.list!, drafts: handles.drafts! }; +} + +/** The keys a keyed handle lists. */ +async function keysOf(handle: KeyedHandle<{ body: string }>): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; +} + +describe("singleton documents", () => { + it("read initial() until written and again after a retire, whose frame replaces the root", async () => { + const { list } = await owner(); + const initial = list.value; + expect(initial).toEqual({ items: [] }); + const frames = framesOf(await list.watch(context)); + await list.change((draft) => draft.items.push("a"), context); + expect(list.value).toEqual({ items: ["a"] }); + await list.retire(context); + expect(list.value).toBe(initial); + await list.retire(context); + await delivered(); + expect(frames).toEqual([ + [{ items: ["a"] }, [["p", ["items"], 0, 0, ["a"]]]], + [initial, [["r", initial]]], + ]); + }); + + it("throw not_ready when read in the factory", async () => { + const { host } = await loaded((h) => + h.serve(Owner, (_, self) => { + const list = self.state(List); + void list.value; + return { list, touch: async () => null }; + }), + ); + await expect(host.get(Owner, { id: 1 }, context)).rejects.toThrow("list: read before activation"); + }); +}); + +describe("keyed documents", () => { + it("are absent until written; get, keys and retire by key", async () => { + const { drafts } = await owner(); + expect(await drafts.get("a", context)).toBeUndefined(); + await drafts.change( + "a", + (draft) => { + draft.body = "x"; + }, + context, + ); + await drafts.change( + "b", + (draft) => { + draft.body = "y"; + }, + context, + ); + await drafts.change("c", () => {}, context); + expect(await drafts.get("a", context)).toEqual({ body: "x" }); + expect(await keysOf(drafts)).toEqual(["a", "b"]); + await drafts.retire("a", context); + await drafts.retire("missing", context); + expect(await drafts.get("a", context)).toBeUndefined(); + expect(await keysOf(drafts)).toEqual(["b"]); + }); +}); + +describe("when the owner ends", () => { + it("documents are dropped: every use throws released and watches end", async () => { + const { reference, list, drafts } = await owner(); + const watch = await list.watch(context); + reference.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const released = { code: "released", reason: "ended", message: "app.owner(1): ended" }; + expect(() => list.value).toThrow("app.owner(1): ended"); + expect(await rejection(drafts.get("a", context))).toEqual(released); + expect(await rejection(keysOf(drafts))).toEqual(released); + expect(await rejection(list.change(() => {}, context))).toEqual(released); + }); +}); diff --git a/packages/chord/test/next/documents/fence.test.ts b/packages/chord/test/next/documents/fence.test.ts new file mode 100644 index 000000000..3bb649e38 --- /dev/null +++ b/packages/chord/test/next/documents/fence.test.ts @@ -0,0 +1,14 @@ +// Seal tokens (slice 6a §3, decision "The seal fence"): open until sealed, sealed for good. + +import { describe, expect, it } from "vitest"; +import { SealToken } from "../../../src/next/documents/internal.ts"; + +describe("seal tokens", () => { + it("start open and stay sealed once sealed", () => { + const token = new SealToken(); + expect(token.sealed).toBe(false); + token.seal(); + token.seal(); + expect(token.sealed).toBe(true); + }); +}); diff --git a/packages/chord/test/next/documents/handles.test.ts b/packages/chord/test/next/documents/handles.test.ts new file mode 100644 index 000000000..a574cb51a --- /dev/null +++ b/packages/chord/test/next/documents/handles.test.ts @@ -0,0 +1,75 @@ +// Document handles (slice 6a §4): keyed handles' reads, watches and `at(key)`, and sealed handles refusing every use. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { instancesOf, rejection, restartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Body = defineState("body", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); +const Notes = defineState("notes", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "" }), + keyed: true, +}); + +/** The keys a keyed handle lists. */ +async function keysOf(handle: KeyedHandle<{ text: string }>): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; +} + +describe("keyed handles", () => { + it("list keys in code point order, not UTF-16 order", async () => { + let notes: KeyedHandle<{ text: string }> | undefined; + await restartable(objects, (_, self) => { + notes = self.state(Notes); + }); + // U+FF5E sorts before U+1F600 by code point; as UTF-16 its unit 0xFF5E sorts after the surrogate 0xD83D. + for (const key of ["\u{1F600}", "\uFF5E", "b", "a"]) + await notes!.change( + key, + (d) => { + d.text = key; + }, + context, + ); + expect(await keysOf(notes!)).toEqual(["a", "b", "\uFF5E", "\u{1F600}"]); + expect(await notes!.get("\uFF5E", context)).toEqual({ text: "\uFF5E" }); + }); +}); + +describe("sealed handles", () => { + it("still read synchronously and from memory; changes and retirements are refused released", async () => { + let body: SingletonHandle<{ text: string }> | undefined; + let notes: KeyedHandle<{ text: string }> | undefined; + const { fail } = await restartable(objects, (_, self) => { + body = self.state(Body); + notes = self.state(Notes); + }); + await body!.change((d) => { + d.text = "x"; + }, context); + await notes!.change( + "k", + (d) => { + d.text = "y"; + }, + context, + ); + fail(); + expect(body!.value).toEqual({ text: "x" }); + expect(await notes!.get("k", context)).toEqual({ text: "y" }); + expect(await keysOf(notes!)).toEqual(["k"]); + const released = { code: "released", reason: "ended", message: "app.restartable(1): ended" }; + expect(await rejection(body!.change(() => {}, context))).toEqual(released); + expect(await rejection(body!.retire(context))).toEqual(released); + expect(await rejection(notes!.change("k", () => {}, context))).toEqual(released); + expect(await rejection(notes!.retire("k", context))).toEqual(released); + }); +}); diff --git a/packages/chord/test/next/documents/keyed-slot.test.ts b/packages/chord/test/next/documents/keyed-slot.test.ts new file mode 100644 index 000000000..fbc3c2a3e --- /dev/null +++ b/packages/chord/test/next/documents/keyed-slot.test.ts @@ -0,0 +1,794 @@ +// Keyed slots (slice 6a §4, §10, §11): keyed reads with a context, per-key watches and `at(key)` sources with their +// pins, persisted keyed documents loaded on demand on the object's line (one load per key, shared), and their +// eviction. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { keyed, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { RETENTION_MS } from "../../../src/next/documents/residency.ts"; +import type { StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import type { Watchable, WatchHandle } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + fakeTimers, + flushed, + holdNextCommit, + instancesOf, + loaded, + memoryStore, + rejection, + restartable, + storeRestartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +type Note = { title: string }; + +const Memo = defineState("notes.memo", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + keyed: true, +}); + +const Note = defineState("notes.note", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +const stores: ChordStore[] = []; + +afterEach(async () => { + for (const store of stores.splice(0)) await store.close(context); +}); + +/** A fresh `:memory:` store, closed after the test. */ +async function freshStore(): Promise { + const store = await memoryStore({ outbox: true }); + stores.push(store); + return store; +} + +/** + * `StoreRestartable` over `store` declaring `Note` (and what `declare` adds) per attempt; `notes()` is the latest + * attempt's handle. + */ +async function persisted( + store: ChordStore, + declare: (attempt: number, self: StoreSelf, notes: KeyedHandle) => void = () => {}, +): Promise> & { readonly notes: () => KeyedHandle }> { + let notes: KeyedHandle | undefined; + const restartable = await storeRestartable( + objects, + (attempt, self) => { + notes = self.state(Note); + declare(attempt, self, notes); + }, + { store }, + ); + return { ...restartable, notes: () => notes! }; +} + +/** Writes `title` at `key` in `store`'s `Note` documents through a host of its own. */ +async function seed(store: ChordStore, entries: Record): Promise { + const { host, notes } = await persisted(store); + for (const [key, title] of Object.entries(entries)) { + await notes().change( + key, + (draft) => { + draft.title = title; + }, + context, + ); + } + await host.close(context); +} + +describe("keyed reads in memory", () => { + it("get and keys take a context; keys come in code point order", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + for (const key of ["b", "\u{1F600}", "a", "\uFFFF"]) + await memos!.change( + key, + (d) => { + d.title = key; + }, + context, + ); + expect(await memos!.get("a", context)).toEqual({ title: "a" }); + expect(await memos!.get("z", context)).toBeUndefined(); + const keys: string[] = []; + for await (const key of memos!.keys(context)) keys.push(key); + expect(keys).toEqual(["a", "b", "\uFFFF", "\u{1F600}"]); + await host.close(context); + }); +}); + +describe("migration on access", () => { + it("a newer version migrates each in-memory key when it is read; a key whose migrate fails fails the owner then", async () => { + const migrated: string[] = []; + const MemoV2 = defineState("notes.memo", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + keyed: true, + migrate: (old) => { + const title = old.title as string; + if (title === "bad") throw new Error("cannot migrate"); + migrated.push(title); + return { title: `${title}!` }; + }, + }); + const memos: KeyedHandle[] = []; + const { attempts, restart, errors, reference } = await restartable(objects, (attempt, self) => { + memos.push(self.state(attempt === 1 ? Memo : MemoV2)); + }); + for (const key of ["a", "b", "c"]) { + await memos[0]!.change( + key, + (draft) => { + draft.title = key === "c" ? "bad" : key; + }, + context, + ); + } + await restart(); + // The activation migrated nothing: no key was read. + expect(migrated).toEqual([]); + expect(await memos[1]!.get("a", context)).toEqual({ title: "a!" }); + expect(migrated).toEqual(["a"]); + await attempts[1]!.tx(async (tx) => { + expect(await tx.draft(memos[1]!, "b")).toEqual({ title: "b!" }); + }, context); + expect(migrated).toEqual(["a", "b"]); + expect(await rejection(memos[1]!.get("c", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + }); +}); + +describe("keys and the text rules", () => { + /** Every key of `handle`, in order. */ + async function keysOf(handle: KeyedHandle): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; + } + + const title = (value: string) => (draft: Note) => { + draft.title = value; + }; + + it("a lone surrogate in a key is U+FFFD at every entry point, in memory and in the store; NUL is refused", async () => { + const store = await freshStore(); + const { host, notes, attempts } = await persisted(store); + await notes().change("x\uD800", title("one"), context); + expect(await keysOf(notes())).toEqual(["x\uFFFD"]); + // One key: the change through either spelling reaches the same document and record. + await notes().change("x\uFFFD", title("two"), context); + await attempts[0]!.tx(async (tx) => { + (await tx.draft(notes(), "x\uD800")).title += "!"; + }, context); + expect(await notes().get("x\uD800", context)).toEqual({ title: "two!" }); + expect((await notes().watch("x\uD800", context))?.value).toEqual({ title: "two!" }); + for (const use of [ + () => notes().change("a\0b", title("nul"), context), + () => notes().get("a\0b", context), + () => notes().watch("a\0b", context), + () => notes().retire("a\0b", context), + ]) { + expect(await rejection(use())).toMatchObject({ code: "invalid", reason: "args" }); + } + await notes().retire("x\uD800", context); + expect(await keysOf(notes())).toEqual([]); + await host.close(context); + // The store has the key as memory had it. + const again = await persisted(store); + expect(await again.notes().get("x\uFFFD", context)).toBeUndefined(); + await again.host.close(context); + }); + + it("in memory too: one key per spelling, NUL refused", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + await memos!.change("x\uDC00", title("one"), context); + expect(await memos!.get("x\uFFFD", context)).toEqual({ title: "one" }); + expect(await keysOf(memos!)).toEqual(["x\uFFFD"]); + expect(await rejection(memos!.change("\0", title("nul"), context))).toMatchObject({ code: "invalid" }); + await host.close(context); + }); + + it("the empty key is a key: listed first, also on a later page", async () => { + const store = await freshStore(); + const entries: Record = { "": "empty" }; + for (let i = 0; i < 120; i++) entries[`k${String(i).padStart(3, "0")}`] = "x"; + await seed(store, entries); + const { host, notes } = await persisted(store); + const keys = await keysOf(notes()); + expect(keys.length).toBe(121); + expect(keys[0]).toBe(""); + expect(await notes().get("", context)).toEqual({ title: "empty" }); + await host.close(context); + }); +}); + +describe("keyed watches", () => { + it("watch(key) is undefined when absent; it follows the address through a retirement and a recreation", async () => { + let memos: KeyedHandle | undefined; + const { host, attempts } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + expect(await memos!.watch("a", context)).toBeUndefined(); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + const watch = (await memos!.watch("a", context))!; + expect(watch.value).toEqual({ title: "x" }); + const changes = changesOf(watch); + await memos!.change( + "a", + (d) => { + d.title = "y"; + }, + context, + ); + await delivered(); + await memos!.retire("a", context); + await delivered(); + await memos!.change( + "a", + (d) => { + d.title = "z"; + }, + context, + ); + await delivered(); + await attempts[0]!.tx(async (tx) => { + await tx.retire(memos!, "a"); + (await tx.draft(memos!, "a")).title = "w"; + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { title: "y" }, ops: [["s", ["title"], "y"]], retired: false }, + { kind: "value", value: { title: "" }, ops: [["r", { title: "" }]], retired: true }, + { kind: "value", value: { title: "z" }, ops: [["r", { title: "z" }]], retired: false }, + { kind: "value", value: { title: "w" }, ops: [["r", { title: "w" }]], retired: true }, + ]); + await host.close(context); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("a stopped watch unpins its key: its source is collected", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + let watch: WatchHandle | undefined = (await memos!.watch("a", context))!; + // The watch's source: the slot's per-key source while pinned. + const slot = (memos as unknown as { slot: { source(key: string): object } }).slot; + const source = new WeakRef(slot.source("a")); + await watch.stop(); + watch = undefined; + await flushed(); + await expectCollected(source); + await host.close(context); + }); +}); + +describe("at(key)", () => { + it("is initial() while absent, a state member, derive and on source; retirement arrives with the flag", async () => { + let memos: KeyedHandle | undefined; + let pinned: Watchable | undefined; + let upper: Watchable | undefined; + const seen: { title: string; retired: boolean }[] = []; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + pinned = memos.at("a"); + upper = self.derive([pinned], ([note]) => note.title.toUpperCase()); + self.on(pinned, async ({ value, retired }) => void seen.push({ title: value.title, retired })); + }); + expect(pinned!.value).toEqual({ title: "" }); + const changes = changesOf(await upper!.watch(context)); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + await delivered(); + expect(pinned!.value).toEqual({ title: "x" }); + await memos!.retire("a", context); + await delivered(); + await flushed(); + expect(changes).toEqual([ + { kind: "value", value: "X", ops: [["r", "X"]], retired: false }, + { kind: "value", value: "", ops: [["r", ""]], retired: true }, + ]); + expect(seen).toEqual([ + { title: "", retired: false }, + { title: "x", retired: false }, + { title: "", retired: true }, + ]); + await host.close(context); + }); + + it("only in the factory", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + expect(() => memos!.at("a")).toThrow("valid only in the factory"); + await host.close(context); + }); + + it("a persisted key is loaded before the object goes live and stays pinned until the next factory returns", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + let pinned: Watchable | undefined; + const { host, restart, notes } = await persisted(store, (attempt, _, notes) => { + pinned = notes.at(attempt === 1 ? "a" : "b"); + }); + expect(pinned!.value).toEqual({ title: "x" }); + const slot = (notes() as unknown as { slot: { source(key: string): object | undefined } }).slot; + const first = new WeakRef(slot.source("a")!); + await restart(); + expect(pinned!.value).toEqual({ title: "y" }); + expect(slot.source("a")).toBeUndefined(); + await expectCollected(first); + await host.close(context); + }); +}); + +describe("cold keyed loads on the line", () => { + it("load a cold key once; a cached get touches no SQLite; an absent key is not cached", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + const read = vi.spyOn(store, "read"); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + expect(await notes().get("z", context)).toBeUndefined(); + expect(await notes().get("z", context)).toBeUndefined(); + expect(read).toHaveBeenCalledTimes(3); + await host.close(context); + }); + + it("two readers share one load; a cancelled waiter rejects alone", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts } = await persisted(store); + const read = vi.spyOn(store, "read"); + // Hold the line so that the load waits behind it. + const { promise: holding, resolve: release } = Promise.withResolvers(); + const tx = attempts[0]!.tx(() => holding, context); + const cancel = new AbortController(); + const first = notes().get("a", { ...context, abortSignal: cancel.signal }); + const second = notes().get("a", context); + cancel.abort(new Error("gone")); + await expect(first).rejects.toThrow("gone"); + release(); + await tx; + expect(await second).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + await host.close(context); + }); + + it("a cold get racing a draft of the same key: the load installs only while the key is absent", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts } = await persisted(store); + let get: Promise | undefined; + await attempts[0]!.tx(async (tx) => { + get = notes().get("a", context); + const draft = await tx.draft(notes(), "a"); + expect(draft.title).toBe("x"); + draft.title = "y"; + }, context); + expect(await get).toEqual({ title: "y" }); + expect(await notes().get("a", context)).toEqual({ title: "y" }); + await host.close(context); + }); + + it("keys come from the store in code point order, a page per line job", async () => { + const store = await freshStore(); + const entries: Record = {}; + for (let i = 0; i < 150; i++) entries[`k${String(i).padStart(3, "0")}`] = "x"; + entries["\u{1F600}"] = "x"; + entries["\uFFFF"] = "x"; + await seed(store, entries); + const { host, notes } = await persisted(store); + const keys: string[] = []; + for await (const key of notes().keys(context)) keys.push(key); + expect(keys).toEqual([...Object.keys(entries).filter((key) => key.startsWith("k")), "\uFFFF", "\u{1F600}"]); + await host.close(context); + }); + + it("tx.retire of a cold key finds it without loading it; an absent key retires nothing", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + const seq = store.lastSeq; + await notes().retire("z", context); + expect(store.lastSeq).toBe(seq); + await notes().retire("a", context); + expect(store.lastSeq).toBe(seq + 1); + expect(await notes().get("a", context)).toBeUndefined(); + await host.close(context); + }); + + it.each([ + ["a newer version", 2, { title: str(), n: str() }, "stored at version 2, newer than the code's 1"], + ["another fingerprint at its version", 1, { title: str(), n: str() }, "stored with another schema at version 1"], + ])( + "tx.retire of a cold key stored at %s fails the owner broken as a read would and keeps it (R15, audit)", + async (_, version, schema, message) => { + const store = await freshStore(); + // Written by other code: a newer version, or the same version with another schema. + const Other = defineState("notes.note", { + schema: obj(schema), + version, + initial: () => ({ title: "", n: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const writer = await storeRestartable( + objects, + (_, self) => { + void self + .state(Other) + .change("a", (draft) => void Object.assign(draft, { title: "kept", n: "1" }), context); + }, + { store }, + ); + await flushed(); + await writer.host.close(context); + const { host, notes, errors, reference } = await persisted(store); + expect(await rejection(notes().retire("a", context))).toMatchObject({ + message: expect.stringContaining(message), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + const reader = await storeRestartable(objects, (_, self) => void self.state(Other), { store }); + expect(await store.read((executor) => executor.all("SELECT key FROM chord_documents"), context)).toEqual([ + { key: "a" }, + ]); + await reader.host.close(context); + }, + ); + + it("a retirement cut by its attempt's seal while it finds the document fails no successor (audit)", async () => { + const store = await freshStore(); + const Newer = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const writer = await storeRestartable( + objects, + (_, self) => + void self.state(Newer).change( + "a", + (draft) => { + draft.title = "kept"; + }, + context, + ), + { store }, + ); + await flushed(); + await writer.host.close(context); + const { host, notes, errors, reference, fail, retry } = await persisted(store); + // The find's read is held past the seal: the retirement resumes on a cut transaction. + const read = store.read.bind(store); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + store.read = async (fn, ctx) => { + store.read = read; + const value = await read(fn, ctx); + entered.resolve(); + await gate.promise; + return value; + }; + const retired = rejection(notes().retire("a", context)); + await entered.promise; + fail(); + await retry(); + gate.resolve(); + expect(await retired).toMatchObject({ code: "released" }); + await flushed(); + expect(reference.state.value).toEqual({ kind: "live" }); + expect(errors.filter(({ kind }) => kind === "object_failed")).toHaveLength(1); + await host.close(context); + }); + + it("a sealed handle reads cached documents; a cold read is refused released", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + const { host, notes, restart } = await persisted(store); + const old = notes(); + await old.get("a", context); + await restart(); + expect(await old.get("a", context)).toEqual({ title: "x" }); + expect(await rejection(old.get("b", context))).toMatchObject({ code: "released" }); + await host.close(context); + }); + + it("a sealed handle joining a cold load in flight is refused released (rule 4)", async () => { + const store = await freshStore(); + await seed(store, { b: "y" }); + const { host, notes, restart } = await persisted(store); + const gate = holdNextRead(store); + const old = notes(); + const first = old.get("b", context); + await gate.reached; + const restarting = restart(); + await delivered(); + expect(await rejection(old.get("b", context))).toMatchObject({ code: "released" }); + gate.open(); + expect(await first).toEqual({ title: "y" }); + await restarting; + await host.close(context); + }); + + it("cold reads queued on the line before the object's end are refused released when they run after it", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y", c: "z" }); + const { host, notes } = await persisted(store); + const gate = holdNextRead(store); + const first = notes().get("a", context); + await gate.reached; + const get = rejection(notes().get("b", context)); + const watch = rejection(notes().watch("c", context)); + const closing = host.close(context); + gate.open(); + expect(await get).toMatchObject({ code: "released" }); + expect(await watch).toMatchObject({ code: "released" }); + // The load holding the line before the end finishes. + expect(await first).toEqual({ title: "x" }); + await closing; + }); + + it("cold reads queued behind an admitted step are refused released when their attempt failed live meanwhile", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const P = defineState("notes.p", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + persist: true, + checkpointWhen: () => false, + }); + let p: SingletonHandle | undefined; + const { host, notes, attempts } = await persisted(store, (_, self) => { + p = self.state(P); + }); + const held = holdNextCommit(store); + const change = p!.change((draft) => { + draft.title = "p"; + }, context); + await held.entered; + const get = rejection(notes().get("a", context)); + const watch = rejection(notes().watch("a", context)); + const keys = rejection( + (async () => { + for await (const _ of notes().keys(context)); + })(), + ); + // The attempt is sealed; the object stays open. + failLive(objects, attempts[0]!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + expect(await get).toMatchObject({ code: "released" }); + expect(await watch).toMatchObject({ code: "released" }); + expect(await keys).toMatchObject({ code: "released" }); + await host.close(context); + }); + + it("a still-valid waiter sharing that load gets the document: only the sealed handle's read is refused", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const Notes = local("app.store", { version: 1, params: {}, members: { notes: keyed(obj({ title: str() })) } }); + let self: StoreSelf | undefined; + let notes: KeyedHandle | undefined; + const { host } = await loaded( + (h) => + h.serve(Notes, (_, s) => { + self = s; + notes = s.state(Note); + return { notes }; + }), + { store }, + ); + const ref = await host.get(Notes, {}, context); + const held = holdNextCommit(store); + const change = notes!.change( + "b", + (draft) => { + draft.title = "y"; + }, + context, + ); + await held.entered; + const sealed = rejection(notes!.get("a", context)); + // A consumer reads as the provider's own code may (the object's own token), joining the same load. + const shared = ref.notes.get("a", context); + await flushed(); + failLive(objects, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + expect(await sealed).toMatchObject({ code: "released" }); + expect(await shared).toEqual({ title: "x" }); + await host.close(context); + }); + + it("a load cut by the seal never installs: a successor's retirement stands", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts, restart } = await persisted(store); + const gate = holdNextRead(store); + const old = notes(); + const cut = attempts[0]!.tx(async (tx) => { + (await tx.draft(old, "a")).title = "z"; + }, context); + await gate.reached; + // The seal cuts the transaction and releases the line; the successor retires the key. + await restart(); + await notes().retire("a", context); + gate.open(); + expect(await rejection(cut)).toMatchObject({ code: "released" }); + expect(await notes().get("a", context)).toBeUndefined(); + await host.close(context); + }); + + it("a plugin migrate throwing a ChordError fails its owner like any other error", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const V2 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + migrate: () => { + throw new ChordError({ code: "unavailable", message: "plugin says no" }); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + let notes: KeyedHandle | undefined; + const { host, errors, reference } = await storeRestartable( + objects, + (_, self) => { + notes = self.state(V2); + }, + { store }, + ); + expect(await rejection(notes!.get("a", context))).toMatchObject({ message: "plugin says no" }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + }); + + it("a key whose migrate fails fails its owner broken; the get rejects with the cause", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const V2 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + migrate: () => { + throw new Error("cannot migrate"); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + let notes: KeyedHandle | undefined; + const { host, errors, reference } = await storeRestartable( + objects, + (_, self) => { + notes = self.state(V2); + }, + { store }, + ); + expect(await rejection(notes!.get("a", context))).toMatchObject({ + code: "unavailable", + message: expect.stringContaining("cannot migrate"), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + }); +}); + +describe("eviction", () => { + fakeTimers(); + + it("an unpinned key idle for the retention leaves memory; a pinned one stays", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + const { host, notes } = await persisted(store); + const read = vi.spyOn(store, "read"); + await notes().get("a", context); + const watch = (await notes().watch("b", context))!; + expect(read).toHaveBeenCalledTimes(2); + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await notes().get("a", context); + await notes().get("b", context); + expect(read).toHaveBeenCalledTimes(3); + await watch.stop(); + await flushed(); + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await notes().get("b", context); + expect(read).toHaveBeenCalledTimes(4); + await host.close(context); + }); + + it("an evicted document is collected", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + let value: Note | undefined = await notes().get("a", context); + const collected = new WeakRef(value!); + value = undefined; + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await expectCollected(collected); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + await host.close(context); + }); +}); + +/** Holds `store`'s next read after the backend returned its result, until `open()`. */ +function holdNextRead(store: ChordStore): { readonly reached: Promise; open(): void } { + const reached = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const read = store.read.bind(store); + store.read = async (fn, ctx) => { + store.read = read; + const result = await read(fn, ctx); + reached.resolve(); + await gate.promise; + return result; + }; + return { reached: reached.promise, open: () => gate.resolve() }; +} diff --git a/packages/chord/test/next/documents/line.test.ts b/packages/chord/test/next/documents/line.test.ts new file mode 100644 index 000000000..b473cc378 --- /dev/null +++ b/packages/chord/test/next/documents/line.test.ts @@ -0,0 +1,45 @@ +// The line (§9): holders run one at a time in the order they asked; a release given twice changes nothing. + +import { describe, expect, it } from "vitest"; +import { Line } from "../../../src/next/documents/line.ts"; +import { flushed } from "../support.ts"; + +describe("the line", () => { + it("hands the line to each holder in the order they asked, once the previous one released", async () => { + const line = new Line(); + const order: string[] = []; + const first = await line.acquire(); + const second = line.acquire().then((release) => { + order.push("second"); + return release; + }); + const third = line.acquire().then((release) => { + order.push("third"); + return release; + }); + await flushed(); + expect(order).toEqual([]); + first(); + (await second)(); + await third; + expect(order).toEqual(["second", "third"]); + }); + + it("ignores a second release of the same holder", async () => { + const line = new Line(); + const first = await line.acquire(); + const second = line.acquire(); + let thirdHeld = false; + const third = line.acquire().then(() => { + thirdHeld = true; + }); + first(); + first(); + await second; + await flushed(); + expect(thirdHeld).toBe(false); + (await second)(); + await third; + expect(thirdHeld).toBe(true); + }); +}); diff --git a/packages/chord/test/next/documents/persistence.test.ts b/packages/chord/test/next/documents/persistence.test.ts new file mode 100644 index 000000000..b4be944bc --- /dev/null +++ b/packages/chord/test/next/documents/persistence.test.ts @@ -0,0 +1,285 @@ +// Persisted documents' store side (slice 6a §8, §10): loads with the version and fingerprint checks and `migrate`, +// finds, keys, and the writes one commit stages (creation, base or delta by `checkpointWhen`, retirement). + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { track } from "../../../src/delta/index.ts"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type JsonObject } from "../../../src/next/documents/index.ts"; +import { Persistence, type Stored, storedAfter } from "../../../src/next/documents/persistence.ts"; +import type { AnyDefinition } from "../../../src/next/documents/slot.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { memoryStore, metered } from "../support.ts"; + +const context = BACKGROUND_CONTEXT; + +const Note = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 1, + initial: () => ({ title: "", n: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 2, +}); + +let store: ChordStore; +let notes: Persistence; + +beforeEach(async () => { + store = await memoryStore({ outbox: true }); + notes = new Persistence(store, "app.store", "chord", "notes.note"); +}); + +afterEach(async () => { + await store.close(context); +}); + +/** Stages the change of `before` to `after` at `key` and commits it; resolves with what is stored afterwards. */ +async function change( + key: string, + before: JsonObject | undefined, + after: (draft: JsonObject) => void, + stored: Stored | undefined, + definition: AnyDefinition = Note, +): Promise { + const tracker = track(before ?? definition.initial(context)); + const draft = tracker.beginChange(); + after(draft.state); + const prepared = draft.prepare(); + const writes: StagedWrite[] = []; + const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes); + await store.commit(writes, { admit: () => true, adopt: () => {} }, context); + tracker.adopt(prepared); + return storedAfter(result); +} + +/** The document at `key` read from the store and decoded by `definition`, `undefined` when absent. */ +async function load(key: string, definition: AnyDefinition, at: typeof context) { + const document = await notes.read(key, at); + return document === undefined ? undefined : notes.decode(document, definition, at); +} + +/** The kinds of the outbox's writes, per commit. */ +async function outboxKinds(): Promise { + const commits = await store.outbox!.read(0, 100, context); + return commits.map(({ writes }) => + writes.map((write) => (write.kind === "document.change" ? `change:${write.content.kind}` : write.kind)), + ); +} + +describe("loads", () => { + it("load an absent document as undefined and a stored one with its record", async () => { + expect(await load("a", Note, context)).toBeUndefined(); + const stored = await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + ); + const { counts, context: counting } = await metered(); + expect(await load("a", Note, counting)).toEqual({ + value: { title: "x", n: 0 }, + stored: { id: stored!.id, revision: 1, version: 1, deltasSinceBase: 0 }, + }); + expect(counts()["chord.store.loads"]).toBe(1); + }); + + it("migrate an older stored version and check the result once", async () => { + const V1 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + persist: true, + checkpointWhen: () => false, + }); + await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + V1, + ); + const V2 = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + migrate: (old) => ({ title: String(old.title), n: 7 }), + persist: true, + checkpointWhen: () => false, + }); + const { counts, context: counting } = await metered(); + expect(await load("a", V2, counting)).toMatchObject({ + value: { title: "x", n: 7 }, + stored: { version: 1 }, + }); + expect(counts()["chord.data.validations"]).toBe(1); + const Bad = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + // A migrate returning a wrong shape: the load is refused. + migrate: (old) => ({ title: String(old.title) }) as { title: string; n: number }, + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Bad, context)).rejects.toThrow("migrate() from version 1 does not match"); + }); + + it("refuse a stored version newer than the code's and the same version with another fingerprint (D9)", async () => { + await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + ); + const Old = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 0, + initial: () => ({ title: "", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Old, context)).rejects.toThrow("stored at version 1, newer than the code's 0"); + const Edited = defineState("notes.note", { + schema: obj({ title: str(), n: int(), extra: str() }), + version: 1, + initial: () => ({ title: "", n: 0, extra: "" }), + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Edited, context)).rejects.toThrow("stored with another schema at version 1"); + }); + + it("find a live record without loading it, and list keys in code point order by pages", async () => { + expect(await notes.find("a", context)).toBeUndefined(); + for (const key of ["b", "\u{1F600}", "a", "\uFFFF"]) + await change( + key, + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + expect(await notes.find("a", context)).toMatchObject({ + stored: { revision: 1, version: 1 }, + fingerprint: expect.any(String), + }); + expect(await notes.keys(undefined, 10, context)).toEqual(["a", "b", "\uFFFF", "\u{1F600}"]); + expect(await notes.keys("b", 1, context)).toEqual(["\uFFFF"]); + }); +}); + +describe("staged writes", () => { + it("store a creation as a base, changes as deltas until checkpointWhen asks for a base", async () => { + let stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + expect(stored).toMatchObject({ revision: 1, deltasSinceBase: 0 }); + let value = (await load("a", Note, context))!.value; + for (const n of [2, 3, 4]) { + stored = await change( + "a", + value, + (d) => { + d.n = n; + }, + stored, + ); + value = (await load("a", Note, context))!.value; + } + expect(stored).toMatchObject({ revision: 4, deltasSinceBase: 0 }); + expect(value).toEqual({ title: "", n: 4 }); + expect(await outboxKinds()).toEqual([["document.create"], ["change:delta"], ["change:delta"], ["change:base"]]); + expect((await load("a", Note, context))!.stored.deltasSinceBase).toBe(0); + }); + + it("store a base when the stored version is older than the definition's (the migrated base written back)", async () => { + const stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + const V2 = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + migrate: (old) => old as { title: string; n: number }, + persist: true, + checkpointWhen: () => false, + }); + const after = await change( + "a", + { title: "", n: 1 }, + (d) => { + d.n = 2; + }, + stored, + V2, + ); + expect(after).toMatchObject({ version: 2, revision: 2, deltasSinceBase: 0 }); + expect((await outboxKinds()).at(-1)).toEqual(["change:base"]); + expect(await load("a", V2, context)).toMatchObject({ value: { n: 2 }, stored: { version: 2 } }); + }); + + it("stage a retirement before a recreation, and number creations by their order in the commit", async () => { + const stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + const writes: StagedWrite[] = []; + const fresh = (n: number) => { + const tracker = track(Note.initial(context)); + const draft = tracker.beginChange(); + draft.state.n = n; + return draft.prepare(); + }; + const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes); + const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes); + const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes); + expect([first, again, gone]).toMatchObject([ + { kind: "create", version: 1 }, + { kind: "create", version: 1 }, + { kind: "retire" }, + ]); + let ids: readonly number[] = []; + await store.commit( + writes, + { + admit: () => true, + adopt: (_seq, created) => { + ids = created; + }, + }, + context, + ); + expect(ids).toEqual([stored!.id + 1, stored!.id + 2]); + expect(storedAfter(again)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); + expect((await outboxKinds()).at(-1)).toEqual(["document.retire", "document.create", "document.create"]); + }); + + it("serialize at staging: a value JSON cannot encode fails there, never in the step (rule 8)", () => { + // Drafts refuse such values when placed; a prepared change made by hand shows where the text is made. + const value = { title: "", n: 1n } as unknown as JsonObject; + const prepared = { base: value, value, ops: [], baseRevision: 0, abort: () => {} }; + expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [])).toThrow(TypeError); + }); +}); diff --git a/packages/chord/test/next/documents/residency.test.ts b/packages/chord/test/next/documents/residency.test.ts new file mode 100644 index 000000000..1451cee02 --- /dev/null +++ b/packages/chord/test/next/documents/residency.test.ts @@ -0,0 +1,77 @@ +// Residency (slice 6a §10, rule 23): Durable's idle structure, one timer per object, the granule rule, eviction on +// the line oldest first. + +import { describe, expect, it, vi } from "vitest"; +import { track } from "../../../src/delta/index.ts"; +import { RETENTION_MS, Residency, Resident } from "../../../src/next/documents/residency.ts"; +import { fakeTimers, flushed } from "../support.ts"; + +/** A residency whose sweeps run at once (no line), and the keys it evicted, in order. */ +function idle() { + const evicted: string[] = []; + const slot = { evict: (resident: Resident) => void evicted.push(resident.key) }; + + const residency = new Residency((sweep) => { + sweep(); + }); + const resident = (key: string) => new Resident(slot, key, track({}), undefined, 1); + return { residency, resident, evicted }; +} + +describe("the idle structure", () => { + fakeTimers(); + + it("evict idle documents oldest first once their retention passed, with one timer", async () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + const b = resident("b"); + residency.touch(a); + vi.advanceTimersByTime(5_000); + residency.touch(b); + expect(vi.getTimerCount()).toBe(1); + vi.advanceTimersByTime(RETENTION_MS - 5_000); + await flushed(); + expect(evicted).toEqual(["a"]); + expect(vi.getTimerCount()).toBe(1); + vi.advanceTimersByTime(5_000); + expect(evicted).toEqual(["a", "b"]); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual(["a", "b"]); + }); + + it("move a touched document to the back at most once per granule", () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + const b = resident("b"); + residency.touch(a); + residency.touch(b); + vi.advanceTimersByTime(500); + residency.touch(a); + vi.advanceTimersByTime(RETENTION_MS - 500); + // Within its granule the touch did not move `a`: it leaves up to a granule early. + expect(evicted).toEqual(["a", "b"]); + residency.touch(a); + residency.touch(b); + vi.advanceTimersByTime(1_000); + residency.touch(a); + vi.advanceTimersByTime(RETENTION_MS - 1_000); + expect(evicted).toEqual(["a", "b", "b"]); + vi.advanceTimersByTime(1_000); + expect(evicted).toEqual(["a", "b", "b", "a"]); + }); + + it("forget pinned documents; after end nothing is evicted and no timer is left", () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + residency.touch(a); + residency.forget(a); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual([]); + residency.touch(resident("b")); + residency.end(); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual([]); + }); +}); diff --git a/packages/chord/test/next/documents/slot.test.ts b/packages/chord/test/next/documents/slot.test.ts new file mode 100644 index 000000000..7141fc6ab --- /dev/null +++ b/packages/chord/test/next/documents/slot.test.ts @@ -0,0 +1,375 @@ +// Singleton slots (slice 6a §4, §10, §11): retirement, slots kept across attempts, persisted singletons loaded once. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { + defineState, + type JsonObject, + type KeyedHandle, + type SingletonHandle, +} from "../../../src/next/documents/index.ts"; +import type { HostError, StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { + changesOf, + delivered, + instancesOf, + loaded, + memoryStore, + rejection, + restartable, + StoreRestartable, + storeRestartable, + valuesOf, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const ListV1 = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); + +describe("retirement", () => { + it("reaches watches as initial() with retired: true", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + }); + const initial = list!.value; + const changes = changesOf(await list!.watch(context)); + await list!.change((draft) => void draft.items.push("a"), context); + await list!.retire(context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { items: ["a"] }, ops: [["p", ["items"], 0, 0, ["a"]]], retired: false }, + { kind: "value", value: initial, ops: [["r", initial]], retired: true }, + ]); + }); + + it("a retire and a recreate of a present document in one transaction: one frame, retired, the new value", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const { attempts } = await restartable(objects, (_, self) => { + list = self.state(ListV1); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const changes = changesOf(await list!.watch(context)); + await attempts[0]!.tx(async (tx) => { + await tx.retire(list!); + (await tx.draft(list!)).items.push("b"); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { items: ["b"] }, ops: [["r", { items: ["b"] }]], retired: true }, + ]); + }); +}); + +describe("across attempts", () => { + it("keep in-memory documents across a failure and retry; initial() is taken once", async () => { + let initials = 0; + const List = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => { + initials++; + return { items: [] }; + }, + }); + const handles: SingletonHandle<{ items: string[] }>[] = []; + const { restart } = await restartable(objects, (_, self) => { + handles.push(self.state(List)); + }); + const changes = changesOf(await handles[0]!.watch(context)); + await handles[0]!.change((draft) => void draft.items.push("a"), context); + await restart(); + expect(handles[1]!.value).toEqual({ items: ["a"] }); + await handles[1]!.change((draft) => void draft.items.push("b"), context); + await delivered(); + // The watch taken through the first attempt's handle follows the slot across attempts. + expect(valuesOf(changes)).toEqual([{ items: ["a"] }, { items: ["a", "b"] }]); + expect(initials).toBe(1); + }); + + it("drop the in-memory documents a later factory no longer declares: their watches end", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const { restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) list = self.state(ListV1); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const watch = await list!.watch(context); + await restart(); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("keep them across a throwing factory: only a factory that returns drops what it does not declare", async () => { + const handles: SingletonHandle<{ items: string[] }>[] = []; + const { fail, retry, errors } = await restartable(objects, (attempt, self) => { + if (attempt === 2) throw new Error("factory threw"); + if (attempt !== 2) handles.push(self.state(ListV1)); + }); + await handles[0]!.change((draft) => void draft.items.push("a"), context); + fail(); + await rejection(retry()); + expect(failure(errors)).toBe("app.restartable(1): factory threw"); + await retry(); + expect(handles[1]!.value).toEqual({ items: ["a"] }); + }); + + it("migrate in memory when a later attempt declares a newer version; the old handles are refused", async () => { + let initials = 0; + const ListV2 = defineState("list", { + schema: obj({ entries: arr(obj({ text: str() })) }), + version: 2, + initial: () => { + initials++; + return { entries: [] }; + }, + migrate: (old) => ({ entries: (old.items as string[]).map((text) => ({ text })) }), + }); + const Notes1 = defineState("notes", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + }); + const Notes2 = defineState("notes", { + schema: obj({ n: int(), m: int() }), + version: 2, + initial: () => ({ n: 0, m: 0 }), + keyed: true, + migrate: (old) => ({ n: old.n as number, m: 1 }), + }); + const lists: SingletonHandle[] = []; + const notes: KeyedHandle[] = []; + const inFactory: unknown[] = []; + const { restart } = await restartable(objects, (attempt, self) => { + lists.push(self.state(attempt === 1 ? ListV1 : ListV2) as unknown as SingletonHandle); + notes.push(self.state(attempt === 1 ? Notes1 : Notes2) as unknown as KeyedHandle); + // The new code's handle in its own factory: the values are still at version 1 until the activation. + if (attempt === 2) inFactory.push(rejectionOf(() => lists[1]!.value)); + }); + await lists[0]!.change((draft) => void (draft.items as string[]).push("a"), context); + await notes[0]!.change( + "k", + (draft) => { + draft.n = 5; + }, + context, + ); + const changes = changesOf(await lists[0]!.watch(context)); + await restart(); + expect(lists[1]!.value).toEqual({ entries: [{ text: "a" }] }); + expect(await notes[1]!.get("k", context)).toEqual({ n: 5, m: 1 }); + const stale = "list: a handle of version 1; the documents are at version 2"; + expect(() => lists[0]!.value).toThrow(stale); + expect(await rejection(notes[0]!.get("k", context))).toMatchObject({ code: "released" }); + await delivered(); + expect(changes).toEqual([ + { + kind: "value", + value: { entries: [{ text: "a" }] }, + ops: [["r", { entries: [{ text: "a" }] }]], + retired: false, + }, + ]); + expect(initials).toBe(1); + expect(inFactory).toEqual([expect.objectContaining({ code: "not_ready" })]); + }); + + it("fail the attempt broken for a flipped keyed, an older version or a schema changed without a bump", async () => { + const KeyedList = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => ({ items: [] }), + keyed: true, + }); + const Changed = defineState("list", { + schema: obj({ items: arr(int()) }), + version: 1, + initial: () => ({ items: [] }), + }); + const Newer = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 2, + initial: () => ({ items: [] }), + migrate: (old) => old as { items: string[] }, + }); + const cases = [ + [ListV1, KeyedList, "list: declared keyed, its documents are not"], + [ListV1, Changed, "list: the schema changed without a version bump (version 1)"], + [Newer, ListV1, "list: version 1 is older than its documents' (2)"], + ] as const; + for (const [first, later, message] of cases) { + const { fail, retry, errors } = await restartable(objects, (attempt, self) => { + self.state(attempt === 1 ? first : later); + }); + fail(); + await rejection(retry()); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "activation_failed"]); + expect(failure(errors)).toBe(`app.restartable(1): ${message}`); + } + }); +}); + +describe("persisted singletons", () => { + const Prefs = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 1, + initial: () => ({ theme: "light", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + + /** A store host declaring `definition` per attempt; `prefs()` is the latest attempt's handle. */ + async function prefsIn(store: ChordStore, definition: typeof Prefs = Prefs) { + let prefs: SingletonHandle<{ theme: string; n: number }> | undefined; + const restartable = await storeRestartable( + objects, + (_, self) => { + prefs = self.state(definition); + }, + { store }, + ); + return { ...restartable, prefs: () => prefs! }; + } + + /** The kinds of the outbox's writes, per commit. */ + async function outboxKinds(store: ChordStore): Promise { + const commits = await store.outbox!.read(0, 100, context); + return commits.map(({ writes }) => + writes.map((write) => (write.kind === "document.change" ? `change:${write.content.kind}` : write.kind)), + ); + } + + it("load once per object at activation (not again on a retry), and survive the host", async () => { + const store = await memoryStore({ outbox: true }); + const first = await prefsIn(store); + expect(first.prefs().value).toEqual({ theme: "light", n: 0 }); + await first.prefs().change((draft) => { + draft.theme = "dark"; + }, context); + await first.prefs().change((draft) => { + draft.n = 1; + }, context); + await first.host.close(context); + const read = vi.spyOn(store, "read"); + const second = await prefsIn(store); + expect(second.prefs().value).toEqual({ theme: "dark", n: 1 }); + expect(read).toHaveBeenCalledTimes(1); + await second.restart(); + expect(second.prefs().value).toEqual({ theme: "dark", n: 1 }); + expect(read).toHaveBeenCalledTimes(1); + await second.prefs().retire(context); + expect(second.prefs().value).toEqual({ theme: "light", n: 0 }); + expect(await outboxKinds(store)).toEqual([["document.create"], ["change:delta"], ["document.retire"]]); + await second.host.close(context); + await store.close(context); + }); + + it("migrate a stored older version at load; the next change writes the migrated base back", async () => { + const store = await memoryStore({ outbox: true }); + const old = await prefsIn(store); + await old.prefs().change((draft) => { + draft.n = 1; + }, context); + await old.host.close(context); + const V2 = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 2, + initial: () => ({ theme: "light", n: 0 }), + migrate: (value) => ({ theme: "migrated", n: Number(value.n) }), + persist: true, + checkpointWhen: () => false, + }); + const { host, prefs } = await prefsIn(store, V2 as unknown as typeof Prefs); + expect(prefs().value).toEqual({ theme: "migrated", n: 1 }); + await prefs().change((draft) => { + draft.n = 2; + }, context); + expect((await outboxKinds(store)).at(-1)).toEqual(["change:base"]); + await host.close(context); + await store.close(context); + }); + + it("fail the activation broken for a newer stored version or another fingerprint at the same version (D9)", async () => { + const store = await memoryStore(); + const V2 = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 2, + initial: () => ({ theme: "light", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const Edited = defineState("prefs", { + schema: obj({ theme: str() }), + version: 2, + initial: () => ({ theme: "light" }), + persist: true, + checkpointWhen: () => false, + }); + const writer = await prefsIn(store, V2 as unknown as typeof Prefs); + await writer.prefs().change((draft) => { + draft.n = 1; + }, context); + await writer.host.close(context); + for (const [definition, message] of [ + [Prefs, "prefs: stored at version 2, newer than the code's 1"], + [Edited, "prefs: stored with another schema at version 2 (a schema changed without a bump)"], + ] as const) { + const { host, errors } = await loaded( + (h) => + h.serve(StoreRestartable, (_, self) => { + self.state(definition); + return { touch: async () => null }; + }), + { store }, + ); + await rejection(host.get(StoreRestartable, {}, context)); + expect(failure(errors)).toBe(`app.store: ${message}`); + await host.close(context); + } + await store.close(context); + }); + + it("refuse persisting without a stable identity or a store, and a flipped persist", async () => { + const InMemory = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 1, + initial: () => ({ theme: "light", n: 0 }), + }); + const keyed = await restartable(objects, (_, self) => { + // The types refuse it; a plugin may still try (P3). + (self as StoreSelf).state(Prefs); + }).catch((error: unknown) => error); + expect(String(keyed)).toContain("persist needs a stable identity"); + const storeless = await storeRestartable(objects, (_, self) => void self.state(Prefs)).catch((error) => error); + expect(String(storeless)).toContain("persist needs a store"); + const store = await memoryStore(); + const { fail, retry, errors } = await storeRestartable( + objects, + (attempt, self) => void self.state(attempt === 1 ? Prefs : InMemory), + { store }, + ); + fail(); + await rejection(retry()); + expect(failure(errors)).toBe("app.store: prefs: declared in memory, its documents are not"); + await store.close(context); + }); +}); + +/** The message of the last reported activation failure's error. */ +function failure(errors: readonly HostError[]): string | undefined { + const last = errors.findLast((error) => error.kind === "activation_failed"); + return last?.kind === "activation_failed" ? last.failure.error.message : undefined; +} + +/** What `read` throws, as ChordError JSON when it is one. */ +function rejectionOf(read: () => unknown): unknown { + try { + read(); + } catch (error) { + return isChordError(error) ? error.toJSON() : error; + } + return undefined; +} diff --git a/packages/chord/test/next/documents/transaction.test.ts b/packages/chord/test/next/documents/transaction.test.ts new file mode 100644 index 000000000..e26052957 --- /dev/null +++ b/packages/chord/test/next/documents/transaction.test.ts @@ -0,0 +1,747 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import type { Draft } from "../../../src/delta/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle, type Tx } from "../../../src/next/documents/index.ts"; +import type { Staged, Transaction } from "../../../src/next/documents/internal.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite as StoreStaged } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import type { TableOf } from "../../../src/next/store/table-handle.ts"; +import { + delivered, + framesOf, + instancesOf, + loaded, + memoryStore, + metered, + rejection, + restartable, + storeRestartable, +} from "../support.ts"; + +const instances = instancesOf(ChordObject, "attempt"); + +const A = defineState("a", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const B = defineState("b", { schema: obj({ s: str() }), version: 1, initial: () => ({ s: "" }) }); +const Owner = local("app.owner", { version: 1, params: { id: int() }, members: { sum: str() } }); + +type Handles = { + readonly self: Self; + readonly a: SingletonHandle<{ n: number }>; + readonly b: SingletonHandle<{ s: string }>; +}; + +/** Objects with two documents and a derive over them; returns each object's handles. */ +async function objects(telemetry?: Awaited>["telemetry"]) { + const handles = new Map(); + const { host } = await loaded( + (h) => { + h.serve(Owner, ({ id }, self) => { + const a = self.state(A); + const b = self.state(B); + handles.set(id, { self, a, b }); + return { sum: self.derive([a, b], ([x, y]) => `${x.n}${y.s}`) }; + }); + }, + { telemetry }, + ); + const reference = await host.get(Owner, { id: 1 }, context); + await host.get(Owner, { id: 2 }, context); + return { reference, one: handles.get(1)!, two: handles.get(2)! }; +} + +describe("transactions", () => { + it("run one at a time per object and return what their callback returns", async () => { + const { one } = await objects(); + const order: string[] = []; + const gate = Promise.withResolvers(); + const first = one.self.tx(async (tx) => { + order.push("first"); + await gate.promise; + (await tx.draft(one.a)).n = 1; + order.push("first done"); + return "one"; + }, context); + const second = one.self.tx(async (tx) => { + order.push(`second sees ${one.a.value.n}`); + (await tx.draft(one.a)).n++; + return "two"; + }, context); + await delivered(); + gate.resolve(); + expect(await Promise.all([first, second])).toEqual(["one", "two"]); + expect(order).toEqual(["first", "first done", "second sees 1"]); + expect(one.a.value).toEqual({ n: 2 }); + }); + + it("give the same draft for the same document", async () => { + const { one } = await objects(); + await one.self.tx(async (tx) => { + expect(await tx.draft(one.a)).toBe(await tx.draft(one.a)); + }, context); + }); + + it("settle a warm draft at once: a continuation the callback did not await runs before the commit (rule 3)", async () => { + const { one } = await objects(); + await one.self.tx((tx) => { + void tx.draft(one.a).then((draft) => { + draft.n = 7; + }); + }, context); + expect(one.a.value).toEqual({ n: 7 }); + }); + + it("reject a draft of a handle a transaction cannot use instead of throwing", async () => { + const { one, two } = await objects(); + await one.self.tx(async (tx) => { + const draft = tx.draft(two.a); + expect(draft).toBeInstanceOf(Promise); + await expect(draft).rejects.toThrow("drafts only its own object's documents"); + }, context); + }); + + it("commit nothing when nothing changed", async () => { + const telemetry = await metered(); + const { one, reference } = await objects(telemetry.telemetry); + const frames = framesOf(await reference.sum.watch(context)); + const value = one.a.value; + const computes = telemetry.counts()["chord.derive.computes"]; + await one.self.tx(async (tx) => { + void (await tx.draft(one.a)).n; + }, context); + await delivered(); + expect(one.a.value).toBe(value); + expect(frames).toEqual([]); + expect(telemetry.counts()["chord.derive.computes"]).toBe(computes); + }); + + it("discard everything on a throw and on an aborted context", async () => { + const { one } = await objects(); + const change = async (tx: Tx) => { + (await tx.draft(one.a)).n = 5; + }; + await expect( + one.self.tx(async (tx) => { + await change(tx); + throw new Error("no"); + }, context), + ).rejects.toThrow("no"); + const { context: cancelled, cancel } = withCancel(context); + await expect( + one.self.tx(async (tx) => { + await change(tx); + cancel(new Error("cancelled")); + }, cancelled), + ).rejects.toThrow("cancelled"); + expect(one.a.value).toEqual({ n: 0 }); + }); + + it("revoke their drafts when they settle", async () => { + const { one } = await objects(); + let kept: Draft<{ n: number }> | undefined; + await one.self.tx(async (tx) => { + kept = await tx.draft(one.a); + kept.n = 1; + }, context); + expect(() => kept!.n).toThrow(); + await expect(one.self.tx(async (tx) => tx, context).then((tx) => tx.draft(one.a))).rejects.toThrow( + "app.owner(1): the transaction has settled", + ); + }); + + it("refuse another object's documents", async () => { + const { one, two } = await objects(); + expect(await rejection(one.self.tx((tx) => tx.draft(two.a), context))).toEqual( + new Error("app.owner(1): a transaction drafts only its own object's documents"), + ); + }); + + it("commit the object's documents in one batch: one compute, one frame", async () => { + const telemetry = await metered(); + const { one, reference } = await objects(telemetry.telemetry); + const frames = framesOf(await reference.sum.watch(context)); + const computes = telemetry.counts()["chord.derive.computes"]!; + await one.self.tx(async (tx) => { + (await tx.draft(one.a)).n = 1; + (await tx.draft(one.b)).s = "b"; + }, context); + await delivered(); + expect(frames).toEqual([["1b", [["r", "1b"]]]]); + expect(telemetry.counts()["chord.derive.computes"]).toBe(computes + 1); + }); + + it("commit their other changes when they retire an absent document (R13)", async () => { + const { one } = await objects(); + await one.self.tx(async (tx) => { + await tx.retire(one.b); + (await tx.draft(one.a)).n = 7; + }, context); + expect([one.a.value, one.b.value]).toEqual([{ n: 7 }, { s: "" }]); + }); + + it("draft a retired document anew: watches get a root replacement", async () => { + const { one } = await objects(); + await one.a.change((draft) => { + draft.n = 3; + }, context); + const frames = framesOf(await one.a.watch(context)); + await one.self.tx(async (tx) => { + await tx.retire(one.a); + (await tx.draft(one.a)).n += 1; + }, context); + await delivered(); + expect(frames).toEqual([[{ n: 1 }, [["r", { n: 1 }]]]]); + }); +}); + +describe("the in-memory change's cost", () => { + it("settles an in-memory change within five microtask turns (a guard: an extra await per tx operation shows here)", async () => { + const { one } = await objects(); + for (let run = 0; run < 3; run++) { + let turns = 0; + let running = true; + const change = one.a.change((draft) => { + draft.n++; + }, context); + void change.then(() => { + running = false; + }); + while (running) { + turns++; + await null; + } + await change; + expect(turns).toBeLessThanOrEqual(5); + } + expect(one.a.value.n).toBe(3); + }); +}); + +describe("staged changes", () => { + /** A staged change recording what happens to it, and the document value its adoption sees. */ + function recorder(events: string[], a: SingletonHandle<{ n: number }>): Staged { + return { + adopt: () => void events.push(`adopt sees ${a.value.n}`), + committed: () => void events.push("committed"), + discarded: () => void events.push("discarded"), + }; + } + + it("adopt in the commit step with the documents, then hear of the commit once the line is free", async () => { + const { one } = await objects(); + const events: string[] = []; + const owner = {}; + await one.self.tx(async (tx) => { + (await tx.draft(one.a)).n = 7; + const transaction = tx as Transaction; + const staged = transaction.stage(owner, () => recorder(events, one.a)); + // One per owner and transaction. + expect(transaction.stage(owner, () => recorder([], one.a))).toBe(staged); + events.push("callback done"); + }, context); + expect(events).toEqual(["callback done", "adopt sees 7", "committed"]); + }); + + it("adopt without any document change, and hear of a discard instead when the transaction throws", async () => { + const { one } = await objects(); + const events: string[] = []; + await one.self.tx((tx) => void (tx as Transaction).stage({}, () => recorder(events, one.a)), context); + await rejection( + one.self.tx((tx) => { + (tx as Transaction).stage({}, () => recorder(events, one.a)); + throw new Error("no"); + }, context), + ); + expect(events).toEqual(["adopt sees 0", "committed", "discarded"]); + }); +}); + +describe("the seal fence", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + + /** A restartable object with `N`; its handles by attempt. */ + async function sealable() { + const handles: SingletonHandle<{ n: number }>[] = []; + const object = await restartable(instances, (_, self) => { + handles.push(self.state(N)); + }); + return { ...object, handles }; + } + + it("discards the transaction holding the line at the seal: the line is free, its later uses are refused", async () => { + const { attempts, handles, restart } = await sealable(); + const gate = Promise.withResolvers(); + let held: Tx | undefined; + const first = attempts[0]!.tx(async (tx) => { + (await tx.draft(handles[0]!)).n = 1; + held = tx; + await gate.promise; + return "late"; + }, context); + await delivered(); + await restart(); + // The next attempt's transaction runs while the cut one's callback still waits. + await handles[1]!.change((draft) => { + draft.n = 2; + }, context); + gate.resolve(); + expect(await rejection(first)).toMatchObject({ code: "released" }); + expect(await rejection(held!.draft(handles[1]!))).toMatchObject({ + code: "released", + message: "app.restartable(1): the transaction was discarded at the seal", + }); + expect(handles[1]!.value).toEqual({ n: 2 }); + }); + + it("refuses a sealed attempt's transactions waiting for the line and every new one; its handles still read", async () => { + const { attempts, handles, fail } = await sealable(); + const gate = Promise.withResolvers(); + const first = attempts[0]!.tx(() => gate.promise, context); + const queued = handles[0]!.change((draft) => { + draft.n = 1; + }, context); + await delivered(); + fail(); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + expect(await rejection(handles[0]!.change(() => {}, context))).toMatchObject({ code: "released" }); + expect(handles[0]!.value).toEqual({ n: 0 }); + gate.resolve(); + expect(await rejection(first)).toMatchObject({ code: "released" }); + }); +}); + +describe("tables in a transaction", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Edits = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); + type EditsTable = TableOf; + + /** A store object with `N` and `Edits`, on a store with an outbox; its handles by attempt. */ + async function stored(telemetry?: Awaited>["telemetry"]) { + const store: ChordStore = await memoryStore({ outbox: true }); + const handles: { n: SingletonHandle<{ n: number }>; edits: EditsTable }[] = []; + const object = await storeRestartable( + instances, + (_, self) => { + handles.push({ n: self.state(N), edits: self.table(Edits) }); + }, + { store, telemetry }, + ); + return { ...object, store, handles, self: object.attempts[0]!, ...handles[0]! }; + } + + it("commit rows and documents in one step: the documents adopt inside it, one seq", async () => { + const { self, n, edits, store, host } = await stored(); + const watch = await n.watch(context); + const seen: number[] = []; + watch.start(async () => { + seen.push(store.lastSeq); + }); + await delivered(); + const before = store.lastSeq; + await self.tx(async (tx) => { + tx.put(edits, { id: "e1", at: 1 }); + (await tx.draft(n)).n = 1; + }, context); + await delivered(); + expect(store.lastSeq).toBe(before + 1); + expect(seen.at(-1)).toBe(before + 1); + expect(n.value).toEqual({ n: 1 }); + expect(await edits.get({ id: "e1" }, context)).toEqual({ id: "e1", at: 1 }); + watch.stop(); + await host.close(context); + await store.close(context); + }); + + it("in-memory-only commits take no step and no seq (W12)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, n, store, host } = await stored(telemetry); + const commits = counts()["chord.store.commits"] ?? 0; + const seq = store.lastSeq; + await self.tx(async (tx) => { + (await tx.draft(n)).n = 5; + }, counting); + expect(store.lastSeq).toBe(seq); + expect(counts()["chord.store.commits"] ?? 0).toBe(commits); + await host.close(context); + await store.close(context); + }); + + it("read committed rows with tx.page and tx.rows, overlay own writes with tx.get", async () => { + const { self, edits, store, host } = await stored(); + await self.tx((tx) => tx.put(edits, { id: "e1", at: 1 }), context); + await self.tx(async (tx) => { + tx.put(edits, { id: "e2", at: 2 }); + expect((await tx.page(edits, query(Edits), 10)).items.map((row) => row.id)).toEqual(["e1"]); + const all: string[] = []; + for await (const row of tx.rows(edits, query(Edits))) all.push(row.id); + expect(all).toEqual(["e1"]); + expect(await tx.get(edits, { id: "e2" })).toEqual({ id: "e2", at: 2 }); + }, context); + expect(await edits.count(query(Edits), context)).toBe(2); + await host.close(context); + await store.close(context); + }); + + it("write nothing on a throw or an abort, and refuse a callback that settles with a pending read (rule 3)", async () => { + const { self, edits, store, host } = await stored(); + const seq = store.lastSeq; + await expect( + self.tx((tx) => { + tx.put(edits, { id: "e1", at: 1 }); + throw new Error("no"); + }, context), + ).rejects.toThrow("no"); + const { context: cancelled, cancel } = withCancel(context); + const aborted = self.tx(async (tx) => { + tx.put(edits, { id: "e1", at: 1 }); + cancel(); + }, cancelled); + await expect(aborted).rejects.toThrow(); + await expect( + self.tx((tx) => { + void tx.get(edits, { id: "e1" }); + tx.put(edits, { id: "e1", at: 1 }); + }, context), + ).rejects.toThrow("settled before its pending tx operations"); + expect(store.lastSeq).toBe(seq); + expect(await edits.count(query(Edits), context)).toBe(0); + await host.close(context); + await store.close(context); + }); + + it("refuse another object's table", async () => { + const { self, store, host } = await stored(); + const other = await stored(); + await expect(self.tx((tx) => tx.put(other.edits, { id: "e1", at: 1 }), context)).rejects.toThrow( + "writes only its own object's tables", + ); + await host.close(context); + await other.host.close(context); + await store.close(context); + await other.store.close(context); + }); + + it("a commit queued behind another owner's step is refused at the seal; an admitted one is adopted", async () => { + const { attempts, handles, store, fail, retry, host } = await stored(); + // Hold the commit queue with a step that waits. + const gate = Promise.withResolvers(); + const holding = store.commit( + [ + { + order: "write", + apply: async () => { + await gate.promise; + return [{ kind: "dropIndex", table: "x", name: "y" }]; + }, + }, + ], + { admit: () => true, adopt: () => {} }, + context, + ); + const queued = attempts[0]!.tx(async (tx) => { + tx.put(handles[0]!.edits, { id: "e1", at: 1 }); + (await tx.draft(handles[0]!.n)).n = 1; + }, context); + await delivered(); + fail(); + gate.resolve(); + await holding.catch(() => {}); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + await retry(); + expect(handles[1]!.n.value).toEqual({ n: 0 }); + expect(await handles[1]!.edits.get({ id: "e1" }, context)).toBeUndefined(); + // Admitted, then sealed while in the step: durable, adopted into the object's slots, `self.tx` resolves. + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => + commit(writes, { admit: () => sealedAfter(fence.admit()), adopt: fence.adopt }, ctx); + const sealedAfter = (admitted: boolean): boolean => { + if (admitted) fail(); + return admitted; + }; + const admitted = await attempts[1]!.tx(async (tx) => { + tx.put(handles[1]!.edits, { id: "e2", at: 2 }); + (await tx.draft(handles[1]!.n)).n = 2; + return "done"; + }, context); + store.commit = commit; + expect(admitted).toBe("done"); + await retry(); + expect(handles[2]!.n.value).toEqual({ n: 2 }); + expect(await handles[2]!.edits.get({ id: "e2" }, context)).toEqual({ id: "e2", at: 2 }); + await host.close(context); + await store.close(context); + }); +}); + +describe("persisted documents in a transaction", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const P = defineState("p", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const K = defineState("k", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + + /** A store object with in-memory `N`, persisted `P` and keyed persisted `K`, on a store with an outbox. */ + async function stored(telemetry?: Awaited>["telemetry"], on?: ChordStore) { + const store: ChordStore = on ?? (await memoryStore({ outbox: true })); + const handles: { + n: SingletonHandle<{ n: number }>; + p: SingletonHandle<{ n: number }>; + k: KeyedHandle<{ n: number }>; + }[] = []; + const object = await storeRestartable( + instances, + (_, self) => { + handles.push({ n: self.state(N), p: self.state(P), k: self.state(K) }); + }, + { store, telemetry }, + ); + return { ...object, store, self: object.attempts[0]!, ...handles[0]! }; + } + + it("commit persisted and in-memory documents in one step; a delta commit takes three statements", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, n, p, store, host } = await stored(telemetry); + await self.tx(async (tx) => { + (await tx.draft(n)).n = 1; + (await tx.draft(p)).n = 1; + }, counting); + expect(store.lastSeq).toBe(1); + const before = { ...counts() }; + await self.tx(async (tx) => { + (await tx.draft(p)).n = 2; + }, counting); + expect(counts()["chord.store.commits"]! - before["chord.store.commits"]!).toBe(1); + // The revision, the record's conditional UPDATE, lastSeq; plus the outbox commit. + expect(counts()["chord.store.statements"]! - before["chord.store.statements"]!).toBe(4); + expect([n.value, p.value]).toEqual([{ n: 1 }, { n: 2 }]); + await host.close(context); + await store.close(context); + }); + + it("a keyed retirement takes 6a's statements: the counted DELETE, the revisions, lastSeq (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, k, store, host } = await stored(telemetry); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n = 1; + }, counting); + const before = { ...counts() }; + await self.tx(async (tx) => { + await tx.retire(k, "a"); + }, counting); + // No read of the record first, no second DELETE of its row; plus the outbox commit. + expect(counts()["chord.store.statements"]! - before["chord.store.statements"]!).toBe(4); + await host.close(context); + await store.close(context); + }); + + it("assign the created documents' ids by their order in the commit; retire and recreate in one commit", async () => { + const { self, k, store, host } = await stored(); + await self.tx(async (tx) => { + (await tx.draft(k, "b")).n = 1; + (await tx.draft(k, "a")).n = 2; + }, context); + await self.tx(async (tx) => { + await tx.retire(k, "b"); + (await tx.draft(k, "b")).n = 3; + }, context); + const commits = await store.outbox!.read(0, 10, context); + const created = commits.flatMap(({ writes }) => + writes.flatMap((write) => (write.kind === "document.create" ? [[write.record.key, write.record.id]] : [])), + ); + expect(created).toEqual([ + ["b", 2], + ["a", 3], + ["b", 4], + ]); + expect(commits[1]!.writes.map(({ kind }) => kind)).toEqual(["document.retire", "document.create"]); + // What is stored of `b` is the new record's: the next change is a delta on it. + await k.change( + "b", + (draft) => { + draft.n = 4; + }, + context, + ); + expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 4 }]); + await host.close(context); + await store.close(context); + }); + + it("draft a cold keyed document by loading it inside the transaction; refuse a callback with a pending load", async () => { + const first = await stored(); + const { store } = first; + await first.k.change( + "a", + (draft) => { + draft.n = 1; + }, + context, + ); + await first.host.close(context); + const { self, k, host } = await stored(undefined, store); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n += 1; + }, context); + expect(await k.get("a", context)).toEqual({ n: 2 }); + await expect( + self.tx(async (tx) => { + // Not awaited: the draft is still loading when the callback returns (it rejects later, settled). + tx.draft(k, "b").catch(() => {}); + }, context), + ).rejects.toThrow("the callback settled before its pending tx operations"); + await host.close(context); + await store.close(context); + }); + + it("settle a warm keyed draft at once: a continuation the callback did not await runs before the commit", async () => { + const { self, k, host, store } = await stored(); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n = 1; + }, context); + await self.tx((tx) => { + void tx.draft(k, "a").then((draft) => { + draft.n = 7; + }); + }, context); + expect(await k.get("a", context)).toEqual({ n: 7 }); + await host.close(context); + await store.close(context); + }); + + it("a commit cut at the seal while it waits in the store's queue rejects released at once, writing nothing", async () => { + const { p, store, host } = await stored(); + // A suspended export holds the commit queue. + const parts = store.export(context); + await parts.next(); + const change = rejection( + p.change((draft) => { + draft.n = 5; + }, context), + ); + await delivered(); + await host.close(context); + expect(await change).toMatchObject({ code: "released" }); + await parts.return!(); + expect(store.lastSeq).toBe(0); + const next = await stored(undefined, store); + expect(next.p.value).toEqual({ n: 0 }); + await next.p.change((draft) => { + draft.n = 6; + }, context); + expect(store.lastSeq).toBe(1); + await next.host.close(context); + await store.close(context); + }); + + it("a host closing while an admitted commit is in the SQL waits for its adoption; the store stays usable", async () => { + const { p, store, host } = await stored(); + // The next commit's step waits inside the backend transaction, after its admission. + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StoreStaged = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return [{ kind: "dropIndex", table: "x", name: "y" }]; + }, + }; + return commit([...writes, held], fence, ctx); + }; + // Not from a method handler: the drain does not wait for it. + const change = p.change((draft) => { + draft.n = 5; + }, context); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await delivered(); + expect(closed).toBe(false); + gate.resolve(); + await change; + await closing; + store.commit = commit; + expect(store.lastSeq).toBe(1); + const next = await stored(undefined, store); + expect(next.p.value).toEqual({ n: 5 }); + await next.p.change((draft) => { + draft.n = 6; + }, context); + expect(store.lastSeq).toBe(2); + await next.host.close(context); + await store.close(context); + }); + + it("a handler's commit held in the SQL past closeTimeout still adopts before the object ends; the store stays usable", async () => { + const Writer = local("app.writer", { version: 1, params: {}, members: { write: method() } }); + const store = await memoryStore({ outbox: true }); + const serve = (h: Parameters[0]>[0]) => + h.serve(Writer, (_, self) => { + const p = self.state(P); + return { + write: async (ctx) => { + await p.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; + }); + const { host, errors } = await loaded(serve, { store, closeTimeout: 20 }); + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StoreStaged = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return [{ kind: "dropIndex", table: "x", name: "y" }]; + }, + }; + return commit([...writes, held], fence, ctx); + }; + const call = rejection((await host.get(Writer, {}, context)).write(context)); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + // The drain gave up on the handler; the object's teardown still waits for the admitted step. + expect(errors.map(({ kind }) => kind)).toEqual(["drain_timeout"]); + expect(closed).toBe(false); + gate.resolve(); + await closing; + // The stuck call was released at the end; its commit, admitted before, is durable and was adopted. + expect(await call).toMatchObject({ code: "released" }); + store.commit = commit; + expect(store.lastSeq).toBe(1); + const next = await loaded(serve, { store }); + await (await next.host.get(Writer, {}, context)).write(context); + expect(store.lastSeq).toBe(2); + expect(next.errors).toEqual([]); + await next.host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/examples/01-objects.ts b/packages/chord/test/next/examples/01-objects.ts new file mode 100644 index 000000000..57a691e58 --- /dev/null +++ b/packages/chord/test/next/examples/01-objects.ts @@ -0,0 +1,68 @@ +// Objects on one host: a contract, a facet serving it, references, calls, and how long objects live. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/01-objects.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +// Every call takes a context, used for cancellation; BACKGROUND_CONTEXT is never cancelled. +const context = BACKGROUND_CONTEXT; + +// A contract declares what an object type offers: state members (schemas) and methods (arguments, result). +// `params` select the instance: one object per params value. +const Greeter = local("example.greeter", { + version: 1, + params: { language: str() }, + members: { language: str(), greet: method(obj({ name: str() }), str()) }, +}); +// Empty params: one object for the whole host. +const Clock = local("example.clock", { version: 1, params: {}, members: { started: int() } }); + +// A facet is a plugin's setup for one host. It serves types with factories; a factory only declares and returns the +// implementation: a watchable per state member, an async function per method. +const facet = defineFacet((host) => { + host.serve(Clock, () => { + console.log("clock: created at load"); + return { started: constant(Date.now()) }; + }); + host.serve(Greeter, ({ language }) => { + console.log(`greeter ${language}: created by its first get`); + return { + language: constant(language), + greet: async ({ name }) => (language === "de" ? `Hallo ${name}` : `Hello ${name}`), + }; + }); +}); + +// onError receives background errors, such as a watch listener that throws. +const host = createHost({ onError: (error) => console.error("background error:", error) }); +// load runs every setup and activates the types with empty params before it resolves. +await host.load([{ name: "@example/greeter", facet }], context); + +// A reference is live once its get resolves: its state members have values, its methods can be called. +const german = await host.get(Greeter, { language: "de" }, context); +console.log("language:", german.language.value); +console.log("greet:", await german.greet({ name: "Mario" }, context)); + +// Arguments are checked against the contract at the object. +try { + // @ts-expect-error: `name` must be a string + await german.greet({ name: 42 }, context); +} catch (error) { + if (isChordError(error, "invalid")) console.log("refused:", error.reason, error.message); +} + +// The same params share one object; releasing the last reference ends it. +const again = await host.get(Greeter, { language: "de" }, context); +german.release(); +again.release(); +try { + german.language.value; +} catch (error) { + if (isChordError(error, "released")) console.log("after release:", error.message); +} + +// close ends every object. +await host.close(context); diff --git a/packages/chord/test/next/examples/02-documents.ts b/packages/chord/test/next/examples/02-documents.ts new file mode 100644 index 000000000..1387cf0dc --- /dev/null +++ b/packages/chord/test/next/examples/02-documents.ts @@ -0,0 +1,79 @@ +// Documents and transactions: an object's in-memory JSON state, written atomically. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/02-documents.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// A singleton document: one per object, `initial()` until written. +const Todos = defineState("todos", { + schema: obj({ items: arr(obj({ id: int(), text: str() })), next: int() }), + version: 1, + initial: () => ({ items: [], next: 1 }), +}); +// A keyed document: many per object, one per string key, absent until written. +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, +}); + +const Store = local("example.todos", { + version: 1, + params: {}, + members: { + todos: obj({ items: arr(obj({ id: int(), text: str() })), next: int() }), + add: method(obj({ text: str(), note: opt(str()) }), int()), + note: method(obj({ id: int() }), str()), + clear: method(), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Store, (_, self) => { + const todos = self.state(Todos); + const notes = self.state(Notes); + return { + // A singleton handle is a watchable, so it can be a state member as it is. + todos, + add: async ({ text, note }, ctx) => { + // One transaction over two documents: both change together, or neither does. + const id = await self.tx(async (tx) => { + const draft = await tx.draft(todos); + const id = draft.next++; + draft.items.push({ id, text }); + if (note !== undefined) (await tx.draft(notes, String(id))).body = note; + // Return plain values: drafts are revoked once the transaction settles. + return id; + }, ctx); + return id; + }, + // Read keyed documents outside transactions; an absent one is `undefined`. + note: async ({ id }, ctx) => (await notes.get(String(id), ctx))?.body ?? "(none)", + clear: async (ctx) => { + // Shorthands for one document: retire resets a singleton to `initial()`. + await todos.retire(ctx); + for await (const key of notes.keys(ctx)) await notes.retire(key, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/todos", facet }], context); +const store = await host.get(Store, {}, context); + +const first = await store.add({ text: "buy milk", note: "the oat one" }, context); +await store.add({ text: "call Anna" }, context); +console.log("todos:", store.todos.value.items); +console.log("note of", first, "->", await store.note({ id: first }, context)); + +await store.clear(context); +console.log("after clear:", store.todos.value, await store.note({ id: first }, context)); + +await host.close(context); diff --git a/packages/chord/test/next/examples/03-derives.ts b/packages/chord/test/next/examples/03-derives.ts new file mode 100644 index 000000000..3d756253c --- /dev/null +++ b/packages/chord/test/next/examples/03-derives.ts @@ -0,0 +1,66 @@ +// Derives: values computed from documents and from other objects' members, recomputed only when needed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/03-derives.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const NotesDoc = defineState("notes", { + schema: obj({ list: arr(obj({ text: str() })) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const Notes = local("example.notes", { + version: 1, + params: {}, + members: { titles: arr(str()), count: int(), add: method(obj({ text: str() })) }, +}); +const Badge = local("example.badge", { version: 1, params: { user: str() }, members: { label: str() } }); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const notes = self.state(NotesDoc); + // mapShared calls its function once per note object: unchanged notes keep the same title, so the derive's + // diff after an append is one insert, not a comparison of every title. + const title = mapShared((note: { text: string }) => { + console.log(" title computed for:", note.text); + return note.text.slice(0, 12); + }); + return { + // A derive declares its sources; its compute is synchronous and pure. + titles: self.derive([notes], ([n]) => title(n.list)), + count: self.derive([notes], ([n]) => n.list.length), + add: async ({ text }, ctx) => { + await notes.change((draft) => { + draft.list.push({ text }); + }, ctx); + return null; + }, + }; + }); + // A derive over another object's member, through a reference declared with self.get. + host.serve(Badge, ({ user }, self) => { + const notes = self.get(Notes, {}); + return { label: self.derive([notes.count], ([count]) => `${user}: ${count} notes`) }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/notes", facet }], context); +const notes = await host.get(Notes, {}, context); +const badge = await host.get(Badge, { user: "mario" }, context); + +await notes.add({ text: "buy milk and bread" }, context); +await notes.add({ text: "call Anna" }, context); +// Nobody watches `titles`, so the commits only marked it; reading computes it, each title once. +console.log("titles:", notes.titles.value); +await notes.add({ text: "water plants" }, context); +console.log("titles:", notes.titles.value); +console.log("badge:", badge.label.value); + +await host.close(context); diff --git a/packages/chord/test/next/examples/04-watches.ts b/packages/chord/test/next/examples/04-watches.ts new file mode 100644 index 000000000..f6f81972c --- /dev/null +++ b/packages/chord/test/next/examples/04-watches.ts @@ -0,0 +1,101 @@ +// Watches: follow a state member from its current value through the commits that change it; a busy listener gets +// what changed meanwhile folded into one call; a derive that throws gives a `failed` call, the watch stays open, and +// its next value arrives as a root replacement (it heals on a later change). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/04-watches.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("example.counter", { + version: 1, + params: {}, + members: { count: obj({ n: int() }), double: int(), half: int(), bump: method(), reset: method() }, +}); + +const facet = defineFacet((host) => { + host.serve(Counter, (_, self) => { + const count = self.state(Count); + return { + count, + double: self.derive([count], ([c]) => c.n * 2), + half: self.derive([count], ([c]) => { + if (c.n % 2 === 1) throw new Error(`${c.n} has no half`); + return c.n / 2; + }), + bump: async (ctx) => { + await count.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + reset: async (ctx) => { + await count.retire(ctx); + return null; + }, + }; + }); +}); + +// A derive whose compute throws is reported once per failure episode (`derive_failed`). +const host = createHost({ + onError: (error) => { + if (error.kind === "derive_failed") console.log(`reported: ${error.object}.${error.member}: ${error.error}`); + else console.error("background error:", error); + }, +}); +await host.load([{ name: "@example/counter", facet }], context); +const counter = await host.get(Counter, {}, context); + +// A watch holds the value at acquisition; its listener then gets the changes, asynchronously and in order, never +// overlapping itself. A listener that keeps up gets one call per commit: the new value and the ops that turn the +// previous value into it. +const watch = await counter.count.watch(context); +console.log("acquired:", watch.value); +watch.start(async (call) => { + if (call.kind === "value") console.log("count frame:", call.value, JSON.stringify(call.ops)); +}); + +// A derive's frame carries the diff of its old and new value. +const { context: cancellable, cancel } = withCancel(context); +const doubled = await counter.double.watch(cancellable); +doubled.start(async (call) => { + if (call.kind === "value") console.log("double frame:", call.value, JSON.stringify(call.ops)); +}); + +// A failing derive: one `failed` call per failure, the watch stays open; the next value replaces the root. +const half = await counter.half.watch(context); +half.start(async (call) => { + if (call.kind === "failed") console.log("half failed:", (call.error as Error).message); + else console.log("half frame:", call.value, JSON.stringify(call.ops)); +}); + +// A slow listener: commits made while it is busy fold into its next call, with the newest value and the ops since +// its previous call (an `s` on one path replaces the one before it). +const slow = await counter.count.watch(context); +slow.start(async (call) => { + if (call.kind !== "value") return; + console.log("slow frame:", call.value, JSON.stringify(call.ops)); + await new Promise((resolve) => setTimeout(resolve, 20)); +}); + +await counter.bump(context); +await new Promise((resolve) => setTimeout(resolve, 0)); +await counter.bump(context); +await counter.bump(context); +await new Promise((resolve) => setTimeout(resolve, 50)); +// Retiring the singleton resets it to initial(): watches get a root replacement. +await counter.reset(context); +await new Promise((resolve) => setTimeout(resolve, 50)); + +// Cancelling the watch's context ends it; stop() ends it too. Releasing the reference ends the watches taken through it. +cancel(); +console.log("double watch:", await doubled.closed); +counter.release(); +console.log("count watch:", await watch.closed); + +await host.close(context); diff --git a/packages/chord/test/next/examples/05-errors.ts b/packages/chord/test/next/examples/05-errors.ts new file mode 100644 index 000000000..fabdaf3b9 --- /dev/null +++ b/packages/chord/test/next/examples/05-errors.ts @@ -0,0 +1,94 @@ +// Errors: what Chord refuses and how a caller tells the cases apart (`ChordError` with `code` and `reason`). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/05-errors.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** Prints how `work` failed: a ChordError's code, reason and message, or a plain error's message. */ +async function attempt(label: string, work: () => unknown): Promise { + try { + await work(); + console.log(`${label}: ok`); + } catch (error) { + if (isChordError(error)) + console.log(`${label}: ${error.code}${error.reason ? `/${error.reason}` : ""} - ${error.message}`); + else console.log(`${label}: ${error instanceof Error ? error.message : String(error)}`); + } +} + +const Echo = local("example.echo", { + version: 1, + params: { id: int() }, + members: { id: int(), echo: method(obj({ text: str() }), str()) }, +}); +// Two types that get each other while they activate: a cycle. +const Ping = local("example.ping", { version: 1, params: {}, members: { n: int() } }); +const Pong = local("example.pong", { version: 1, params: {}, members: { n: int() } }); +const Broken = local("example.broken", { version: 1, params: { id: int() }, members: { n: int() } }); +const Sloppy = local("example.sloppy", { version: 1, params: { id: int() }, members: { n: int() } }); + +const facet = defineFacet((host) => { + host.serve(Echo, ({ id }) => ({ id: constant(id), echo: async ({ text }) => text })); + host.serve(Broken, () => { + throw new Error("broken: missing configuration"); + }); + // Plugins are bundled without type checks; Chord checks the implementation's members when the object activates. + host.serve(Sloppy, () => ({ n: constant(1), extra: constant(2) }) as never); +}); +// Ping and Pong are served by a second plugin: types with empty params activate at load. +const cycle = defineFacet((host) => { + host.serve(Ping, (_, self) => ({ n: self.get(Pong, {}).n })); + host.serve(Pong, (_, self) => ({ n: self.get(Ping, {}).n })); +}); + +// Failed activations are reported to onError; 07-failures.ts shows the other reports. +const onError = (error: HostError): void => { + if (error.kind === "activation_failed") console.log(`reported: ${error.object} failed ${error.failure.kind}`); + else console.log("reported:", error.kind); +}; + +// A dependency cycle fails every object on it broken, with invalid/cycle naming the chain. Load still resolves: the +// start barrier waits for each object's first outcome, live or failed. +const host = createHost({ onError }); +await host.load( + [ + { name: "@example/errors", facet }, + { name: "@example/cycle", facet: cycle }, + ], + context, +); +await attempt("get in a cycle", () => host.get(Ping, {}, context)); +const echo = await host.get(Echo, { id: 1 }, context); + +// Arguments are checked against the contract at the object: wrong types and unknown fields are invalid/args. +// @ts-expect-error: text must be a string +await attempt("wrong argument", () => echo.echo({ text: 1 }, context)); +// @ts-expect-error: an unknown field +await attempt("unknown field", () => echo.echo({ text: "hi", extra: true }, context)); + +// Params select the object and are checked on every get: invalid/params. +// @ts-expect-error: id must be an integer +await attempt("bad params", () => host.get(Echo, { id: "one" }, context)); + +// An implementation with a member its contract does not declare fails activation. +await attempt("unknown member", () => host.get(Sloppy, { id: 1 }, context)); + +// A get needs the provider's version and fingerprint; another declaration of the type does not fit. That is not the +// caller's fault (a reload may bring the version): the get waits up to its readyTimeout, then rejects with the cause. +const EchoV2 = local("example.echo", { version: 2, params: { id: int() }, members: { id: int() } }); +await attempt("contract mismatch", () => host.get(EchoV2, { id: 1 }, context, { readyTimeout: 100 })); + +// A factory that throws fails broken: the get rejects at once with its error, and no object is left behind. +await attempt("throwing factory", () => host.get(Broken, { id: 1 }, context)); + +// After release, every use of a reference throws released. +echo.release(); +await attempt("released reference", () => echo.echo({ text: "hi" }, context)); + +await host.close(context); +await attempt("get after close", () => host.get(Echo, { id: 1 }, context)); diff --git a/packages/chord/test/next/examples/06-natives.ts b/packages/chord/test/next/examples/06-natives.ts new file mode 100644 index 000000000..8e74f06d6 --- /dev/null +++ b/packages/chord/test/next/examples/06-natives.ts @@ -0,0 +1,96 @@ +// Natives: values an object opens during activation and closes when it ends (a file watcher, a connection, a timer). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/06-natives.ts +import { setTimeout as sleep } from "node:timers/promises"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Clock = local("example.clock", { + version: 1, + params: { zone: str() }, + members: { zone: str(), now: method(undefined, str()) }, +}); +const Leaky = local("example.leaky", { version: 1, params: {}, members: { n: int() } }); + +const facet = defineFacet((host) => { + host.serve(Clock, ({ zone }, self) => { + // `self.open(open, close)` declares a native; it opens after the references are live, in declaration order. + const config = self.open( + async () => { + console.log(`${zone}: open config`); + return { prefix: `[${zone}]` }; + }, + (_config, reason) => console.log(`${zone}: close config (${reason.kind})`), + ); + // A later opener reads an earlier native's value. Its context is aborted if the object ends while it opens. + const ticker = self.open( + async () => { + console.log(`${zone}: open ticker, config ${config.value.prefix}`); + const ticker = { ticks: 0, timer: setInterval(() => ticker.ticks++, 10) }; + return ticker; + }, + (ticker, reason) => { + clearInterval(ticker.timer); + console.log(`${zone}: close ticker (${reason.kind})`); + }, + ); + // The factory only declares: a value is there once its open resolved. + try { + config.value; + } catch (error) { + if (isChordError(error, "not_ready")) console.log(`${zone}: in the factory, ${error.message}`); + } + return { + zone: constant(zone), + now: async () => `${config.value.prefix} ${ticker.value.ticks > 0 ? "ticking" : "starting"}`, + }; + }); + // Closes that throw or hang: they never stop the other closes. + host.serve(Leaky, (_, self) => { + self.open( + async () => "socket", + () => { + throw new Error("socket close failed"); + }, + ); + self.open( + async () => "worker", + () => new Promise(() => {}), + { closeTimeout: 50 }, + ); + return { n: constant(1) }; + }); +}); + +// onError gets what fails in the background: here a close past its closeTimeout. +const host = createHost({ + onError: (error) => console.log("reported:", error.kind, "object" in error ? error.object : ""), +}); +await host.load([{ name: "@example/natives", facet }], context); + +const vienna = await host.get(Clock, { zone: "vienna" }, context); +console.log("now:", await vienna.now(context)); +await sleep(30); +console.log("now:", await vienna.now(context)); + +// The last reference gone, the object ends: natives close in reverse order, with reason `end`. +vienna.release(); +await sleep(0); + +// host.close ends every object at once and closes their natives with reason `close`; it rejects with every close +// that threw. +await host.get(Clock, { zone: "tokyo" }, context); +try { + await host.close(context); +} catch (error) { + if (error instanceof AggregateError) + console.log( + "host.close:", + error.errors.map((e: Error) => e.message), + ); +} diff --git a/packages/chord/test/next/examples/07-failures.ts b/packages/chord/test/next/examples/07-failures.ts new file mode 100644 index 000000000..31dc81a42 --- /dev/null +++ b/packages/chord/test/next/examples/07-failures.ts @@ -0,0 +1,97 @@ +// Failures: how an activation fails (transient or broken), how Chord retries it, and what getters see. +// Run from packages/chord (takes about 3 s: the retries wait 1 s, then 2 s): +// node --conditions=source --experimental-strip-types test/next/examples/07-failures.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, str } from "../../../src/next/contracts/index.ts"; +import { broken, isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; +const start = Date.now(); +const at = (): string => `${((Date.now() - start) / 1000).toFixed(1)} s`; + +// A connection to a remote service, which is down for the first two attempts. +const Remote = local("example.remote", { version: 1, params: {}, members: { status: str() } }); +// An inbox per user, which needs the connection. +const Inbox = local("example.inbox", { version: 1, params: { user: str() }, members: { status: str() } }); +// A mirror whose configuration is missing: retrying cannot help. +const Mirror = local("example.mirror", { version: 1, params: { id: int() }, members: { status: str() } }); +// A service whose open never finishes. +const Stuck = local("example.stuck", { version: 1, params: { id: int() }, members: { status: str() } }); + +let attempts = 0; +const facet = defineFacet((host) => { + host.serve(Remote, (_, self) => { + const attempt = ++attempts; + self.open( + async () => { + console.log(`${at()}: remote, attempt ${attempt}`); + // A throwing open is transient: Chord retries it after 1 s, then 2 s, 4 s, ... up to 60 s. + if (attempt <= 2) throw new Error("connection refused"); + return { connected: true }; + }, + () => {}, + ); + return { status: constant("connected") }; + }); + host.serve(Inbox, ({ user }, self) => { + // A dependency that fails parks this object (`failed { waiting }`, no timer of its own, the factory not run + // again); it goes on the moment the dependency is live (31-parked-dependencies.ts). + const remote = self.get(Remote, {}); + console.log(`${at()}: inbox ${user}, attempt`); + return { status: self.derive([remote.status], ([status]) => `${user}: ${status}`) }; + }); + host.serve(Mirror, (_, self) => { + self.open( + async () => { + // broken(error) says retrying is pointless: no retry, and getters are rejected at once. + throw broken(new Error("no mirror configured")); + }, + () => {}, + ); + return { status: constant("mirroring") }; + }); + host.serve(Stuck, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { status: constant("never") }; + }); +}); + +// Every failed attempt is reported, with its object and cause. +const onError = (error: HostError): void => { + if (error.kind === "activation_failed") { + console.log(`${at()}: reported ${error.object}, ${error.failure.kind}: ${error.failure.error.message}`); + } +}; + +/** Prints how a get ended. */ +async function attempt(label: string, work: () => Promise<{ status: { value: string } }>): Promise { + try { + const reference = await work(); + console.log(`${at()}: ${label}: ${reference.status.value}`); + } catch (error) { + if (isChordError(error)) console.log(`${at()}: ${label}: ${error.code} - ${error.message}`); + } +} + +// Every attempt must be live within readyTimeout (here 500 ms); getters wait up to it too, unless they say otherwise. +const host = createHost({ onError, readyTimeout: 500 }); +// The start barrier: load resolves once the remote (empty params: it lives as long as the host) had its first outcome. +await host.load([{ name: "@example/failures", facet }], context); +console.log(`${at()}: loaded`); + +// A broken failure rejects the get at once. +await attempt("mirror", () => host.get(Mirror, { id: 1 }, context)); + +// An attempt not live within readyTimeout is abandoned and fails transient; this get waits longer (700 ms) and then +// rejects with the last failure's error. +await attempt("stuck", () => host.get(Stuck, { id: 1 }, context, { readyTimeout: 700 })); + +// Transient failures keep a get waiting through the retries, up to its deadline: the inbox goes live with the remote. +await attempt("inbox", () => host.get(Inbox, { user: "mario" }, context, { readyTimeout: 10_000 })); + +await host.close(context); diff --git a/packages/chord/test/next/examples/08-remote.ts b/packages/chord/test/next/examples/08-remote.ts new file mode 100644 index 000000000..fb538aecb --- /dev/null +++ b/packages/chord/test/next/examples/08-remote.ts @@ -0,0 +1,119 @@ +// Remote objects over one link: a server and a client host, a remote contract with two variants, a factory choosing +// the variant per principal over a shared local store, and the client's replicas following the server's commits. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/08-remote.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The app's principals: who calls. Links stamp them; a caller never sets its own. +type P = { kind: "owner" } | { kind: "guest"; session: string } | { kind: "server" }; + +const Note = obj({ text: str(), author: str() }); +const PublicNote = obj({ text: str() }); + +// The server's host kind: every remote contract names the address of the host serving its objects. +const Server = hostKind("server"); + +// A remote contract: one object per (type, params, principal key), in the variant its factory chooses. +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(obj({ text: str() })) } }, + reader: { state: { notes: arr(PublicNote), count: int() } }, + }, +}); + +// The shared store: the notes of a session once, behind per-principal views (design §6). Local types never cross a link. +const NoteStore = local("example.note-store", { + version: 1, + params: { session: str() }, + members: { notes: arr(Note), add: method(Note) }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(Note) }), version: 1, initial: () => ({ list: [] }) }); + +// A redaction computed once per store element, so a reader's derive keeps unchanged notes identical. +const redact = mapShared((note: { text: string; author: string }) => ({ text: note.text })); + +const facet = defineFacet

((host) => { + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + notes: self.derive([doc], ([d]) => d.list), + add: async (note, ctx) => { + await doc.change((draft) => void draft.list.push(note), ctx); + return null; + }, + }; + }); + // A remote factory gets the principal its object is for and returns one variant with exactly its members. + host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); + const count = self.derive([store.notes], ([n]) => n.length); + if (principal.kind === "guest") { + return { variant: "reader", members: { notes: self.derive([store.notes], ([n]) => redact(n)), count } }; + } + return { + variant: "editor", + members: { + notes: store.notes, + count, + add: async ({ text }, ctx) => store.add({ text, author: principal.kind }, ctx), + }, + }; + }); +}); + +// 1. The server, at its address: it acts as `server` for its own requests. +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); +await server.load([{ name: "@example/notes", facet }], context); + +// 2. A client (a tab): no principal of its own. The entry point authenticated it as a guest of S1 and stamps that on +// the link; the client connects before it loads (its plugins may reference server types). +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "guest", session: "S1" } }, context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +client.connect(down, { principal: { kind: "server" } }, context); +await client.load([], context); + +// 3. The server writes as an editor; the guest reads the redacted view over the link. +const own = await server.get(Notes, { session: "S1" }, context); +if (own.variant === "editor") await own.add({ text: "hello" }, context); +const notes = await client.get(Notes, { session: "S1" }, context); +console.log("guest variant:", notes.variant); +console.log("guest notes:", JSON.stringify(notes.notes.value), "count:", notes.count.value); + +// 4. A commit on the server reaches the client as one state message: both members change in the same batch. +const watch = await notes.notes.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("frame:", JSON.stringify(call.value), "ops:", JSON.stringify(call.ops)); +}); +if (own.variant === "editor") await own.add({ text: "world" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("count after the commit:", notes.count.value); + +// 5. TypeScript narrows on `variant`: a reader has no `add` (a forged call would fail `invalid/member` at the server). +if (notes.variant === "reader") { + // @ts-expect-error: `add` is not a member of the reader variant + void notes.add; +} + +// 6. Closing the server closes its links first: the client's reference goes loading and keeps its last values until +// a new link holds it again (10-drops.ts). Closing the client releases it and ends its watches. +await server.close(context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("after the server closed:", notes.state.value, "count:", notes.count.value); +await client.close(context); +console.log("after the client closed:", notes.state.value, await watch.closed); diff --git a/packages/chord/test/next/examples/09-calls.ts b/packages/chord/test/next/examples/09-calls.ts new file mode 100644 index 000000000..5c099fd5d --- /dev/null +++ b/packages/chord/test/next/examples/09-calls.ts @@ -0,0 +1,115 @@ +// Calls over a link: results, bytes as raw segments, ChordErrors and plugin errors, the caller's principal and +// incarnation in `ctx`, cancellation, and release rejecting the calls made through a reference. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/09-calls.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { bytes, hostKind, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError, PluginError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + +const Files = remote("example.files", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + owner: { + methods: { + whoami: method(undefined, str()), + // A `bytes()` leaf: the argument and result travel as raw segments, never base64. + upload: method(obj({ name: str(), data: bytes() }), int()), + download: method(obj({ name: str() }), bytes()), + quota: method(), + slow: method(), + }, + }, + }, +}); + +const stored = new Map(); + +const facet = defineFacet

((host) => { + host.serve(Files, () => ({ + variant: "owner", + members: { + // ctx.principal is the caller as its link stamped it; ctx.incarnation the calling host's run (R20). + whoami: async (ctx) => `${ctx.principal?.kind} from ${ctx.incarnation.slice(0, 8)}…`, + upload: async ({ name, data }) => { + stored.set(name, data); + return data.byteLength; + }, + download: async ({ name }) => stored.get(name) ?? new Uint8Array(), + // A PluginError crosses with its code, message and details; any other throw as code `internal`. + quota: async () => { + throw new PluginError("quota", "the store is full", { limit: 1 }); + }, + // A handler's context is aborted when its caller cancels or releases the reference. + slow: (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => { + console.log("server: slow aborted:", (ctx.abortSignal?.reason as Error).message); + resolve(null); + }), + ), + }, + })); +}); + +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); +await server.load([{ name: "@example/files", facet }], context); +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "owner" } }, context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +client.connect(down, { principal: { kind: "server" } }, context); +await client.load([], context); + +const files = await client.get(Files, {}, context); + +// 1. A call resolves with its result; a method without arguments takes only the context. +console.log("whoami:", await files.whoami(context)); + +// 2. Bytes: 1 MiB goes out as one raw segment next to the JSON part. +const data = new Uint8Array(1024 * 1024).fill(7); +console.log("uploaded bytes:", await files.upload({ name: "chart.png", data }, context)); +const back = await files.download({ name: "chart.png" }, context); +console.log("downloaded:", back.byteLength, "bytes, a", back.constructor.name); + +// 3. Errors: arguments are checked at the provider (a ChordError); a plugin's own error keeps its code. +try { + // @ts-expect-error: `name` must be a string + await files.upload({ name: 1, data }, context); +} catch (error) { + if (isChordError(error, "invalid")) console.log("refused:", error.reason, error.message); +} +try { + await files.quota(context); +} catch (error) { + if (error instanceof PluginError) console.log("plugin error:", error.code, error.message, error.details); +} + +// 4. Cancellation: the caller is rejected at once, the provider aborts the handler. +const { context: cancellable, cancel } = withCancel(context); +const slow = files.slow(cancellable); +setTimeout(() => cancel(new Error("the user gave up")), 5); +await slow.catch((error: unknown) => console.log("client: slow rejected:", (error as Error).message)); + +// 5. Release rejects the calls made through the reference; the provider aborts their handlers. +const pending = files.slow(context); +await new Promise((resolve) => setTimeout(resolve, 5)); +files.release(); +await pending.catch((error: unknown) => console.log("client: pending call:", (error as Error).message)); +await new Promise((resolve) => setTimeout(resolve, 5)); + +await client.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/10-drops.ts b/packages/chord/test/next/examples/10-drops.ts new file mode 100644 index 000000000..f0ca424f7 --- /dev/null +++ b/packages/chord/test/next/examples/10-drops.ts @@ -0,0 +1,115 @@ +// Surviving drops: a reference held before any link exists, `ref.state` as it follows the link, calls parked while +// loading and sent in order once live, a call in flight when the link drops, and the re-hold after `connect` again. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/10-drops.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { arr, hostKind, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { + editor: { + state: { notes: arr(str()) }, + methods: { add: method(obj({ text: str() }), int()), slow: method() }, + }, + }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const facet = defineFacet

((host) => { + // Lingers for a minute after its last reference: a client that reconnects within it finds its notes again. + host.serve( + Notes, + (_, _principal, self) => { + const doc = self.state(NotesDoc); + return { + variant: "editor", + members: { + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + slow: (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve(null))), + }, + }; + }, + { linger: 60_000 }, + ); +}); + +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); +await server.load([{ name: "@example/notes", facet }], context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +await client.load([], context); + +/** A new link between the hosts: the server stamps the client as its owner. Returns the server's end. */ +function connect(server: Host

, client: Host

, context: Context): Link { + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "owner" } }, context); + client.connect(down, { principal: { kind: "server" } }, context); + return up; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +// 1. A hold before any link: a reference at once, `loading`, its variant unknown. Calls on it park. +const notes = client.hold(Notes, { session: "S1" }, context); +console.log("held without a link:", notes.state.value, "variant:", notes.variant); +const watch = await notes.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(" ref.state →", call.value.kind); +}); +const parked = notes.add({ text: "written while loading" }, context); + +// 2. Connect: the reference is held over the link, goes live, and the parked call is sent. +let link = connect(server, client, context); +console.log("parked call's result once live:", await parked); +console.log("notes:", notes.notes.value, "variant:", notes.variant); + +// 3. The link drops (the server's end closes it). A call in flight fails `unavailable` and is never resent; the +// reference goes loading and keeps its last values. +const inFlight = notes.slow(context).catch((error: Error) => `rejected: ${error.message}`); +await settle(); +link.close(); +console.log("in-flight call:", await inFlight); +await settle(); +console.log("after the drop:", notes.state.value, "last notes:", notes.notes.value); + +// 4. Calls made while the link is down park; they wait at most the reference's readyTimeout (60 s here). +const first = notes.add({ text: "first while down" }, context); +const second = notes.add({ text: "second while down" }, context); + +// 5. The app connects again (its link adapter would): every reference is held again with fresh ids, its members get +// the new snapshot as one frame `[["r", value]]`, then the parked calls are sent in the order they were made. +const frames = await notes.notes.watch(context); +frames.start(async (call) => { + if (call.kind === "value") console.log(" notes frame:", JSON.stringify(call.ops)); +}); +link = connect(server, client, context); +console.log("parked results, in order:", await first, await second); +await settle(); +console.log("notes after the re-hold:", notes.notes.value); + +// 6. Release: `ref.state` is `ended` (its error says the holder released it) and its watch ends. +notes.release(); +console.log("watch ended:", await watch.closed, "state:", notes.state.value); +await client.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/11-linger-and-retry.ts b/packages/chord/test/next/examples/11-linger-and-retry.ts new file mode 100644 index 000000000..391d55f64 --- /dev/null +++ b/packages/chord/test/next/examples/11-linger-and-retry.ts @@ -0,0 +1,109 @@ +// `linger` and `ref.retry()`: an object that stays warm after its last reference, failed references with `retryAt`, +// a Retry button that skips the backoff, and a broken object renewed by a retry. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/11-linger-and-retry.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type RefState } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Draft = local("example.draft", { + version: 1, + params: { conversation: str() }, + members: { text: str(), type: method(obj({ text: str() })) }, +}); + +const Mirror = local("example.mirror", { version: 1, params: { id: int() }, members: { status: str() } }); + +const DraftDoc = defineState("draft", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); + +// What the mirror's open finds: nothing configured, then a server that is down, then one that answers. +const remote = { configured: false, up: false }; + +const facet = defineFacet((host) => { + // 1. A draft lingers 200 ms after its last reference: a tab that closes and reopens quickly finds its text. + host.serve( + Draft, + ({ conversation }, self) => { + console.log(` (draft ${conversation} activated)`); + const doc = self.state(DraftDoc); + return { + text: self.derive([doc], ([d]) => d.text), + type: async ({ text }, ctx) => { + await doc.change((draft) => { + draft.text = text; + }, ctx); + return null; + }, + }; + }, + { linger: 200 }, + ); + host.serve(Mirror, (_, self) => { + self.open( + async () => { + // `broken`: retrying cannot help until someone changes something (no automatic retry). + if (!remote.configured) throw broken(new Error("no mirror configured")); + // A plain throw is transient: retried with backoff (1 s, doubling). + if (!remote.up) throw new Error("mirror unreachable"); + return null; + }, + () => {}, + ); + return { status: constant("mirroring") }; + }); +}); + +const host = createHost({ onError: (error) => console.log(` onError: ${error.kind}`) }); +await host.load([{ name: "@example/app", facet }], context); +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** A state for the log: a failure with its kind, when it retries by itself, and its error. */ +function describe(state: RefState): string { + if (state.kind !== "failed") return state.kind; + const retry = state.retryAt === undefined ? "no automatic retry" : `retries in ${state.retryAt - Date.now()} ms`; + return `failed, ${state.cause.kind}, ${retry}: ${state.cause.error.message}`; +} + +// 2. Linger: release, get again within 200 ms, the same object and its in-memory document. +let draft = await host.get(Draft, { conversation: "c1" }, context); +await draft.type({ text: "hello" }, context); +draft.release(); +await wait(100); +draft = await host.get(Draft, { conversation: "c1" }, context); +console.log("within linger:", JSON.stringify(draft.text.value)); +draft.release(); +await wait(300); +draft = await host.get(Draft, { conversation: "c1" }, context); +console.log("after linger:", JSON.stringify(draft.text.value)); +draft.release(); + +// 3. A held reference follows its object: broken first (no retryAt), so nothing happens by itself. +const mirror = host.hold(Mirror, { id: 1 }, context); +const watch = await mirror.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(` ref.state → ${describe(call.value)}`); +}); +await wait(10); + +// 4. The user fixes the configuration and presses Retry: the broken object is renewed and tries again. The server is +// still down: a transient failure, retried automatically at retryAt. +remote.configured = true; +mirror.retry(); +await wait(10); + +// 5. Retry again (a Retry button): the backoff window is skipped; this time it goes live. +remote.up = true; +mirror.retry(); +await wait(10); +console.log("status:", mirror.status.value); + +// 6. A retry while live does nothing. +mirror.retry(); +await wait(10); +mirror.release(); +await host.close(context); diff --git a/packages/chord/test/next/examples/12-routes.ts b/packages/chord/test/next/examples/12-routes.ts new file mode 100644 index 000000000..e9dedbbb5 --- /dev/null +++ b/packages/chord/test/next/examples/12-routes.ts @@ -0,0 +1,161 @@ +// Routes: host kinds and addresses, a contract's host address, a server routing a session's objects to an in-process +// worker (one `resolve` per address, `authorize` per principal), a guest's `ctx.get` on the server reaching the worker +// as the guest, and a worker that drops and is resolved again. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/12-routes.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = + | { kind: "owner" } + | { kind: "guest"; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +// Kinds of addressable hosts; `Session({ id: "S1" })` is an address, its key `session|S1`. +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +// Notes live on their session's worker: the caller computes the address and sends it with its hold. +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { + editor: { + state: { notes: arr(str()) }, + methods: { add: method(obj({ text: str() }), int()), whoami: method(undefined, str()) }, + }, + reader: { state: { notes: arr(str()) }, methods: { whoami: method(undefined, str()) } }, + }, +}); + +// The lobby lives on the server: its handler gets notes for whoever calls it. +const Lobby = remote("example.lobby", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { visitor: { methods: { peek: method(obj({ session: str() }), str()) } } }, +}); + +// The worker's own store of its session's notes, shared by every principal's `Notes` object. +const NoteStore = local("example.note-store", { + version: 1, + params: {}, + members: { list: arr(str()), add: method(obj({ text: str() }), int()) }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +/** The worker's facet: an owner edits, anyone else reads; `whoami` shows whom a call acts for. */ +const workerFacet = defineFacet

((host) => { + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + list: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + }; + }); + host.serve(Notes, (_, principal, self) => { + const store = self.get(NoteStore, {}); + const whoami = async (ctx: { readonly principal: P | undefined }) => JSON.stringify(ctx.principal); + if (principal.kind !== "owner") return { variant: "reader", members: { notes: store.list, whoami } }; + return { variant: "editor", members: { notes: store.list, add: (args, ctx) => store.add(args, ctx), whoami } }; + }); +}); + +const serverFacet = defineFacet

((host) => { + host.serve(Lobby, () => ({ + variant: "visitor", + members: { + // `ctx.get` acts as the caller: over the route, the worker sees the guest, not the server. + peek: async ({ session }, ctx) => { + const notes = await ctx.get(Notes, { session }); + return `${notes.variant} variant, ${await notes.whoami(ctx)}`; + }, + }, + })); +}); + +// 1. The server: an address, and a route for sessions. `resolve` starts an in-process worker over a memory link and +// returns the link at once (the worker connects first, then loads, W6). `authorize` admits owners everywhere and a +// guest only into its own session. +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); +let resolves = 0; +let workerLink: Link | undefined; +server.route( + Session, + { + authorize: (principal, address) => + principal.kind === "owner" || (principal.kind === "guest" && principal.session === address.params.id), + resolve: async (address) => { + console.log(`server: resolving ${address.key} (resolve #${++resolves})`); + const [up, down] = memoryLinkPair(); + const worker = createHost

({ + address, + principal: { kind: "worker", session: address.params.id }, + onError: console.error, + }); + worker.connect(down, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@example/worker", facet: workerFacet }], context); + workerLink = up; + return { link: up, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([{ name: "@example/server", facet: serverFacet }], context); + +/** A tab on the server, stamped as `principal` by the server's entry point. */ +async function tabAs(principal: P): Promise> { + const [up, down] = memoryLinkPair(); + server.accept(up, { principal }, context); + const tab = createHost

({ onError: console.error }); + tab.connect(down, { principal: { kind: "server" } }, context); + await tab.load([], context); + return tab; +} + +// 2. The owner's tab gets S1's notes: tab → server (relays) → route → worker. A second get shares the worker. +const owner = await tabAs({ kind: "owner" }); +const notes = await owner.get(Notes, { session: "S1" }, context); +if (notes.variant === "editor") await notes.add({ text: "milk" }, context); +const again = await owner.get(Notes, { session: "S1" }, context); +console.log("owner:", notes.variant, again.notes.value, "resolves so far:", resolves); + +// 3. A guest of S1 reads S1 and is refused S2 by `authorize` (no worker is started for it). +const guest = await tabAs({ kind: "guest", session: "S1" }); +const read = await guest.get(Notes, { session: "S1" }, context); +console.log("guest:", read.variant, read.notes.value, "acts as", await read.whoami(context)); +await guest.get(Notes, { session: "S2" }, context).catch((error) => { + console.log("guest, S2:", isChordError(error) ? error.code : error, "resolves so far:", resolves); +}); + +// 4. The guest calls the lobby on the server; its handler's `ctx.get` goes over the route as the guest. +const lobby = await guest.get(Lobby, {}, context); +console.log("guest via the lobby:", await lobby.peek({ session: "S1" }, context)); + +// 5. The worker drops: references go loading, the route resolves the address again at once (its link had completed +// its handshake), and the new worker (a fresh run, empty notes here) binds them again. +const states: string[] = []; +const watch = await notes.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); +}); +await new Promise((resolve) => setTimeout(resolve, 0)); +workerLink!.close(); +while (states.length < 2 || states.at(-1) !== "live") await new Promise((resolve) => setTimeout(resolve, 50)); +console.log("after the drop:", states.join(" → "), "notes:", notes.notes.value, "resolves:", resolves); + +await owner.close(context); +await guest.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/13-relay.ts b/packages/chord/test/next/examples/13-relay.ts new file mode 100644 index 000000000..f132e76f8 --- /dev/null +++ b/packages/chord/test/next/examples/13-relay.ts @@ -0,0 +1,146 @@ +// A relay over real transports: a tab on a loopback WebSocket, a server that relays, and a worker per session forked +// over IPC. The worker gets its start data from its start message and serves it as `SessionInfo` through its core +// facet (§18); the server routes by address and parses no part. +// Run from packages/chord (the same file is the worker when its first argument is `worker`): +// node --conditions=source --experimental-strip-types test/next/examples/13-relay.ts +import { fork } from "node:child_process"; +import type { AddressInfo } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type IpcPort, ipcLink, webSocketLink } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Init = obj({ id: str(), title: str(), pid: int() }); + +/** Start data: a local object with empty params (host lifetime) and constant state, served by the core facet. */ +const SessionInfo = local("example.session-info", { version: 1, params: {}, members: { init: Init } }); + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { + owner: { state: { info: Init, notes: arr(str()) }, methods: { add: method(obj({ text: str() }), int()) } }, + }, +}); + +/** What the server sends a worker it forked: the app's own message, not the link's. */ +type Start = { readonly id: string; readonly title: string }; + +/** The worker's core facet, made from its start message (base code, loaded under a fixed name). */ +const coreWorkerFacet = (init: { id: string; title: string; pid: number }) => + defineFacet

((host) => { + host.serve(SessionInfo, () => ({ init: constant(init) })); + }); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +/** A plugin's facet on the worker: its notes, and the start data read through a reference. */ +const notesFacet = defineFacet

((host) => { + host.serve(Notes, (_, _principal, self) => { + const doc = self.state(NotesDoc); + const info = self.get(SessionInfo, {}); + return { + variant: "owner", + members: { + info: info.init, + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + }, + }; + }); +}); + +if (process.argv[2] === "worker") { + // The worker. 1. Its link first: it queues the link's messages until the host listens, and skips the app's own. + const link = ipcLink(process as unknown as IpcPort); + const start = await new Promise((resolve) => { + const listener = (message: unknown) => { + if (message instanceof Uint8Array) return; + process.off("message", listener); + // The start message the server's `resolve` sent: the app's own shape. + resolve(message as Start); + }; + process.on("message", listener); + }); + // 2. Its host at its address: connect first (W6), then load the core facet with the start data, then the plugins. + const worker = createHost

({ + address: Session({ id: start.id }), + principal: { kind: "worker", session: start.id }, + onError: console.error, + }); + worker.connect(link, { principal: { kind: "server" }, trusted: true }, context); + await worker.load( + [ + { name: "@example/core", facet: coreWorkerFacet({ ...start, pid: process.pid }) }, + { name: "@example/notes", facet: notesFacet }, + ], + context, + ); + // The server closing the link ends the worker. + link.onClose(() => process.exit(0)); +} else { + // The server. 3. A route forks a worker per session, sends it its start message and returns the IPC link at once. + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); + server.route( + Session, + { + authorize: (principal) => principal.kind === "owner", + // No `signal` for the fork: the context is aborted when the host closes, which would kill the worker (and emit + // `error` on the child); a link Chord closes, at `host.close` or because it arrived after it, ends the worker. + resolve: async (address) => { + const child = fork(fileURLToPath(import.meta.url), ["worker"], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + child.send({ id: address.params.id, title: `Session ${address.params.id}` } satisfies Start); + console.log(`server: forked pid ${child.pid} for ${address.key}`); + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, + ); + await server.load([], context); + + // 4. Tabs reach the server over a WebSocket; its entry point stamps each as the owner here. + const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await new Promise((resolve) => wss.once("listening", resolve)); + wss.on("connection", (socket) => server.accept(webSocketLink(socket), { principal: { kind: "owner" } }, context)); + const { port } = wss.address() as AddressInfo; + + // 5. A tab: Node's own WebSocket, as a browser's. Its holds go tab → server (relay) → worker. + const tab = createHost

({ onError: console.error }); + tab.connect(webSocketLink(new WebSocket(`ws://127.0.0.1:${port}`)), { principal: { kind: "server" } }, context); + await tab.load([], context); + const s1 = await tab.get(Notes, { session: "S1" }, context); + console.log("tab: S1 info", s1.info.value, "server pid", process.pid); + await s1.add({ text: "milk" }, context); + await s1.add({ text: "bread" }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + console.log("tab: S1 notes", s1.notes.value); + + // 6. Another session is another worker process. + const s2 = await tab.get(Notes, { session: "S2" }, context); + console.log("tab: S2 info", s2.info.value, "notes", s2.notes.value); + + // 7. Closing the server closes its route links; each worker exits when its link closes. + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + console.log("closed"); +} diff --git a/packages/chord/test/next/examples/14-flow-control.ts b/packages/chord/test/next/examples/14-flow-control.ts new file mode 100644 index 000000000..184133150 --- /dev/null +++ b/packages/chord/test/next/examples/14-flow-control.ts @@ -0,0 +1,208 @@ +// Flow control: a slow tab. At most two of a provider's own `bound`/`state` messages are unacknowledged per origin; +// while the window is full the provider folds the ops of every commit into its pending queues and sends them as one +// message when an `ack` frees a slot. A relay folds and drops nothing: it forwards at once under the tab's origin and +// returns each `ack` to the worker that sent the message, so a slow tab folds at the worker, alone. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/14-flow-control.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { hostKind, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { binaryCodec, describeMessage, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +/** The worker's own document of a streamed reply, and its one write. */ +const Reply = local("example.reply", { + version: 1, + params: {}, + members: { reply: obj({ text: str() }), append: method(str()) }, +}); + +/** What a tab holds: the reply, on its session's worker. */ +const View = remote("example.view", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { owner: { state: { reply: obj({ text: str() }) } } }, +}); + +const ReplyDoc = defineState("reply", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); + +const workerFacet = defineFacet

((host) => { + host.serve(Reply, (_, self) => { + const doc = self.state(ReplyDoc); + return { + reply: doc, + append: async (token, ctx) => { + await doc.change((draft) => { + draft.text += token; + }, ctx); + return null; + }, + }; + }); + host.serve(View, (_params, _principal, self) => ({ + variant: "owner", + members: { reply: self.get(Reply, {}).reply }, + })); +}); + +/** A worker for `id`, loaded. */ +async function worker(id: string): Promise> { + const host = createHost

({ + address: Session({ id }), + principal: { kind: "worker", session: id }, + onError: console.error, + }); + await host.load([{ name: "@example/worker", facet: workerFacet }], context); + return host; +} + +/** A link end that logs, as `label`, every `bound`, `state` and `ack` its host receives. */ +function tap(link: Link, label: string): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + const decoded = binaryCodec.decode(message); + const shown = decoded.kind === "ack" || decoded.kind === "bound" || decoded.kind === "state"; + if (shown) console.log(` ${label} ←`, describeMessage(decoded).slice(0, 110)); + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** A slow tab's end of its link: while paused, what arrives waits (so the tab acknowledges nothing). */ +class Pausable implements Link { + paused = false; + readonly #link: Link; + readonly #waiting: Uint8Array[] = []; + #listener: (message: Uint8Array) => void = () => {}; + + constructor(link: Link) { + this.#link = link; + link.onMessage((message) => { + if (this.paused) this.#waiting.push(message); + else this.#listener(message); + }); + } + + resume(): void { + this.paused = false; + for (const message of this.#waiting.splice(0)) this.#listener(message); + } + + send(message: Uint8Array): void { + this.#link.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#listener = listener; + } + + close(): void { + this.#link.close(); + } + + onClose(listener: () => void): void { + this.#link.onClose(listener); + } +} + +/** A tab connected over `link`, loaded. */ +async function tab(link: Link): Promise> { + const host = createHost

({ onError: console.error }); + host.connect(link, { principal: { kind: "server" } }, context); + await host.load([], context); + return host; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +// 1. Folding at the provider: a tab directly on a worker. Paused, it acknowledges nothing; the worker sends two +// `state` messages (the window), then folds 1,000 appends into its pending queue. +console.log("a slow tab on the provider:"); +const w1 = await worker("S1"); +const [up, down] = memoryLinkPair(); +w1.accept(up, { principal: { kind: "owner" } }, context); +const slow = new Pausable(down); +const t1 = await tab(tap(slow, "tab")); +const view1 = await t1.get(View, { session: "S1" }, context); +const reply1 = await w1.get(Reply, {}, context); +slow.paused = true; +for (let i = 0; i < 1_000; i++) await reply1.append(" tok", context); +await settle(); +console.log( + " paused: the tab shows", + view1.reply.value.text.length, + "characters, the worker has", + reply1.reply.value.text.length, +); + +// 2. Resumed: the tab applies and acknowledges the two, and the freed slot carries one `state` with one folded append. +slow.resume(); +await settle(); +console.log(" resumed: the tab shows", view1.reply.value.text.length, "characters"); + +// 3. The same slow tab through a server: its holds reach the worker with the tab's ids under the tab link's origin +// (`["@",1,…]`), the worker's window for that origin fills and it folds there; a second, fast tab is another origin and +// receives every commit. +console.log("a slow tab through a relay:"); +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); +let w2: Host

| undefined; +server.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const [a, b] = memoryLinkPair(); + w2 = await worker(address.params.id); + w2.connect(tap(b, "worker"), { principal: { kind: "server" }, trusted: true }, context); + return { link: a, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([], context); +const [up2, down2] = memoryLinkPair(); +server.accept(up2, { principal: { kind: "owner" } }, context); +const slow2 = new Pausable(down2); +const t2 = await tab(tap(slow2, "tab")); +const view2 = await t2.get(View, { session: "S2" }, context); +const [up3, down3] = memoryLinkPair(); +server.accept(up3, { principal: { kind: "owner" } }, context); +const t3 = await tab(down3); +const view3 = await t3.get(View, { session: "S2" }, context); +let fast = 0; +const watch = await view3.reply.watch(context); +watch.start(async () => { + fast++; +}); +const reply2 = await w2!.get(Reply, {}, context); +slow2.paused = true; +for (let i = 0; i < 5; i++) await reply2.append(` t${i}`, context); +await settle(); +console.log(" paused: the slow tab shows", view2.reply.value, "the worker has", reply2.reply.value); +console.log(" the fast tab saw", fast, "changes:", view3.reply.value); + +// 4. Resumed: the slow tab acknowledges the two; the relay returns each `ack` to the worker with the origin, and the +// worker's freed slot carries one `state` with the three folded appends. +slow2.resume(); +await settle(); +console.log(" resumed: the slow tab shows", view2.reply.value); + +await t1.close(context); +await w1.close(context); +await watch.stop(); +await t2.close(context); +await t3.close(context); +await server.close(context); +await w2!.close(context); diff --git a/packages/chord/test/next/examples/15-revocation.ts b/packages/chord/test/next/examples/15-revocation.ts new file mode 100644 index 000000000..9ee2a3648 --- /dev/null +++ b/packages/chord/test/next/examples/15-revocation.ts @@ -0,0 +1,142 @@ +// Revocation: the app says which revocations apply to a principal (`revokedBy`, the same function on every host); +// `host.revoke(r)` ends that principal's objects at once, whatever references remain, closes its child links (tabs) and +// sends `revoke` down every route link, where each worker applies it with its own `revokedBy`. Nothing is replayed. +// What ends gets the error `forbidden` with reason `revoked` (a UI can say "access revoked"): holders get it in `gone`, +// running handlers in their aborted context. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/15-revocation.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import type { ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** The app's principals: a guest holds a grant to one session; the server and its workers act as themselves. */ +type P = + | { kind: "owner" } + | { kind: "guest"; grant: string; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +/** What the app revokes: a grant, or everything of a session. */ +type Revocation = { grant: string } | { session: string }; + +/** The same function on every host: a guest is revoked by its grant and by its session. */ +const revokedBy = (p: P): Revocation[] => (p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] : []); + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { + all: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() })), follow: method() } }, + }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const notesFacet = defineFacet

((host) => { + host.serve(Notes, (_, principal, self) => { + const doc = self.state(NotesDoc); + console.log(` worker: Notes for ${principal.kind}${principal.kind === "guest" ? ` ${principal.grant}` : ""}`); + return { + variant: "all", + members: { + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return null; + }, + // A long-running call: it ends when its context is aborted, and says why. + follow: async (ctx) => { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + const { code, reason } = ctx.abortSignal!.reason as ChordError; + console.log(` worker: ${principal.kind}'s follow aborted: ${code}/${reason}`); + return null; + }, + }, + }; + }); +}); + +// 1. A server with `revokedBy`, routing sessions to in-process workers (trusted links) with the same `revokedBy`. +const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy, + onError: console.error, +}); +const workers: Host[] = []; +server.route( + Session, + { + authorize: (p, address) => p.kind === "owner" || (p.kind === "guest" && p.session === address.params.id), + resolve: async (address) => { + const [up, down] = memoryLinkPair(); + const worker = createHost({ + address, + principal: { kind: "worker", session: address.params.id }, + revokedBy, + onError: console.error, + }); + worker.connect(up, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@example/notes", facet: notesFacet }], context); + workers.push(worker); + return { link: down, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([], context); + +/** A tab whose link the server's entry point stamped with `principal`. */ +async function tab(principal: P) { + const host = createHost

({ onError: console.error }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return host; +} + +// 2. Two tabs of guest grant g1 (one person, two windows) and the owner's tab, all on session S1. +const [one, two, owner] = await Promise.all([ + tab({ kind: "guest", grant: "g1", session: "S1" }), + tab({ kind: "guest", grant: "g1", session: "S1" }), + tab({ kind: "owner" }), +]); +const guestNotes = await one.get(Notes, { session: "S1" }, context); +await two.get(Notes, { session: "S1" }, context); +const ownerNotes = await owner.get(Notes, { session: "S1" }, context); +await guestNotes.add({ text: "from the guest" }, context); +const following = guestNotes.follow(context).catch((error: ChordError) => error.code); +// The call reaches the worker and runs there. +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("worker S1 references:", workers[0]!.references(Notes).value.length); + +// 3. Revoke the grant: the worker gets `revoke { grant: "g1" }` and ends the guest's objects there (their running +// calls aborted with forbidden/revoked), and both guest tabs' links close. +server.revoke({ grant: "g1" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("guest tab:", guestNotes.state.value.kind, "(its link closed; a new one needs a new authentication)"); +console.log("guest's follow call:", await following); +// The owner's Notes is its own object (identities include the principal), untouched. +console.log("owner tab:", ownerNotes.state.value.kind, ownerNotes.notes.value); +console.log( + "worker S1 references:", + workers[0]!.references(Notes).value.map(({ principal }) => principal?.kind), +); + +// 4. A worker resolved later gets no past revocation: its objects are new, and the revoked links are closed already. +const s2 = await owner.get(Notes, { session: "S2" }, context); +console.log("owner on S2:", s2.state.value.kind); + +await Promise.all([one, two, owner].map((host) => host.close(context))); +await server.close(context); +await Promise.all(workers.map((worker) => worker.close(context))); diff --git a/packages/chord/test/next/examples/16-presence-and-activity.ts b/packages/chord/test/next/examples/16-presence-and-activity.ts new file mode 100644 index 000000000..ba69803cf --- /dev/null +++ b/packages/chord/test/next/examples/16-presence-and-activity.ts @@ -0,0 +1,93 @@ +// Presence and keep-alive counts. `host.references(Type)` is a JSON watchable of the references to objects of a type +// on this host, one entry per reference with the object's params, its principal and its incarnation (two tabs are two +// origins); a derive over it is a viewer count, and a link drop is a departure in the same batch. `host.activity` counts +// what keeps a host busy (holds over links, running calls, jobs, lingering objects), published at most once per +// microtask turn: the app's supervisor decides when an idle worker stops. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/16-presence-and-activity.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { hostKind, int, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "user"; name: string } | { kind: "server" }; + +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + +/** A shared document view: who else is looking (`viewers`), and a slow method. */ +const Doc = remote("example.doc", { + version: 1, + params: { id: str() }, + hostAddress: () => Server(), + variants: { v: { state: { viewers: int() }, methods: { think: method(undefined, int()) } } }, +}); + +// 1. The server serves `Doc`; its factory derives the viewer count from `host.references(Doc)` (on the facet's host). +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); +const facet = defineFacet

((host) => { + host.serve( + Doc, + ({ id }, _principal, self) => ({ + variant: "v", + members: { + viewers: self.derive([host.references(Doc)], ([refs]) => refs.filter((ref) => ref.params.id === id).length), + think: async (ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return ctx.principal?.kind === "user" ? 42 : 0; + }, + }, + }), + { linger: 50 }, + ); +}); +await server.load([{ name: "@example/doc", facet }], context); + +// 2. The supervisor's view: every publication of the counts. +const activity = await server.activity.watch(context); +activity.start(async (call) => { + if (call.kind === "value") console.log(" activity:", JSON.stringify(call.value)); +}); + +/** A tab of `name` on a new link; returns the tab and the server's end of its link. */ +async function tab(name: string) { + const host = createHost

({ onError: console.error }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "user", name } }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return { host, link: a }; +} + +// 3. Ada opens the document in two tabs, Bob in one: three references, three incarnations, two objects (one per principal); +// the viewer count is over every object of the document. +const [ada1, ada2, bob] = await Promise.all([tab("ada"), tab("ada"), tab("bob")]); +const doc = await ada1.host.get(Doc, { id: "d1" }, context); +await ada2.host.get(Doc, { id: "d1" }, context); +await bob.host.get(Doc, { id: "d1" }, context); +await new Promise((resolve) => setTimeout(resolve, 5)); +console.log("viewers seen by ada:", doc.viewers.value); +const present = server + .references(Doc) + .value.map( + ({ principal, incarnation }) => `${principal?.kind === "user" ? principal.name : "?"} @ ${incarnation.slice(-4)}`, + ); +console.log("references:", present); + +// 4. A call running counts as activity until it settles. +await doc.think(context); + +// 5. Bob's link drops: a departure, at once. Bob's own `Doc` object (identities include the principal) lingers. +bob.link.close(); +await new Promise((resolve) => setTimeout(resolve, 5)); +console.log("viewers after bob's link dropped:", doc.viewers.value); + +// 6. Ada closes both tabs: the object lingers (50 ms) with no references, then ends; then everything is zero. +await ada1.host.close(context); +await ada2.host.close(context); +await new Promise((resolve) => setTimeout(resolve, 100)); +console.log("references at the end:", server.references(Doc).value.length); +await activity.stop(); +await server.close(context); diff --git a/packages/chord/test/next/examples/17-limits-and-compression.ts b/packages/chord/test/next/examples/17-limits-and-compression.ts new file mode 100644 index 000000000..da3dbe0a9 --- /dev/null +++ b/packages/chord/test/next/examples/17-limits-and-compression.ts @@ -0,0 +1,199 @@ +// Size limits per part and compression at the origin. A host with a synchronous compressor (`nodeCompression` on +// Node) deflates each JSON part above 16 KiB once, when it encodes it; relays forward it as it is; the end that owns it +// inflates it, within its link's decoded limit. A browser has no synchronous deflate: it sends uncompressed and +// inflates with `DecompressionStream`, later messages of its link waiting in order. Only a part that shrinks is sent +// deflated, and never hold params. Clients never send compressed parts: a host sends its call arguments up its parent +// link uncompressed, and a link the server accepted (a child) refuses compressed parts as a protocol error (a small +// part could inflate to 64 MiB there). A part past a limit fails only its own hold or call (`too_large/part`); the +// link stays. Limits and the threshold count UTF-8 bytes. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/17-limits-and-compression.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "user" } | { kind: "server" }; + +const Entry = obj({ id: str(), text: str() }); + +/** The shared store of a conversation: its entries, once, and the method that appends to them. */ +const Store = local("example.store", { + version: 1, + params: { id: str() }, + members: { entries: arr(Entry), append: method(Entry) }, +}); + +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + +/** What a tab holds: the conversation's entries, and appending. */ +const Conversation = remote("example.conversation", { + version: 1, + params: { id: str() }, + hostAddress: () => Server(), + variants: { v: { state: { entries: arr(Entry) }, methods: { append: method(Entry) } } }, +}); + +const Entries = defineState("entries", { + schema: obj({ list: arr(Entry) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Entries); + return { + entries: self.derive([doc], ([d]) => d.list), + append: async (entry, ctx) => { + await doc.change((draft) => void draft.list.push(entry), ctx); + return null; + }, + }; + }); + host.serve(Conversation, ({ id }, _principal, self) => { + const store = self.get(Store, { id }); + return { variant: "v", members: { entries: store.entries, append: (entry, ctx) => store.append(entry, ctx) } }; + }); +}); + +/** A link end counting the bytes its host receives. */ +function counted(link: Link, bytes: { received: number }): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + bytes.received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** + * A server compressing with `compression` and a tab without a compressor (a browser) on a link the server accepts with + * `limits`; the tab's own link to the server has `tabLimits`. + */ +async function setUp(compression: Compression | undefined, limits?: PartLimits, tabLimits?: PartLimits) { + const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + compression, + onError: console.error, + }); + await server.load([{ name: "@example/conversation", facet }], context); + const tab = createHost

({ onError: console.error }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + server.accept(a, { principal: { kind: "user" }, limits }, context); + tab.connect(counted(b, bytes), { principal: { kind: "server" }, limits: tabLimits }, context); + return { server, tab, bytes }; +} + +/** Words for entries that compress like prose (about 3x), not like a repeated string. */ +const WORDS = "the a model tool call file read write test error result agent session note code run build".split(" "); + +/** Entry `i`'s text: about 1 KB of words picked by a fixed pseudo-random sequence. */ +function text(i: number): string { + let seed = i * 7919 + 1; + const words: string[] = []; + for (let length = 0; length < 1000; length += words.at(-1)!.length + 1) { + seed = (seed * 1103515245 + 12345) % 2147483648; + const word = WORDS[seed % WORDS.length]!; + words.push(seed % 5 === 0 ? `${word}${seed % 1000}` : word); + } + return words.join(" "); +} + +/** Fills the server's conversation `id` with 500 entries of about 1 KB (its store kept by this reference). */ +async function fill(server: Host

, id: string): Promise { + const conversation = await server.get(Store, { id }, context); + for (let i = 0; i < 500; i++) await conversation.append({ id: `e${i}`, text: text(i) }, context); +} + +// 1. A 500 KB snapshot to a browser tab, without and with compression at the server. +for (const compression of [undefined, nodeCompression]) { + const { server, tab, bytes } = await setUp(compression); + await fill(server, "c1"); + const started = performance.now(); + const conversation = await tab.get(Conversation, { id: "c1" }, context); + const ms = (performance.now() - started).toFixed(1); + const label = compression === undefined ? "uncompressed" : "deflated at the server, inflated asynchronously"; + console.log( + `snapshot ${label}: ${conversation.entries.value.length} entries, ${(bytes.received / 1024).toFixed(1)} KiB received, ${ms} ms`, + ); + await tab.close(context); + await server.close(context); +} + +// 2. Limits per part: the server admits at most 64 KiB per part from this tab. A larger argument fails its call only. +{ + const { server, tab } = await setUp(nodeCompression, { part: 64 * 1024 }); + const conversation = await tab.get(Conversation, { id: "c2" }, context); + try { + await conversation.append({ id: "big", text: "x".repeat(100_000) }, context); + } catch (error) { + console.log( + "a 100 KB argument:", + isChordError(error) ? `${error.code}/${error.reason}: ${error.message}` : error, + ); + } + await conversation.append({ id: "small", text: "fits" }, context); + console.log( + "the link stays: entries", + conversation.entries.value.map(({ id }) => id), + ); + await tab.close(context); + await server.close(context); +} + +// 3. A compression bomb: 50 KB deflated that would inflate to 6 MB, past the tab's decoded limit of 1 MiB. The tab +// stops inflating at the limit; only that hold fails. +{ + const { server, tab } = await setUp(nodeCompression, undefined, { decoded: 1024 * 1024 }); + const bomb = await server.get(Store, { id: "bomb" }, context); + for (let i = 0; i < 6; i++) await bomb.append({ id: `z${i}`, text: "0".repeat(1_000_000) }, context); + await fill(server, "fine"); + try { + await tab.get(Conversation, { id: "bomb" }, context); + } catch (error) { + console.log("the bomb's hold:", isChordError(error) ? `${error.code}/${error.reason}` : error); + } + const fine = await tab.get(Conversation, { id: "fine" }, context); + console.log("another hold on the same link:", fine.entries.value.length, "entries"); + await tab.close(context); + await server.close(context); +} + +// 4. A client with a compressor still sends its arguments up uncompressed: the server, which accepted its link as a +// child, would refuse a compressed part. Its 100 KB argument arrives as JSON, within the server's default limits. +{ + const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + compression: nodeCompression, + onError: console.error, + }); + await server.load([{ name: "@example/conversation", facet }], context); + const client = createHost

({ compression: nodeCompression, onError: console.error }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + server.accept(counted(a, bytes), { principal: { kind: "user" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + const conversation = await client.get(Conversation, { id: "c4" }, context); + await conversation.append({ id: "big", text: text(0).repeat(100) }, context); + console.log( + `a compressing client's 100 KB argument: ${(bytes.received / 1024).toFixed(1)} KiB received by the server`, + ); + await client.close(context); + await server.close(context); +} diff --git a/packages/chord/test/next/examples/18-routines.ts b/packages/chord/test/next/examples/18-routines.ts new file mode 100644 index 000000000..8ff6b8d65 --- /dev/null +++ b/packages/chord/test/next/examples/18-routines.ts @@ -0,0 +1,115 @@ +// Routines: an object's standing behaviour, declared in its factory. `self.on(source, fn)` reconciles from a value: the +// first call gets the current value (`[["r", value]]`), then one call per change, never overlapping; a call that +// throws is reported (`routine_failed`) and called again with the whole current value after a backoff (1 s doubling to +// 60 s), or at once when the source changes. `self.onLive(fn)` runs once the object is live: a loop with `sleep` is a +// periodic task; its ctx is aborted when the object ends, which `sleep` turns into a rejection, so nothing is left +// running. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/18-routines.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bool, local, map, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** The catalog of sessions: another owner's state, which the forwards reconcile from. */ +const Catalog = local("example.catalog", { + version: 1, + params: {}, + members: { sessions: map(obj({ archived: bool() })), archive: method(obj({ session: str() })) }, +}); + +/** Port forwards per session; a forward of an archived session must be closed (an invariant across owners, §9). */ +const ForwardStore = local("example.forwards", { + version: 1, + params: {}, + members: { open: arr(str()), cleanups: arr(str()) }, +}); + +const Sessions = defineState("example.sessions", { + schema: obj({ sessions: map(obj({ archived: bool() })) }), + version: 1, + initial: () => ({ sessions: { s1: { archived: false }, s2: { archived: false } } }), +}); + +const Forwards = defineState("example.forward-list", { + schema: obj({ open: arr(str()), cleanups: arr(str()) }), + version: 1, + initial: () => ({ open: ["s1", "s2"], cleanups: [] }), +}); + +const started = Date.now(); +const at = () => `${Math.round((Date.now() - started) / 100) / 10}s`; + +// 1. The host reports background failures: a routine call that threw arrives as `routine_failed`. +const host = createHost({ + onError: (error) => { + if (error.kind === "routine_failed") { + const { kind, position } = error.routine; + console.log(` [${at()}] ${error.object} ${kind} #${position} failed: ${(error.error as Error).message}`); + } else console.error(error); + }, +}); + +let flaky = true; +const facet = defineFacet((h) => { + h.serve(Catalog, (_, self) => { + const doc = self.state(Sessions); + return { + sessions: self.derive([doc], ([d]) => d.sessions), + archive: async ({ session }, ctx) => { + await doc.change((d) => { + d.sessions[session] = { archived: true }; + }, ctx); + return null; + }, + }; + }); + h.serve(ForwardStore, (_, self) => { + const forwards = self.state(Forwards); + const catalog = self.get(Catalog, {}); + // 2. Reconcile: close every forward of an archived session. Idempotent, from the current value; the first close + // fails once (a flaky remote), and the retry comes with the whole value. + self.on(catalog.sessions, async ({ value: sessions, ops }, ctx) => { + console.log(` [${at()}] on: ${JSON.stringify(ops)}`); + for (const session of forwards.value.open) { + if (!sessions[session]?.archived) continue; + if (flaky) { + flaky = false; + throw new Error(`closing the forward of ${session} timed out`); + } + await forwards.change((d) => { + d.open = d.open.filter((open) => open !== session); + }, ctx); + console.log(` [${at()}] closed the forward of ${session}`); + } + }); + // 3. A periodic task: a loop with `sleep`, ended by the object's end (the abort rejects `sleep`). + self.onLive(async (ctx) => { + for (;;) { + await sleep(400, ctx); + await forwards.change((d) => void d.cleanups.push(at()), ctx); + } + }); + return { + open: self.derive([forwards], ([d]) => d.open), + cleanups: self.derive([forwards], ([d]) => d.cleanups), + }; + }); +}); +await host.load([{ name: "@example/forwards", facet }], context); +const store = await host.get(ForwardStore, {}, context); +const catalog = await host.get(Catalog, {}, context); +await sleep(50, context); + +// 4. Archive s2: the routine's call throws once, is retried after 1 s with the whole value, and closes the forward. +console.log("archiving s2"); +await catalog.archive({ session: "s2" }, context); +await sleep(1_300, context); +console.log("open forwards:", store.open.value); +console.log("periodic cleanups so far:", store.cleanups.value); + +// 5. Closing the host aborts the routines: the periodic loop ends in its `sleep`, nothing is reported. +await host.close(context); +console.log("closed; nothing left running"); diff --git a/packages/chord/test/next/examples/19-jobs.ts b/packages/chord/test/next/examples/19-jobs.ts new file mode 100644 index 000000000..94c9646ca --- /dev/null +++ b/packages/chord/test/next/examples/19-jobs.ts @@ -0,0 +1,219 @@ +// Jobs: owned work a call starts that outlives the call, in Durable's sandwich: intent (`start`, one transaction), run +// (any effect, outside transactions) and finish (one transaction on the current code, with the cleanup). Here a shell +// job of a shared chat store: its state document carries the output, throttled by the author and flushed in a +// `finally`, so the last output reaches `outcome.state` after an abort too; `running` is ordinary state, which +// per-principal views filter (a guest sees no output); `finish` records an entry and chains a second job. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/19-jobs.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, type Static, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; +import { OutputBuffer, Progress } from "../../../src/output.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "guest" } | { kind: "server" }; + +// 1. The job: closed args (checked at every start), a result, a state document per run, one run at a time. +const ShellState = defineState("example.shell.state", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); +const ShellJob = defineJob("example.shell", { + version: 1, + args: obj({ command: str(), lines: int() }), + result: obj({ exit: int(), output: str() }), + state: ShellState, + concurrency: 1, +}); +/** A second job `finish` chains: it announces a finished command. */ +const NotifyJob = defineJob("example.notify", { version: 1, args: obj({ text: str() }), result: str() }); + +const Entries = defineState("example.entries", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** A shell as its owner sees it; a guest's view has no output. */ +const Shell = obj({ id: str(), phase: str(), command: str(), output: str() }); +const GuestShell = obj({ id: str(), phase: str(), command: str() }); + +/** The shared store: the entries and the shells once, whoever views them (§6). */ +const ChatStore = local("example.chat-store", { + version: 1, + params: {}, + members: { + entries: arr(str()), + shells: arr(Shell), + runShell: method(obj({ command: str(), lines: int(), requestId: str() }), str()), + abortShell: method(obj({ id: str() })), + }, +}); + +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + +/** Per-principal views: the owner runs shells and sees their output, a guest sees which shells run. */ +const Chat = remote("example.chat", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + owner: { + state: { entries: arr(str()), shells: arr(Shell) }, + methods: { + runShell: method(obj({ command: str(), lines: int(), requestId: str() }), str()), + abortShell: method(obj({ id: str() })), + }, + }, + guest: { state: { entries: arr(str()), shells: arr(GuestShell) } }, + }, +}); + +/** A fake process: one line every 30 ms; `sleep` rejects when the run is aborted. */ +async function runProcess(lines: number, ctx: Context, onLine: (line: string) => void): Promise { + for (let n = 1; n <= lines; n++) { + await sleep(30, ctx); + onLine(`line ${n}\n`); + } + return 0; +} + +const facet = defineFacet

((host) => { + host.serve(ChatStore, (_, self) => { + const entries = self.state(Entries); + const notify = self.job(NotifyJob, { + run: async ({ text }) => `notified: ${text}`, + finish: async (outcome, tx) => { + if (outcome.kind === "done") (await tx.draft(entries)).list.push(outcome.result); + }, + }); + const shell = self.job(ShellJob, { + run: async ({ lines }, ctx) => { + // The output's bounds are required: a tail of at most 2,000 lines and 64 KiB. + const out = new OutputBuffer({ maxBytes: 64 * 1024, maxLines: 2_000, retain: "tail" }); + // Every state change is a commit and, while watched, a message: the author throttles them (50 ms here). + const write = async () => { + const text = out.snapshot().text; + await ctx.state.change((draft) => { + draft.output = text; + }, ctx); + return text.length; + }; + const progress = new Progress(write, (error) => console.error("progress failed:", error), 50); + try { + const exit = await runProcess(lines, ctx, (line) => { + out.push(line); + progress.mark(); + }); + return { exit, output: out.snapshot().text }; + } finally { + // The last output reaches `outcome.state`, also after an abort (changes are taken until run settles). + await progress.stop(); + await write(); + } + }, + finish: async (outcome, tx) => { + const { command } = outcome.args; + const lines = outcome.state.output.split("\n").length - 1; + const text = + outcome.kind === "done" + ? `${command}: exit ${outcome.result.exit}, ${lines} lines` + : `${command}: interrupted (${outcome.error.kind}), ${lines} lines kept`; + (await tx.draft(entries)).list.push(text); + // Chaining: the next job starts atomically with this outcome (both commit or neither). + tx.start(notify, { text: command }); + }, + }); + // What the views read: one object per run, kept while the run's entry is the same object (`mapShared`). + const toShell = mapShared( + (entry: (typeof shell.running.value)[number]): Static => + entry.phase === "blocked" + ? { id: entry.id, phase: entry.phase, command: "?", output: "" } + : { id: entry.id, phase: entry.phase, command: entry.args.command, output: entry.state.output }, + ); + return { + entries: self.derive([entries], ([d]) => d.list), + shells: self.derive([shell.running], ([running]) => toShell(running)), + runShell: async ({ command, lines, requestId }, ctx) => + (await shell.start({ command, lines }, ctx, { key: requestId })).id, + abortShell: async ({ id }) => { + void shell.abort(id, context); + return null; + }, + }; + }); + host.serve(Chat, (_, principal, self) => { + const store = self.get(ChatStore, {}); + if (principal.kind !== "owner") { + // A guest sees no output: the view drops it (computed once per variant in a real store, §12). + const strip = mapShared( + (s: Static): Static => ({ id: s.id, phase: s.phase, command: s.command }), + ); + return { + variant: "guest", + members: { entries: store.entries, shells: self.derive([store.shells], ([list]) => strip(list)) }, + }; + } + return { + variant: "owner", + members: { + entries: store.entries, + shells: store.shells, + runShell: (args, ctx) => store.runShell(args, ctx), + abortShell: (args, ctx) => store.abortShell(args, ctx), + }, + }; + }); +}); + +// 2. A server and two tabs, the owner's and a guest's. +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error(error), +}); +await server.load([{ name: "@example/chat", facet }], context); +async function tab(principal: P) { + const host = createHost

({ onError: (error) => console.error(error) }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return host.get(Chat, {}, context); +} +const owner = await tab({ kind: "owner" }); +const guest = await tab({ kind: "guest" }); +if (owner.variant !== "owner") throw new Error("expected the owner's variant"); + +// 3. The owner runs a command; its request id dedupes a resent request (the same run comes back). +const id = await owner.runShell({ command: "npm test", lines: 5, requestId: "r1" }, context); +const again = await owner.runShell({ command: "npm test", lines: 5, requestId: "r1" }, context); +console.log("a resent request returns the same run:", again === id); +await sleep(100, context); +console.log("owner sees:", owner.shells.value); +console.log("guest sees:", guest.shells.value); + +// 4. One run at a time: a second command is refused while the first runs. +try { + await owner.runShell({ command: "npm run build", lines: 1, requestId: "r2" }, context); +} catch (error) { + console.log("second run refused:", (error as Error).message); +} +await sleep(300, context); +console.log("entries after the first run:", owner.entries.value); + +// 5. Abort is a request: the run stops at its next `sleep`, its last output is flushed, `finish` records it. +const long = await owner.runShell({ command: "tail -f log", lines: 100, requestId: "r3" }, context); +await sleep(100, context); +await owner.abortShell({ id: long }, context); +await sleep(100, context); +console.log("entries after the abort:", owner.entries.value); +console.log("running now:", owner.shells.value); +await server.close(context); diff --git a/packages/chord/test/next/examples/20-tables.ts b/packages/chord/test/next/examples/20-tables.ts new file mode 100644 index 000000000..5b626fe3a --- /dev/null +++ b/packages/chord/test/next/examples/20-tables.ts @@ -0,0 +1,101 @@ +// Tables: an object's rows in the host's store, declared with `defineTable`, written in transactions (with the +// object's documents, if it has any), read by key, by page with cursors, and counted. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/20-tables.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bool, int, local, method, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { add, defineTable, eq, openStore, prefix, query, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// Flat columns (str, int, num, bool, opt(...), json(...)), defaults, a key of non-opt str or int columns, and +// indexes, which end with the key. +const Edits = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + hits: int({ default: 0 }), + done: bool({ default: false }), + note: opt(str()), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); + +const History = local("example.history", { + version: 1, + params: {}, + members: { + record: method(obj({ id: str(), path: str(), at: int() })), + hit: method(str()), + finish: method(str()), + hits: method(str(), int()), + open: method(obj({ after: opt(str()) }), obj({ ids: arr(str()), next: opt(str()) })), + finished: method(obj({}), int()), + }, +}); + +const facet = defineFacet((host) => { + // Tables need a stable identity: a local type with empty params (one object per host). + host.serve(History, (_, self) => { + const edits = self.table(Edits); + // A query of the table: a condition and an order (the key is the tie-break); immutable, built once. + const open = query(Edits) + .where((e) => eq(e.done, false)) + .orderBy((e) => e.at, "desc"); + return { + record: async (row, ctx) => { + // Writes are staged synchronously (a bad row throws here) and commit together in one step. + await self.tx(async (tx) => tx.put(edits, row), ctx); + return null; + }, + hit: async (id, ctx) => { + // A set is values or an expression over the row's columns, computed by the store. + await self.tx(async (tx) => tx.update(edits, { id }, (e) => ({ hits: add(e.hits, 1) })), ctx); + return null; + }, + finish: async (path, ctx) => { + // One statement for every matching row; its count is recorded with the write. + await self.tx(async (tx) => tx.updateWhere(edits, (e) => prefix(e.path, path), { done: true }), ctx); + return null; + }, + hits: async (id, ctx) => (await edits.get({ id }, ctx))?.hits ?? 0, + open: async ({ after }, ctx) => { + const page = await edits.page(open, 1, ctx, after === undefined ? {} : { after }); + return { ids: page.items.map((row) => row.id), ...(page.next === undefined ? {} : { next: page.next }) }; + }, + finished: async (_, ctx) => + edits.count( + query(Edits).where((e) => eq(e.done, true)), + ctx, + ), + }; + }); +}); + +// A store over SQLite (`:memory:` here, a file in an app): the app opens it, gives it to one host, and closes it. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const host = createHost({ store, onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/history", facet }], context); +const history = await host.get(History, {}, context); + +for (let i = 0; i < 5; i++) { + await history.record({ id: `e${i}`, path: `src/${i % 2 === 0 ? "a" : "b"}.ts`, at: i }, context); +} +await history.hit("e1", context); +await history.hit("e1", context); +await history.finish("src/a", context); + +console.log("hits of e1:", await history.hits("e1", context)); +// Pages continue from an opaque cursor. +const first = await history.open({}, context); +console.log("open, first page:", first.ids); +console.log("open, next page:", (await history.open({ after: first.next }, context)).ids); +console.log("finished:", await history.finished({}, context)); + +await host.close(context); +await store.close(context); diff --git a/packages/chord/test/next/examples/21-persisted-documents.ts b/packages/chord/test/next/examples/21-persisted-documents.ts new file mode 100644 index 000000000..e9771455b --- /dev/null +++ b/packages/chord/test/next/examples/21-persisted-documents.ts @@ -0,0 +1,91 @@ +// Persisted documents: `persist: true` keeps an object's documents in the host's store, so a new host on the same +// store finds them again; `checkpointWhen` decides when a change is stored as a new base instead of a delta. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/21-persisted-documents.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { openStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// A persisted singleton: loaded once when its object activates; every 20th stored change is a new base. +const Tags = defineState("tags", { + schema: obj({ tags: arr(str()) }), + version: 1, + initial: () => ({ tags: [] }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 20, +}); +// A persisted keyed document: each key loaded on its first read, then cached until idle for ten minutes. +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +const Notebook = local("example.notebook", { + version: 1, + params: {}, + members: { + tags: obj({ tags: arr(str()) }), + write: method(obj({ id: str(), body: str(), tag: str() })), + read: method(str(), str()), + }, +}); + +const facet = defineFacet((host) => { + // Persisted documents need a stable identity, as tables do: a local type with empty params. + host.serve(Notebook, (_, self) => { + const tags = self.state(Tags); + const notes = self.state(Notes); + return { + tags, + write: async ({ id, body, tag }, ctx) => { + // Both documents commit in one store step, or neither does. + await self.tx(async (tx) => { + (await tx.draft(notes, id)).body = body; + const draft = await tx.draft(tags); + if (!draft.tags.includes(tag)) draft.tags.push(tag); + }, ctx); + return null; + }, + read: async (id, ctx) => (await notes.get(id, ctx))?.body ?? "(none)", + }; + }); +}); + +/** A host on the store at `path`, run until `work` is done with the notebook. */ +async function session(path: string, work: (notebook: Reference) => Promise) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + await host.load([{ name: "@example/notebook", facet }], context); + await work(await host.get(Notebook, {}, context)); + await host.close(context); + await store.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +const path = join(directory, "notebook.db"); +try { + await session(path, async (notebook) => { + await notebook.write({ id: "groceries", body: "oat milk", tag: "home" }, context); + await notebook.write({ id: "standup", body: "demo the store", tag: "work" }, context); + console.log("first host, tags:", notebook.tags.value.tags); + }); + // A new host on the same file: the documents come back from the store. + await session(path, async (notebook) => { + console.log("second host, tags:", notebook.tags.value.tags); + console.log("second host, standup:", await notebook.read("standup", context)); + }); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/22-migrations.ts b/packages/chord/test/next/examples/22-migrations.ts new file mode 100644 index 000000000..f9932070d --- /dev/null +++ b/packages/chord/test/next/examples/22-migrations.ts @@ -0,0 +1,100 @@ +// Table migrations: `evolveTable` appends a version with its edits (rename, add, json before an optional data step; +// drop, index, dropIndex after it). An object whose store holds an older version migrates it when it activates, one +// version at a time, in chunks of 100 rows; a fresh store gets the newest version directly. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/22-migrations.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { + defineTable, + evolveTable, + openStore, + query, + sqliteBackend, + type TableDefinition, +} from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// Version 1, the base, written in full. +const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), file: str(), at: int() }, key: ["id"] }); +// Version 2: `file` becomes `path`, a `dir` column is added and filled from each row (the data step). +const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); +// Version 3: `dir` is dropped again, an index is added. Data functions stay as deployed; history only grows. +const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +const History = local("example.history", { + version: 1, + params: {}, + members: { record: method(obj({ id: str(), at: int() })), dump: method(obj({}), arr(str())) }, +}); + +/** The facet of one deployment: its newest table version, and how it writes a row. */ +function facetOf(definition: TableDefinition, row: (id: string, at: number) => Record) { + return defineFacet((host) => { + host.serve(History, (_, self) => { + const edits = self.table(definition); + return { + record: async ({ id, at }, ctx) => { + // A deployment's rows match its own version (an example shortcut for two versions in one file). + await self.tx(async (tx) => tx.put(edits, row(id, at) as never), ctx); + return null; + }, + dump: async (_, ctx) => { + const lines: string[] = []; + for await (const row of edits.rows(query(definition), ctx)) lines.push(JSON.stringify(row)); + return lines; + }, + }; + }); + }); +} + +/** A host on the store at `path` serving `facet`, run until `work` is done. */ +async function deployment( + path: string, + facet: ReturnType, + work: (history: Reference) => Promise, +) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + // Objects with empty params activate at load: the migration runs before `load` resolves. + await host.load([{ name: "@example/history", facet }], context); + await work(await host.get(History, {}, context)); + await host.close(context); + await store.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +const path = join(directory, "history.db"); +try { + await deployment( + path, + facetOf(EditsV1, (id, at) => ({ id, file: `src/${id}.ts`, at })), + async (history) => { + for (let i = 0; i < 3; i++) await history.record({ id: `e${i}`, at: i }, context); + console.log("version 1:", await history.dump({}, context)); + }, + ); + // The next deployment declares version 3: the store goes through 2 (rename, add, rows) and 3 (drop, index). + await deployment( + path, + facetOf(EditsV3, (id, at) => ({ id, path: `lib/${id}.ts`, at })), + async (history) => { + await history.record({ id: "e9", at: 9 }, context); + console.log("version 3:", await history.dump({}, context)); + }, + ); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/23-keyed-watches.ts b/packages/chord/test/next/examples/23-keyed-watches.ts new file mode 100644 index 000000000..0a13e62c1 --- /dev/null +++ b/packages/chord/test/next/examples/23-keyed-watches.ts @@ -0,0 +1,84 @@ +// Keyed watches and `at(key)`: a keyed document is watched per key; `handle.at(key)` in the factory makes one key a +// watchable (a state member here) that reads `initial()` while absent. A retirement arrives as `initial()` with +// `retired: true`, so a watcher can tell "deleted" from "changed back". +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/23-keyed-watches.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type Note = { body: string }; + +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: (): Note => ({ body: "" }), + keyed: true, +}); + +const Notebook = local("example.notebook", { + version: 1, + params: {}, + members: { + // The note "pinned" as a state member: `initial()` while absent. + pinned: obj({ body: str() }), + write: method(obj({ id: str(), body: str() })), + remove: method(str()), + }, +}); + +// The example's own window into the object: its keyed handle, to watch other keys from outside. +let notes: KeyedHandle | undefined; + +const facet = defineFacet((host) => { + host.serve(Notebook, (_, self) => { + notes = self.state(Notes); + const handle = notes; + return { + // Pinned while this attempt lives, and loaded before the object goes live. + pinned: handle.at("pinned"), + write: async ({ id, body }, ctx) => { + await handle.change( + id, + (draft) => { + draft.body = body; + }, + ctx, + ); + return null; + }, + remove: async (id, ctx) => { + await handle.retire(id, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/notebook", facet }], context); +const notebook = await host.get(Notebook, {}, context); + +// A watch of the state member: every value call carries the value, its ops and the retired flag. +const pinned = await notebook.pinned.watch(context); +pinned.start(async (call) => { + if (call.kind === "value") console.log("pinned:", call.value, call.retired ? "(retired)" : ""); +}); + +await notebook.write({ id: "pinned", body: "buy milk" }, context); +// A watch of one key; an absent key gives `undefined` (watch it once it exists, or pin it with `at`). +console.log("watch of an absent key:", await notes!.watch("later", context)); +await notebook.write({ id: "later", body: "call Anna" }, context); +const later = (await notes!.watch("later", context))!; +later.start(async (call) => { + if (call.kind === "value") console.log("later:", call.value, call.retired ? "(retired)" : ""); +}); +await notebook.remove("later", context); +await notebook.remove("pinned", context); + +// Frames are delivered after the commit, asynchronously: let them arrive before closing. +await new Promise((resolve) => setTimeout(resolve, 10)); +await host.close(context); diff --git a/packages/chord/test/next/examples/24-keyed-members.ts b/packages/chord/test/next/examples/24-keyed-members.ts new file mode 100644 index 000000000..37f13a08d --- /dev/null +++ b/packages/chord/test/next/examples/24-keyed-members.ts @@ -0,0 +1,64 @@ +// Keyed state members: a `local` contract exposes a keyed document set as one member, `keyed(schema)`. A consumer on +// the same host pins one key with `ref.member.at(key)` in its factory (a watchable, part of its own hydration), or +// reads and watches any key with `ref.member.get/watch(key, ctx)`. One source per key: consumers share it. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/24-keyed-members.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Profile = obj({ name: str(), status: str() }); + +const Profiles = defineState("profiles", { + schema: Profile, + version: 1, + initial: () => ({ name: "", status: "offline" }), + keyed: true, +}); + +// The directory: every user's profile, one keyed member. +const Directory = local("example.directory", { + version: 1, + params: {}, + members: { profiles: keyed(Profile), set: method(obj({ user: str(), name: str(), status: str() })) }, +}); + +// A badge per user: its own object, showing one profile. +const Badge = local("example.badge", { version: 1, params: { user: str() }, members: { label: str() } }); + +const facet = defineFacet((host) => { + host.serve(Directory, (_, self) => { + const profiles = self.state(Profiles); + return { + // The keyed handle is the member as it is. + profiles, + set: async ({ user, name, status }, ctx) => { + await profiles.change(user, (draft) => Object.assign(draft, { name, status }), ctx); + return null; + }, + }; + }); + host.serve(Badge, ({ user }, self) => { + // Pinned for this attempt; the badge goes live once the profile is loaded. + const profile = self.get(Directory, {}).profiles.at(user); + return { label: self.derive([profile], ([p]) => `${p.name || user} (${p.status})`) }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/directory", facet }], context); +const directory = await host.get(Directory, {}, context); +await directory.set({ user: "ana", name: "Ana", status: "online" }, context); + +const badge = await host.get(Badge, { user: "ana" }, context); +console.log("badge:", badge.label.value); +await directory.set({ user: "ana", name: "Ana", status: "away" }, context); +console.log("badge after a change:", badge.label.value); +// Any key, read where it is needed. +console.log("ref.profiles.get(bo):", await directory.profiles.get("bo", context)); + +badge.release(); +await host.close(context); diff --git a/packages/chord/test/next/examples/25-export-import.ts b/packages/chord/test/next/examples/25-export-import.ts new file mode 100644 index 000000000..f8dd8a812 --- /dev/null +++ b/packages/chord/test/next/examples/25-export-import.ts @@ -0,0 +1,102 @@ +// Export and import: `store.export(ctx)` yields the store's state as archive parts (lossless: tables with their rows +// and versions, the live documents), holding the commit queue while it runs; `encodeJsonl` turns them into JSONL. +// `importStore` checks an archive part by part and makes a new store from it in one backend transaction. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/25-export-import.ts +import { createReadStream, createWriteStream } from "node:fs"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { + decodeJsonl, + defineTable, + encodeJsonl, + importStore, + openStore, + type Store, + sqliteBackend, +} from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +const Visits = defineTable("visits", { version: 1, columns: { page: str(), count: int() }, key: ["page"] }); +const Title = defineState("title", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "untitled" }), + persist: true, + checkpointWhen: () => false, +}); + +const Site = local("example.site", { + version: 1, + params: {}, + members: { + title: obj({ text: str() }), + visit: method(str(), int()), + rename: method(str()), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Site, (_, self) => { + const visits = self.table(Visits); + const title = self.state(Title); + return { + title, + visit: async (page, ctx) => + self.tx(async (tx) => { + const count = ((await tx.get(visits, { page }))?.count ?? 0) + 1; + tx.put(visits, { page, count }); + return count; + }, ctx), + rename: async (text, ctx) => { + await title.change((draft) => { + draft.text = text; + }, ctx); + return null; + }, + }; + }); +}); + +/** A host on `store` serving the site, run until `work` is done. */ +async function serve(store: Store, work: (site: Reference) => Promise) { + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + await host.load([{ name: "@example/site", facet }], context); + await work(await host.get(Site, {}, context)); + await host.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +try { + const original = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "a.db"))), context); + await serve(original, async (site) => { + await site.rename("Chord", context); + await site.visit("/", context); + await site.visit("/", context); + await site.visit("/docs", context); + }); + // Export to a local file (then upload it, if needed): the queue is held only while the parts are read. + const archive = join(directory, "archive.jsonl"); + await pipeline(encodeJsonl(original.export(context)), createWriteStream(archive)); + console.log("archive:", (await stat(archive)).size, "bytes at seq", original.lastSeq); + await original.close(context); + + // Import into an empty backend: every part is checked; a refused archive leaves the backend empty. + const parts = decodeJsonl(createReadStream(archive)); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "b.db"))), parts, context); + console.log("imported at seq", copy.lastSeq); + await serve(copy, async (site) => { + console.log("title:", site.title.value.text, "- next visit of /:", await site.visit("/", context)); + }); + await copy.close(context); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/28-persisted-jobs.ts b/packages/chord/test/next/examples/28-persisted-jobs.ts new file mode 100644 index 000000000..4750e3ced --- /dev/null +++ b/packages/chord/test/next/examples/28-persisted-jobs.ts @@ -0,0 +1,125 @@ +// Persisted jobs: `persist: { resume }` keeps each unsettled run's record and state document in the object's store. +// A run survives a crash (counted: five in a row end it `restart_limit`), a clean close (written back `stopped`, resumed +// without counting) and its object's retry; `abort` is stored and joins `run`; a state change may be `deferred`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/28-persisted-jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle } from "../../../src/next/jobs/index.ts"; +import { openStore, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// The run's state: persisted, as a persisted job's must be. +const ImportState = defineState("notes.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 16, +}); + +const ImportJob = defineJob("notes.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + // After a crash or a stop, `run` runs again: it must be idempotent (only work since the last state change repeats). + persist: { resume: "rerun" }, +}); + +const Notes = local("notes.store", { + version: 1, + params: {}, + members: { start: method(str(), str()), abort: method(str(), str()) }, +}); + +/** How the current process's `run` behaves: hang (to be cut off) or finish. */ +let behaviour: "hang" | "finish" = "hang"; +let jobs: JobHandle<{ path: string }, { count: number }, { done: number }> | undefined; + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + console.log(` run ${path}: from done = ${ctx.state.value.done}`); + // Progress, deferred: published at once, stored within 100 ms (or with the next immediate change). + await ctx.state.change( + (draft) => { + draft.done += 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 100 } }, + ); + if (behaviour === "hang") + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + console.log(` finish: ${outcome.kind}`, outcome.kind === "done" ? outcome.result : outcome.error.kind); + }, + }); + jobs = imports; + return { + start: async (path, ctx) => (await imports.start({ path }, ctx)).id, + abort: (id, ctx) => imports.abort(id, ctx), + }; + }); +}); + +/** A process: a store over the file and a host serving `Notes`. */ +async function process(path: string): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ + store, + onError: (error) => console.log(" onError:", error.kind), + closeTimeout: 100, + }); + await host.load([{ name: "@example/notes", facet }], context); + return { store, host }; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 150)); +const directory = await mkdtemp(join(tmpdir(), "chord-example-28-")); +const path = join(directory, "session.sqlite"); + +// 1. Start a run, then crash: the store closes under the live host. +console.log("1. start, then crash"); +let { store, host } = await process(path); +const id = await (await host.get(Notes, {}, context)).start("inbox", context); +await settle(); +await store.close(context); + +// 2. The next process counts the crash and reruns it (its state as stored: the deferred change was stored in time). +console.log("2. reopen: the run is counted and rerun"); +({ store, host } = await process(path)); +await settle(); + +// 3. A clean close writes the run back `stopped`: the next process resumes it without counting. +console.log("3. clean close, reopen: resumed, uncounted"); +await host.close(context); +await store.close(context); +behaviour = "finish"; +({ store, host } = await process(path)); +await settle(); +console.log(" unsettled runs now:", jobs?.running.value.length); + +// 4. A stored abort: marked, joined. +console.log("4. abort"); +behaviour = "hang"; +const notes = await host.get(Notes, {}, context); +const second = await notes.start("drafts", context); +await settle(); +console.log(" abort:", await notes.abort(second, context)); +await settle(); +console.log(" once settled:", await notes.abort(second, context), "/ the first run:", await notes.abort(id, context)); + +await host.close(context); +await store.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/29-registries.ts b/packages/chord/test/next/examples/29-registries.ts new file mode 100644 index 000000000..462bd20b0 --- /dev/null +++ b/packages/chord/test/next/examples/29-registries.ts @@ -0,0 +1,115 @@ +// Registries: an extension point between plugins. The defining plugin shares a token (`defineRegistry`: a name, a +// version, the contribution and value types) and serves it with `compose`, a pure function from the table of +// contributions (sorted by plugin name, never load order) to one value, conflicts resolved by a fallback and reported +// (`registry_conflict`). Facets and host-lifetime objects contribute one value each: a plain value, or a watchable +// (`self.writable`) whose changes recompose. Readers follow the composed value (`self.registry`, a dependency like +// `self.get`; `host.registry` outside objects) and guard each contribution where they call it. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/29-registries.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry, keyedEntries, type TableItem } from "../../../src/next/registries/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// 1. The token, shared by every plugin (bundled like a contract): routes in, a router out. +type Handler = (path: string) => string; +type Routes = { readonly routes: Readonly> }; +type Router = ReadonlyMap>; +const Http = defineRegistry("example.http", { version: 1 }); + +/** The HTTP server: it reads the router at each request and guards the contribution it calls. */ +const Server = local("example.server", { + version: 1, + params: {}, + members: { request: method(obj({ path: str() }), str()) }, +}); + +/** A plugin whose routes are known only at runtime: it contributes a writable it sets. */ +const Mounts = local("example.mounts", { version: 1, params: {}, members: { mount: method(obj({ path: str() })) } }); + +// 2. The defining plugin serves the registry: the first route by plugin name wins, a duplicate is reported. +const httpPlugin = defineFacet((host) => { + host.serve(Http, { compose: (table, { report }) => keyedEntries(table, (entry) => entry.routes, report) }); + host.serve(Server, (_, self) => { + const router = self.registry(Http); + return { + request: async ({ path }) => { + const route = router.value.get(path); + if (route === undefined) return "404"; + // A contribution is another plugin's code: catch, report its plugin, go on. + try { + return route.value(path); + } catch (error) { + console.log(`route ${path} of ${route.plugin} threw:`, (error as Error).message); + return "500"; + } + }, + }; + }); +}); + +// 3. Two plugins contribute from their facets: plain values, in the table while the plugin is loaded. +const notesPlugin = defineFacet((_, self) => { + self.contribute(Http, { routes: { "/notes": () => "notes", "/broken": () => JSON.parse("{") } }); +}); +const searchPlugin = defineFacet((_, self) => { + self.contribute(Http, { routes: { "/search": () => "results", "/notes": () => "search's notes" } }); +}); + +// 4. A host-lifetime object contributes a writable: its changes recompose the router. +const mountsPlugin = defineFacet((host) => { + host.serve(Mounts, (_, self) => { + const routes = self.writable({ routes: {} }); + self.contribute(Http, routes); + return { + mount: async ({ path }) => { + routes.set({ routes: { ...routes.value.routes, [path]: () => `mounted ${path}` } }); + return null; + }, + }; + }); +}); + +const host = createHost({ + onError: (error) => { + if (error.kind === "registry_conflict") console.log("conflict:", error.message, error.plugins); + else console.log("onError:", error.kind); + }, +}); + +// 5. A reader before `load` waits: the provider comes with `load`, which composes every registry before any object. +const early = host.registry(Http, context); +await host.load( + [ + { name: "@example/http", facet: httpPlugin }, + { name: "@example/notes", facet: notesPlugin }, + { name: "@example/search", facet: searchPlugin }, + { name: "@example/mounts", facet: mountsPlugin }, + ], + context, +); +const router = await early; +console.log( + "routes:", + [...router.value].map(([path, { plugin }]) => `${path} (${plugin})`), +); + +// 6. Requests: a duplicate went to the first plugin by name; a throwing contribution is guarded at the use site. +const server = await host.get(Server, {}, context); +console.log("/notes ->", await server.request({ path: "/notes" }, context)); +console.log("/broken ->", await server.request({ path: "/broken" }, context)); + +// 7. A watch of the composed value gets each new router; mounting recomposes it. +const watch = await router.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("router now has:", [...call.value.keys()]); +}); +const mounts = await host.get(Mounts, {}, context); +await mounts.mount({ path: "/files" }, context); +await new Promise((resolve) => setTimeout(resolve, 0)); +console.log("/files ->", await server.request({ path: "/files" }, context)); + +await watch.stop(); +await host.close(context); diff --git a/packages/chord/test/next/examples/30-host-root.ts b/packages/chord/test/next/examples/30-host-root.ts new file mode 100644 index 000000000..885ab4414 --- /dev/null +++ b/packages/chord/test/next/examples/30-host-root.ts @@ -0,0 +1,85 @@ +// `host.root(fn, context)`: a call scope for code outside objects (a Durable tool, a CLI command). `fn` gets a ctx like a +// method handler's: the host's principal (or one a trusted caller names), this host's incarnation, the context's +// telemetry and signal, and `ctx.get` for references. When `fn` settles, or the context aborts, or the host closes, the +// scope is sealed: its signal aborts, its references are released, and a late `ctx.get` is refused `released`; so code +// that ignores its signal fails at its next use of Chord. (p3 wraps it as `chord(fn, context)` for Durable's tools.) +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/30-host-root.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { arr, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type Principal = { readonly kind: "server" } | { readonly kind: "owner"; readonly name: string }; + +const Notes = local("example.notes", { + version: 1, + params: { session: str() }, + members: { list: arr(str()), add: method(obj({ text: str() })) }, +}); + +const List = defineState("example.list", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const notes = self.state(List); + return { + list: self.derive([notes], ([n]) => n.list), + add: async ({ text }, ctx) => { + await notes.change((draft) => { + draft.list.push(`${text} (by ${ctx.principal?.kind})`); + }, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ principal: { kind: "server" }, onError: (error) => console.log(error.kind) }); + +// 1. A scope made before `load` resolved: its `ctx.get` parks until the host is loaded. +const early = host.root(async (ctx) => (await ctx.get(Notes, { session: "s1" })).list.value, context); +await host.load([{ name: "@example/notes", facet }], context); +console.log("before load, read after it:", await early); + +// 2. A tool's call: get, call, return plain data. The reference is released when fn settles. +const addNote = (text: string) => + host.root(async (ctx) => { + const notes = await ctx.get(Notes, { session: "s1" }); + await notes.add({ text }, ctx); + return notes.list.value; + }, context); +console.log("after a tool call:", await addNote("first")); + +// 3. Base code acting for an authenticated user names the principal. +const owner: Principal = { kind: "owner", name: "ada" }; +console.log("as the owner:", await host.root(async (ctx) => ctx.principal, context, { principal: owner })); + +// 4. A kept ctx is sealed once fn settled. +let kept: Ctx | undefined; +await host.root(async (ctx) => { + kept = ctx; +}, context); +console.log("kept ctx aborted:", kept!.abortSignal?.aborted); +await kept!.get(Notes, { session: "s1" }).catch((error: Error) => console.log("kept ctx.get:", error.message)); + +// 5. Cancelling the context rejects at once and seals the scope; an fn that ignores its signal fails at its next +// use of Chord. +const { context: cancellable, cancel } = withCancel(context); +const stubborn = host.root(async (ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + await ctx.get(Notes, { session: "s1" }); +}, cancellable); +cancel(new Error("user pressed stop")); +await stubborn.catch((error: Error) => console.log("cancelled:", error.message)); + +// 6. Closing the host seals the scopes still open and rejects their callers at once: an fn sleeping on its ctx ends. +const sleeping = host.root((ctx) => sleep(60_000, ctx), context).catch((error: Error) => `closed: ${error.message}`); +await host.close(context); +console.log(await sleeping); diff --git a/packages/chord/test/next/examples/31-parked-dependencies.ts b/packages/chord/test/next/examples/31-parked-dependencies.ts new file mode 100644 index 000000000..938bd0b63 --- /dev/null +++ b/packages/chord/test/next/examples/31-parked-dependencies.ts @@ -0,0 +1,148 @@ +// Parked dependencies (dependency model, slice 7a): a reference is a request that outlives every provider it meets. An +// object whose dependency is not there yet is parked, not failed: it shows `failed { waiting }` naming what it waits +// for (the nearest blocker and the root cause), no timer runs for it, its factory is not run again, and it goes on the +// moment its dependency is live. Here a tab's panel needs `Notes` from a server that does not serve it yet; the server +// restarts with the notes plugin, the tab reconnects, and the panel goes live without anyone asking. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/31-parked-dependencies.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type RefState } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import type { NativeWatchable } from "../../../src/watch.ts"; + +const context = BACKGROUND_CONTEXT; +const started = Date.now(); +const at = (): string => `${((Date.now() - started) / 1000).toFixed(1)} s`; + +type Principal = { readonly kind: "server" } | { readonly kind: "owner" }; + +const Server = hostKind("server"); + +/** The server's notes (the notes plugin), placed on the server. */ +const Notes = remote("example.notes", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { editor: { state: { list: arr(str()) }, methods: { add: method(obj({ text: str() })) } } }, +}); + +/** A panel in the tab: a local object over the server's notes. */ +const Panel = local("example.panel", { version: 1, params: { id: int() }, members: { count: int() } }); + +/** A local object the panel's sibling needs, which no plugin of the tab serves. */ +const Theme = local("example.theme", { version: 1, params: {}, members: { name: str() } }); +const Header = local("example.header", { version: 1, params: { id: int() }, members: { title: str() } }); + +const List = defineState("example.list", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** The notes plugin's server facet. */ +const notesPlugin = defineFacet((host) => { + host.serve(Notes, (_params, _principal, self) => { + const list = self.state(List); + return { + variant: "editor", + members: { + list: self.derive([list], ([l]) => l.list), + add: async ({ text }, ctx) => { + await list.change((draft) => void draft.list.push(text), ctx); + return null; + }, + }, + }; + }); +}); + +/** A server with the given plugins, its link to the tab accepted. */ +async function startServer(plugins: { readonly name: string; readonly facet: typeof notesPlugin }[]) { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load(plugins, context); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "owner" } }, context); + return { server, tabLink: down, close: () => up.close() }; +} + +// The tab: its panel declares the notes (a dependency); its header a theme nobody serves. +let panels = 0; +const tab = createHost({ onError: (error) => console.log(`${at()}: reported ${error.kind}`) }); +await tab.load( + [ + { + name: "@example/panel", + facet: defineFacet((host) => { + host.serve(Panel, (_, self) => { + panels++; + const notes = self.get(Notes, {}); + return { count: self.derive([notes.list], ([list]) => list.length) }; + }); + host.serve(Header, (_, self) => ({ title: self.get(Theme, {}).name })); + }), + }, + ], + context, +); + +/** Prints every state of `state` as it changes. */ +async function follow(label: string, state: NativeWatchable): Promise { + const watch = await state.watch(context); + watch.start(async (call) => { + if (call.kind === "value") console.log(`${at()}: ${label} ${describe(call.value)}`); + }); +} + +function describe(state: RefState): string { + if (state.kind !== "failed") return state.kind; + return `failed { ${state.cause.kind} }: ${state.cause.error.message}`; +} + +// 1. The first server has no notes plugin. Its answer is `failed { waiting }`: not the panel's fault. +const first = await startServer([]); +tab.connect(first.tabLink, { principal: { kind: "server" } }, context); +const panel = tab.hold(Panel, { id: 1 }, context); +await follow("panel:", panel.state); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: panel now ${describe(panel.state.value)}`); + +// 2. A local type nobody serves here is the same answer: the request waits unbound, indexed under the type's name. +const header = tab.hold(Header, { id: 1 }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: header ${describe(header.state.value)}`); + +// 3. A getter waits through `waiting` up to its readyTimeout, then rejects with the cause (a hold just shows it). +try { + await tab.get(Panel, { id: 1 }, context, { readyTimeout: 200 }); +} catch (error) { + if (isChordError(error)) console.log(`${at()}: get gave up after 200 ms: ${error.message}`); +} + +// 4. Time passes: no timer runs for the parked panel, its factory is not run again. +await new Promise((resolve) => setTimeout(resolve, 1500)); +console.log(`${at()}: still ${panel.state.value.kind}, factory runs: ${panels}`); + +// 5. The server restarts with the notes plugin (the link drop shows as loading); the tab reconnects: its request is +// held again and binds, and the parked panel goes on. +first.close(); +await new Promise((resolve) => setTimeout(resolve, 10)); +const second = await startServer([{ name: "@example/notes", facet: notesPlugin }]); +tab.connect(second.tabLink, { principal: { kind: "server" } }, context); +const live = await tab.get(Panel, { id: 1 }, context); +console.log(`${at()}: panel live, count ${live.count.value}, factory runs: ${panels}`); +// The panel's notes are the owner's object on the server (a remote object per principal): the tab adds through its own. +const notes = await tab.get(Notes, {}, context); +await notes.add({ text: "first" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: panel count ${live.count.value}`); + +header.release(); +panel.release(); +live.release(); +notes.release(); +await tab.close(context); +await second.server.close(context); +await first.server.close(context); diff --git a/packages/chord/test/next/examples/32-startup.ts b/packages/chord/test/next/examples/32-startup.ts new file mode 100644 index 000000000..2c2169e10 --- /dev/null +++ b/packages/chord/test/next/examples/32-startup.ts @@ -0,0 +1,104 @@ +// Startup (dependency model §6, slice 7a): `load` publishes every facet's declarations in one batch first, then starts +// the host-lifetime objects; requests made here and holds arriving on links wait only for that publication, never for +// the start barrier. Two hosts whose objects need each other's types (an acyclic graph) used to deadlock: the server's +// eager `Bootstrap` needs the worker's `Agent`, the `Agent` needs the server's `Catalog`, and the worker's hold for +// `Catalog` waited for the server's whole `load`, which waited for `Bootstrap`. Now it starts at once, no timeout. +// `load` resolves when each host-lifetime object had its first outcome: live, failed by its own cause, or parked with +// no blocker here still without one (a wait on another host ends the first pass). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/32-startup.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { hostKind, int, local, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; +const started = Date.now(); +const at = (): string => `${Date.now() - started} ms`; + +type Principal = { readonly kind: "server" } | { readonly kind: "worker"; readonly session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +/** On the server, a host-lifetime object (local, empty params): it starts at `load`. */ +const Bootstrap = local("example.bootstrap", { version: 1, params: {}, members: { agents: int() } }); + +/** On the session's worker. */ +const Agent = remote("example.agent", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { models: int() } } }, +}); + +/** On the server. */ +const Catalog = remote("example.catalog", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { models: int() } } }, +}); + +/** The worker of a session, started by the server's route; it connects first, then loads (W6). */ +function startWorker(id: string) { + const [up, down] = memoryLinkPair(); + const worker = createHost({ + address: Session({ id }), + principal: { kind: "worker", session: id }, + onError: (error) => console.log(`${at()}: worker reported ${error.kind}`), + }); + worker.connect(down, { principal: { kind: "server" }, trusted: true }, context); + const agents = defineFacet((host) => + host.serve(Agent, (_params, _principal, self) => { + console.log(`${at()}: worker: Agent declares the server's Catalog`); + return { variant: "v", members: { models: self.get(Catalog, {}).models } }; + }), + ); + void worker + .load([{ name: "@example/agents", facet: agents }], context) + .then(() => console.log(`${at()}: worker loaded`)); + return { worker, link: up }; +} + +const workers: ReturnType[] = []; +const server = createHost({ + address: Server(), + principal: { kind: "server" }, + // No readyTimeout is needed to break anything: a long one shows nothing waits for it. + readyTimeout: 600_000, + onError: (error) => console.log(`${at()}: server reported ${error.kind}`), +}); +server.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const started = startWorker(address.params.id); + workers.push(started); + return { link: started.link, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +const core = defineFacet((host) => { + host.serve(Catalog, () => ({ variant: "v", members: { models: constant(3) } })); + host.serve(Bootstrap, (_, self) => { + console.log(`${at()}: server: Bootstrap declares the worker's Agent`); + return { agents: self.get(Agent, { session: "S1" }).models }; + }); +}); + +// 1. The server's load: publication (Catalog served), then Bootstrap starts and parks on the worker's Agent. A wait on +// another host is a first outcome, so `load` resolves without waiting for the worker. +await server.load([{ name: "@example/core", facet: core }], context); +console.log(`${at()}: server loaded; Bootstrap is ${server.hold(Bootstrap, {}, context).state.value.kind}`); + +// 2. The worker's hold for Catalog bound on the published server; Agent went live, then Bootstrap. +const bootstrap = await server.get(Bootstrap, {}, context); +console.log(`${at()}: Bootstrap live, it sees ${bootstrap.agents.value} models through the worker`); + +bootstrap.release(); +await server.close(context); +for (const { worker } of workers) await worker.close(context); diff --git a/packages/chord/test/next/examples/33-reload.ts b/packages/chord/test/next/examples/33-reload.ts new file mode 100644 index 000000000..d97c4eec7 --- /dev/null +++ b/packages/chord/test/next/examples/33-reload.ts @@ -0,0 +1,185 @@ +// Reload in the kernel (slice 7b): `host.reload(facets, ctx)` switches a running host to a new plugin set. Staging +// runs every setup and every affected object's new factory against a recording `self` (nothing activates; a throw +// rejects the reload and nothing changes), the commit swaps the served types, registries and contributions at once, +// then every object of a changed or removed plugin takes its fate. Here: a pure notes store is swapped in place while +// a call runs on its old code, a search index with a native restarts after its running call drained, and a removed +// plugin's object ends while its holder waits for it to come back. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/33-reload.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type RefState } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** The notes store: a list, its count, and `add` (which can be told to wait, to show a call across a swap). */ +const Notes = local("example.notes", { + version: 1, + params: {}, + members: { list: arr(str()), banner: str(), add: method(obj({ text: str() }), int()) }, +}); + +/** A search index over the notes, holding a native (an index file, say). */ +const Search = local("example.search", { version: 1, params: {}, members: { engine: str(), reindex: method() } }); + +/** A clock plugin, removed by the third reload. */ +const Clock = local("example.clock", { version: 1, params: {}, members: { zone: str() } }); + +const List = defineState("example.list", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +let slowAdd: PromiseWithResolvers | undefined; + +/** The notes plugin, its banner and the text `add` prefixes differing per version: a pure object (no `self.open`). */ +function notesPlugin(banner: string) { + return defineFacet((host) => + host.serve(Notes, (_, self) => { + const list = self.state(List); + return { + list: self.derive([list], ([value]) => value.list), + banner: constant(banner), + add: async ({ text }: { text: string }, ctx: Context) => { + await slowAdd?.promise; + await list.change((draft) => void draft.list.push(`${text} (${banner})`), ctx); + return list.value.list.length; + }, + }; + }), + ); +} + +let slowReindex: PromiseWithResolvers | undefined; + +/** The search plugin: its engine is a native, so every reload of it restarts the object. */ +function searchPlugin(engine: string) { + return defineFacet((host) => + host.serve(Search, (_, self) => { + const index = self.open( + async () => { + console.log(` search: ${engine} opened`); + return engine; + }, + (value, reason) => console.log(` search: ${value} closed (${reason.kind})`), + ); + return { + engine: constant(engine), + reindex: async () => { + console.log(` search: reindex on ${index.value} started`); + await slowReindex?.promise; + console.log(` search: reindex on ${index.value} done`); + return null; + }, + }; + }), + ); +} + +const clockPlugin = defineFacet((host) => host.serve(Clock, () => ({ zone: constant("UTC") }))); + +/** Shows a reference's state as text. */ +function shown(state: RefState): string { + return state.kind === "failed" ? `failed { ${state.cause.kind} }: ${state.cause.error.message}` : state.kind; +} + +// 1. A host with three plugins, loaded once. A reload passes the complete set; a facet passed as the very same object +// is an unchanged plugin (untouched), any other value for a loaded name a changed one. +const host = createHost({ onError: (error) => console.log(` reported: ${error.kind}`) }); +const search = searchPlugin("lucene"); +const notesV2 = notesPlugin("v2"); +const searchV2 = searchPlugin("tantivy"); +await host.load( + [ + { name: "@example/notes", facet: notesPlugin("v1") }, + { name: "@example/search", facet: search }, + { name: "@example/clock", facet: clockPlugin }, + ], + context, +); +const notes = await host.get(Notes, {}, context); +await notes.add({ text: "first" }, context); +console.log("1. loaded:", notes.list.value, "banner", notes.banner.value); + +// 2. A pure swap while a call runs: the running `add` finishes on the old code, the next call runs on the new one. +slowAdd = Promise.withResolvers(); +const running = notes.add({ text: "second" }, context); +const swap = await host.reload( + [ + { name: "@example/notes", facet: notesV2 }, + { name: "@example/search", facet: search }, + { name: "@example/clock", facet: clockPlugin }, + ], + context, +); +console.log("2. reload:", JSON.stringify(swap)); +slowAdd.resolve(); +slowAdd = undefined; +console.log( + " the old call returned", + await running, + "; the new code adds", + await notes.add({ text: "third" }, context), +); +console.log(" list:", notes.list.value, "banner", notes.banner.value, "(same documents, new code)"); + +// 3. A restart: the search object holds a native. Its running call drains first, then the native closes (reload). +const index = await host.get(Search, {}, context); +slowReindex = Promise.withResolvers(); +const reindexing = index.reindex(context); +await new Promise((resolve) => setTimeout(resolve, 0)); +const restart = await host.reload( + [ + { name: "@example/notes", facet: notesV2 }, + { name: "@example/search", facet: searchV2 }, + { name: "@example/clock", facet: clockPlugin }, + ], + context, +); +console.log("3. reload:", JSON.stringify(restart.objects), "- the reference shows", shown(index.state.value)); +slowReindex.resolve(); +await reindexing; +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(" search now", shown(index.state.value), "on", index.engine.value); + +// 4. A removal: the clock plugin is gone. Its object ends; its holder's request is not ended, it waits `not_served`. +const clock = host.hold(Clock, {}, context); +await new Promise((resolve) => setTimeout(resolve, 0)); +const removal = await host.reload( + [ + { name: "@example/notes", facet: notesV2 }, + { name: "@example/search", facet: searchV2 }, + ], + context, +); +console.log("4. reload:", JSON.stringify(removal.plugins), "- clock holder:", shown(clock.state.value)); +console.log(" its last value stays readable:", clock.zone.value); + +// 5. A later reload serves the clock again: the same holder binds again by itself. +await host.reload( + [ + { name: "@example/notes", facet: notesV2 }, + { name: "@example/search", facet: searchV2 }, + { name: "@example/clock", facet: clockPlugin }, + ], + context, +); +await new Promise((resolve) => setTimeout(resolve, 0)); +console.log("5. served again: clock holder", shown(clock.state.value), clock.zone.value); + +// 6. A staging error rejects the reload and changes nothing. +const broken = defineFacet((h) => + h.serve(Notes, () => { + throw new Error("a typo in the new factory"); + }), +); +try { + await host.reload([{ name: "@example/notes", facet: broken }], context); +} catch (error) { + console.log("6. rejected:", (error as Error).message, "- banner still", notes.banner.value); +} + +await host.close(context); diff --git a/packages/chord/test/next/examples/34-versions.ts b/packages/chord/test/next/examples/34-versions.ts new file mode 100644 index 000000000..0f37b1fc4 --- /dev/null +++ b/packages/chord/test/next/examples/34-versions.ts @@ -0,0 +1,186 @@ +// Versions in use (slice 7b §7, §8): a provider keeps serving older consumers through `accepts`, written once with the +// provider, never in the shared contract (Q10). Here the server's notes go from v1 to v2 while an old tab, still on v1, +// stays connected: it reads the new state mapped back (as diffs, never the whole list per commit), and its calls are +// mapped forwards and their results back. A job goes from v1 to v2 too: a run started on v1 finishes on its old code, +// and the new `finish` gets its args and result mapped forwards. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/34-versions.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { + arr, + evolve, + hostKind, + int, + local, + method, + obj, + remote, + str, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type FacetHost } from "../../../src/next/host/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type Principal = { readonly kind: "server" } | { readonly kind: "owner" }; + +const Server = hostKind("server"); + +// The shared contract, as every plugin bundles it: v1, then v2 where a note gained a tag. +const NotesV1 = remote("example.notes", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { editor: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() }), int()) } } }, +}); +const NotesV2 = evolve(NotesV1, (e) => + e + .state("editor", "notes", arr(obj({ text: str(), tag: str() }))) + .method("add", obj({ text: str(), tag: str() }), obj({ count: int() })), +); + +const List = defineState("example.list", { + schema: obj({ notes: arr(obj({ text: str(), tag: str() })) }), + version: 1, + initial: () => ({ notes: [] }), +}); + +/** The server's notes plugin at v2: what a v1 consumer sees is the provider's `accepts[1]`. */ +function serveV2(host: FacetHost): void { + host.serve( + NotesV2, + (_, _principal, self) => { + const list = self.state(List); + return { + variant: "editor", + members: { + notes: self.derive([list], ([value]) => value.notes), + add: async ({ text, tag }, ctx) => { + await list.change((draft) => void draft.notes.push({ text, tag }), ctx); + return { count: list.value.notes.length }; + }, + }, + }; + }, + { + accepts: { + 1: { + state: { editor: { notes: (current) => current.map(({ text }) => text) } }, + methods: { add: { args: (old) => ({ ...old, tag: "untagged" }), result: (current) => current.count } }, + }, + }, + }, + ); +} + +/** The server's notes plugin at v1. */ +function serveV1(host: FacetHost): void { + host.serve(NotesV1, (_, _principal, self) => { + const list = self.state(List); + return { + variant: "editor", + members: { + notes: self.derive([list], ([value]) => value.notes.map(({ text }) => text)), + add: async ({ text }, ctx) => { + await list.change((draft) => void draft.notes.push({ text, tag: "" }), ctx); + return list.value.notes.length; + }, + }, + }; + }); +} + +// 1. A server on v1 and a tab built against v1, connected. +const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); +await server.load([{ name: "@example/notes", facet: defineFacet(serveV1) }], context); +const tab = createHost({ onError: () => {} }); +await tab.load([], context); +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "owner" } }, context); +tab.connect(down, { principal: { kind: "server" } }, context); +const old = await tab.get(NotesV1, {}, context); +await old.add({ text: "milk" }, context); +const watch = await old.notes.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(" tab frame:", JSON.stringify(call.ops)); +}); +console.log("1. v1 everywhere:", old.notes.value); + +// 2. The server reloads to v2 with `accepts[1]`: the old tab's hold is kept; its version is an older one now, so it is +// bound again through the mapping (one snapshot), and from then on gets the mapped diffs. +await server.reload([{ name: "@example/notes", facet: defineFacet(serveV2) }], context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("2. server on v2; the old tab still reads", old.notes.value, "and its state:", old.state.value.kind); + +// 3. The old tab calls v1's `add`: its args are mapped forwards, the v2 result mapped back to v1's count. +console.log("3. old add returned", await old.add({ text: "eggs" }, context)); +await new Promise((resolve) => setTimeout(resolve, 10)); +// A new tab (v2 code) on the same object sees the tags. +const fresh = await tab.get(NotesV2, {}, context); +console.log(" a v2 consumer reads", JSON.stringify(fresh.notes.value)); + +// 4. Jobs: `Export` v2 maps a v1 run's args and result forwards; a v1 run still running finishes on its old code. +const ExportV1 = defineJob("example.export", { version: 1, args: obj({ path: str() }), result: str() }); +const ExportV2 = defineJob("example.export", { + version: 2, + args: obj({ path: str(), format: str() }), + result: obj({ file: str() }), + accepts: { + 1: { args: (old: { path: string }) => ({ ...old, format: "md" }), result: (old: string) => ({ file: old }) }, + }, +}); +const Exports = local("example.exports", { + version: 1, + params: {}, + members: { start: method(obj({ path: str() }), str()), running: arr(unknown()) }, +}); +let release: (() => void) | undefined; + +/** The exports plugin with `job` (v1 or v2): its `run` waits for `release` once, its `finish` prints the outcome. */ +function exportsPlugin(code: string, job: typeof ExportV1 | typeof ExportV2) { + return defineFacet((host) => + host.serve(Exports, (_, self) => { + // One handle for either version of the job (the two definitions' types differ). + const handle = ( + self.job as unknown as ( + definition: unknown, + implementation: unknown, + ) => { + start(args: unknown, ctx: Context): Promise<{ readonly id: string }>; + readonly running: never; + } + )(job, { + run: async (args: { path: string }) => { + await new Promise((resolve) => { + release = resolve; + }); + return job.version === 1 ? `${args.path}.md` : { file: `${args.path}.out` }; + }, + finish: (outcome: unknown) => console.log(` finish on ${code}:`, JSON.stringify(outcome)), + }); + return { + start: async ({ path }: { path: string }, ctx: Context) => + (await handle.start(job.version === 1 ? { path } : { path, format: "html" }, ctx)).id, + running: handle.running, + }; + }), + ); +} + +const worker = createHost({ onError: () => {} }); +await worker.load([{ name: "@example/exports", facet: exportsPlugin("v1 code", ExportV1) }], context); +const exportsRef = await worker.get(Exports, {}, context); +await exportsRef.start({ path: "notes" }, context); +await new Promise((resolve) => setTimeout(resolve, 0)); +await worker.reload([{ name: "@example/exports", facet: exportsPlugin("v2 code", ExportV2) }], context); +console.log("4. after the reload, running shows the v1 run mapped:", JSON.stringify(exportsRef.running.value)); +release?.(); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(" running is empty again:", JSON.stringify(exportsRef.running.value)); + +await worker.close(context); +await tab.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/35-plugins/main.ts b/packages/chord/test/next/examples/35-plugins/main.ts new file mode 100644 index 000000000..e5fee1544 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/main.ts @@ -0,0 +1,110 @@ +// Plugins (slice 7c): plugin directories turned into facets, loaded on their hosts, watched, rebuilt and rolled out. +// A plugin is a directory with a `package.json` (its name) and one file per facet kind (`server.ts`, `worker.ts`); +// `createPlugins` discovers them, builds each facet into a script with esbuild (host packages external) and evaluates +// it on Node. Here (`plugins/`): a contracts-only plugin, `@example/notes` with a server and a worker facet showing one +// label, and `@example/clock`, which the example never edits. A server and a worker load them; the notes' label is +// edited on disk, the watcher marks the plugin, the app builds it and rolls it out (server, then worker), a holder sees +// one frame, and the clock plugin is untouched (not in the reports, its object the same). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/35-plugins/main.ts +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, type Host } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { memoryLinkPair } from "../../../../src/next/link/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { Clock, Index, Notes, Worker } from "./plugins/contracts/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-35-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +// 1. The component: the roots, and the facet kinds, each built for Node with Chord as a host package (kept external in +// the builds). The hosts give the modules themselves when they load (this process's Chord: a plugin's +// `import { defineFacet } from "@earendil-works/chord/next"` is the app's own instance). +const modules = { "@earendil-works/chord/next": chordNext }; +const node = { platform: "node", hostPackages: Object.keys(modules) } as const; +const plugins = createPlugins( + { + roots: [root], + kinds: { server: node, worker: node }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. A worker and a server joined by a memory link; each loads the plugins' facets of its kind (no core facets here). +type P = { readonly kind: "server" } | { readonly kind: "worker" }; +const [up, down] = memoryLinkPair(); +const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); +worker.connect(up, { principal: { kind: "server" } }, context); +const server = createHost

({ principal: { kind: "server" }, onError: () => {} }); +server.route( + Worker, + { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, + context, +); +const workerLoad = await plugins.load(worker, { kind: "worker", core: [], modules }, context); +const serverLoad = await plugins.load(server, { kind: "server", core: [], modules }, context); +console.log( + "1. discovered:", + plugins.discovery.plugins.map(({ name, facets }) => `${name} [${Object.keys(facets)}]`).join(", "), +); +console.log(" load failures: worker", workerLoad.failed.length, "| server", serverLoad.failed.length); +const notes = await server.get(Notes, {}, context); +const index = await server.get(Index, {}, context); +const clock = await server.get(Clock, {}, context); +await notes.add(context); +const evaluatedAt = clock.evaluatedAt.value; +console.log(" notes:", notes.label.value, "count", notes.count.value, "| index:", index.label.value); +const watch = await notes.label.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(" a frame on the server's reference:", JSON.stringify(call.value)); +}); + +// 3. An edit on disk (the app's editor, a git pull): the watcher marks the plugin dirty; nothing is applied yet. +await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); +await until(() => plugins.state.value.dirty.includes("@example/notes")); +console.log("2. marks after the edit:", JSON.stringify(plugins.state.value)); + +// 4. The app builds the dirty plugins and rolls the build out: the server first, then the worker. Each step reloads its +// host with the facets of its kind; the clock plugin was not built, so it keeps its facet and its object. +const steps = [plugins.step("server", server as Host

), plugins.step("worker", worker as Host

)]; +const built = await plugins.build(plugins.state.value.dirty, context); +console.log( + "3. built:", + built.facets.map(({ plugin, facet, kind }) => `${plugin}/${facet}: ${kind}`), +); +const rolled = await plugins.rollout(steps, built, context); +for (const { name, hosts } of rolled.steps) { + for (const host of hosts) { + if ("report" in host) { + console.log( + ` ${name}:`, + JSON.stringify(host.report.plugins), + "objects", + JSON.stringify(host.report.objects), + ); + } + } +} +await until(() => notes.label.value === "v2"); +console.log("4. notes:", notes.label.value, "count", notes.count.value, "| index:", index.label.value); +console.log(" clock untouched:", clock.evaluatedAt.value === evaluatedAt); +console.log(" marks after the rollout:", JSON.stringify(plugins.state.value)); + +await watch.stop(); +await server.close(context); +await worker.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json b/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json new file mode 100644 index 000000000..9cabc2e1a --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/clock", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts new file mode 100644 index 000000000..5bc4b6ee3 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts @@ -0,0 +1,9 @@ +// The clock plugin's server facet: serves `Clock` with the time its script was evaluated. The example never edits +// this plugin, so a reload leaves it as it is: same script, same facet, same object. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Clock } from "../contracts/contract.ts"; + +const evaluatedAt = new Date().toISOString(); + +export default defineFacet((host) => host.serve(Clock, () => ({ evaluatedAt: constant(evaluatedAt) }))); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts b/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts new file mode 100644 index 000000000..cbab43b1e --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts @@ -0,0 +1,21 @@ +// The contracts the plugins share: a contracts-only plugin (no facet file). A plugin that imports this file gets its +// own copy built into its script; the app's code declares the same contracts (same names, versions and shapes). + +import { hostKind, int, local, method, remote, str } from "@earendil-works/chord/next"; + +/** The worker's host kind: `Notes` lives there. */ +export const Worker = hostKind("example.worker"); + +/** Notes on the worker: a label, a count and `add`. */ +export const Notes = remote("example.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); + +/** The notes' label as the server shows it. */ +export const Index = local("example.index", { version: 1, params: {}, members: { label: str() } }); + +/** When the clock plugin's script was evaluated: it changes only when the plugin is reloaded. */ +export const Clock = local("example.clock", { version: 1, params: {}, members: { evaluatedAt: str() } }); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json b/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json new file mode 100644 index 000000000..d1c133527 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/contracts", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts new file mode 100644 index 000000000..249f65179 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The label both of the plugin's facets show; the example edits it. + +export const label = "v1"; diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json b/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts new file mode 100644 index 000000000..ed3ded889 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// The notes plugin's server facet: serves `Index` with the same label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Index } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Index, () => ({ label: constant(label) }))); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts new file mode 100644 index 000000000..b4f4c8ae4 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts @@ -0,0 +1,26 @@ +// The notes plugin's worker facet: serves `Notes`, its count kept in a document. + +import { constant, defineFacet, defineState, int, obj } from "@earendil-works/chord/next"; +import { Notes } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +export default defineFacet((host) => + host.serve(Notes, (_params, _principal, self) => { + const count = self.state(Count); + return { + variant: "main", + members: { + label: constant(label), + count: self.derive([count], ([value]) => value.n), + add: async (ctx) => { + await count.change((draft) => { + draft.n += 1; + }, ctx); + return count.value.n; + }, + }, + }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/main.ts b/packages/chord/test/next/examples/36-plugin-failures/main.ts new file mode 100644 index 000000000..103b1b3fb --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/main.ts @@ -0,0 +1,157 @@ +// Plugin failures (slice 7c): every failure is per plugin. At startup a plugin whose setup throws is left out and the +// others run (a consumer of what it would serve waits). At a reload a plugin whose new code fails keeps its old code and +// is listed in the report's `failed`: a syntax error (its build failed), a module that throws while it is evaluated +// (`mapStack` names the plugin's file and line), a factory that throws while the host stages it. The closure rule: a +// provider and its consumer edited together, the provider broken, the consumer kept on its old code too (its new code +// needs the provider's new version) and still working; the next good edit applies both. Last, a reload whose context +// aborts after its commit: the commit stands, and the component keeps what was committed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/36-plugin-failures/main.ts +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT, withAbortSignal } from "../../../../src/context/index.ts"; +import { int, local, str } from "../../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../../src/next/data/index.ts"; +import { createHost, type ReloadReport } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, mapStack } from "../../../../src/next-node/plugins/index.ts"; +import { Weather } from "./plugins/weather/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-36-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** Replaces `from` with `to` in the plugin file `file` (the copy). */ +async function edit(file: string, from: string, to: string): Promise { + const text = await readFile(join(root, file), "utf8"); + await writeFile(join(root, file), text.replace(from, to)); +} + +/** Waits until the watcher marks every plugin in `names` dirty, then builds the dirty plugins. */ +async function buildDirty(names: readonly string[]) { + await until(() => names.every((name) => plugins.state.value.dirty.includes(name))); + return plugins.build(plugins.state.value.dirty, context); +} + +/** A reload's report: what changed, its objects' fates, and each failed plugin with its error's mapped stack. */ +function print(report: ReloadReport): void { + console.log(" plugins:", JSON.stringify(report.plugins)); + const objects = report.objects.map((each) => `${each.object} ${each.kind}`); + if (objects.length > 0) console.log(" objects:", objects.join(", ")); + for (const { plugin, error } of report.failed) { + // The first line, and the second where it names the plugin's code: a build error, or a mapped stack frame. + const [message, next = ""] = mapStack(error).split("\n"); + const detail = !next.trimStart().startsWith("at ") || next.includes("/server.ts:") ? ` ${next.trim()}` : ""; + console.log(` failed ${plugin}: ${message}${detail}`); + } +} + +// The contracts the app reads (the plugins declare the same ones). +const Forecast = local("example.forecast", { version: 1, params: {}, members: { text: str() } }); +const Greeting = local("example.greeting", { version: 1, params: {}, members: { text: str() } }); +const Boot = local("example.boot", { version: 1, params: {}, members: { mode: str() } }); +const Counter = local("example.counter", { version: 1, params: {}, members: { start: int() } }); +const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); +const Status = local("example.status", { version: 1, params: {}, members: { text: str() } }); + +// The host packages: this process's Chord. +const modules = { "@earendil-works/chord/next": chordNext }; +const plugins = createPlugins( + { + roots: [root], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const host = createHost({ onError: () => {} }); + +// 1. Startup: the weather plugin's setup throws; it is left out and reported, the other plugins run. A reference to +// what it would serve waits (`failed`, cause `waiting`: not served here), and so does its consumer's object. +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); +for (const { plugin, error } of loaded.failed) console.log(`1. left out ${plugin}: ${(error as Error).message}`); +const weather = host.hold(Weather, {}, context); +const forecast = host.hold(Forecast, {}, context); +await until(() => forecast.state.value.kind === "failed"); +console.log(" weather:", JSON.stringify(weather.state.value)); +console.log(" forecast:", JSON.stringify(forecast.state.value)); +const greeting = await host.get(Greeting, {}, context); +const boot = await host.get(Boot, {}, context); +const counter = await host.get(Counter, {}, context); +const summary = await host.get(Summary, {}, context); +const status = await host.get(Status, {}, context); +const values = () => + [greeting.text.value, boot.mode.value, counter.start.value, summary.text.value] + .map((v) => JSON.stringify(v)) + .join(" "); +console.log(" running:", values()); + +// 2. Four bad edits at once. The greeter's label gets a syntax error; the boot module throws while it is evaluated; +// the counter's factory throws; the prices plugin moves its contract to version 2 and breaks its facet in the same +// edit, while the summary plugin (its copy of the contract now at version 2) builds fine. +await edit("greeter/label.ts", '"hello"', '"hello" +'); +await edit("boot/server.ts", "const Boot", 'throw new Error("boot: config missing");\n\nconst Boot'); +await edit("counter/server.ts", "() => ({ start: constant(1) })", '() => {\n\tthrow new Error("counter: no seed");\n}'); +await edit("prices/contract.ts", "version: 1", "version: 2"); +await edit("prices/server.ts", "constant(250)", "constant(250"); +const bad = await buildDirty(["@example/greeter", "@example/boot", "@example/counter", "@example/summary"]); +console.log("2. built:", bad.facets.map(({ plugin, kind }) => `${plugin}: ${kind}`).join(", ")); +// Each failed plugin keeps its old code; the summary's new code would ask for `Price` v2, which only the prices +// plugin's new (failed) code serves: the closure rule keeps the summary on its old code too. +print(await plugins.reload(host, bad, context)); +console.log(" running:", values()); +console.log(" still dirty:", JSON.stringify(plugins.state.value.dirty)); + +// 3. The fixes: the next build and reload apply every plugin, the prices provider and its consumer together. +await edit("greeter/label.ts", '"hello" +', '"hi"'); +await edit("boot/server.ts", 'throw new Error("boot: config missing");\n\n', ""); +await edit("boot/server.ts", 'constant("v1")', 'constant("v2")'); +await edit("counter/server.ts", '() => {\n\tthrow new Error("counter: no seed");\n}', "() => ({ start: constant(2) })"); +await edit("prices/server.ts", "constant(250", "constant(300)"); +const good = await buildDirty(["@example/greeter", "@example/boot", "@example/counter", "@example/prices"]); +console.log("3. fixed:"); +print(await plugins.reload(host, good, context)); +console.log(" running:", values()); +// The summary's new code waited for the new `Price` object (parked at the report): it runs once that one is live. +await until(() => summary.text.value.startsWith("price v2")); +console.log(" then:", values()); + +// 4. A reload whose context aborts after its commit, while it waits for its report: here the app shuts down the moment +// it sees the status object leave `live` (its new code waits for a new `Cache`, which takes 200 ms to open). The +// reload rejects `aborted`, but its commit stands: `details.committed` says what it applied, and the component's +// loaded set is what was committed (a later reload does not roll the cache plugin back). +console.log("4. status:", status.text.value); +await edit("cache/server.ts", 'const generation = "v1"', 'const generation = "v2"'); +const cache = await buildDirty(["@example/cache"]); +const abort = new AbortController(); +const watchStatus = await status.state.watch(context); +watchStatus.start(async (call) => { + if (call.kind !== "value") return; + console.log(" the status reference:", call.value.kind); + if (call.value.kind !== "live") abort.abort(new Error("the app shuts down")); +}); +try { + await plugins.reload(host, cache, withAbortSignal(abort.signal, context)); +} catch (error) { + if (!isChordError(error, "aborted")) throw error; + console.log(" rejected:", error.code, error.message, JSON.stringify(error.details)); +} +await until(() => status.text.value === "cache v2"); +console.log(" status:", status.text.value); +await edit("greeter/label.ts", '"hi"', '"hey"'); +print(await plugins.reload(host, await buildDirty(["@example/greeter"]), context)); +console.log(" after another reload:", status.text.value, "| greeting:", greeting.text.value); + +await watchStatus.stop(); +await host.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json new file mode 100644 index 000000000..2352aa044 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/boot", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts new file mode 100644 index 000000000..3dee3274f --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts @@ -0,0 +1,7 @@ +// The boot plugin's server facet: serves `Boot`. The example makes this module throw while it is evaluated. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const Boot = local("example.boot", { version: 1, params: {}, members: { mode: str() } }); + +export default defineFacet((host) => host.serve(Boot, () => ({ mode: constant("v1") }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json new file mode 100644 index 000000000..8403f6b07 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/cache", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts new file mode 100644 index 000000000..53fd0e247 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts @@ -0,0 +1,27 @@ +// The cache plugin's server facet: `Cache` opens a store per generation, which takes a while (a native); `Status` reads +// the cache of the plugin's generation. A reload to a new generation swaps `Status` in place; its new code needs a new +// `Cache` object, and its first dependency check waits until that one is open: the reload's report waits for it too. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const generation = "v1"; + +const Cache = local("example.cache", { version: 1, params: { generation: str() }, members: { generation: str() } }); +const Status = local("example.status", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => { + host.serve(Cache, (params, self) => { + self.open( + async () => { + await new Promise((resolve) => setTimeout(resolve, 200)); + return {}; + }, + () => {}, + ); + return { generation: constant(params.generation) }; + }); + host.serve(Status, (_params, self) => { + const cache = self.get(Cache, { generation }); + return { text: self.derive([cache.generation], ([value]) => `cache ${value}`) }; + }); +}); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json new file mode 100644 index 000000000..aa61f8c17 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/counter", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts new file mode 100644 index 000000000..08e2ea287 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts @@ -0,0 +1,7 @@ +// The counter plugin's server facet: serves `Counter`. The example makes its factory throw. + +import { constant, defineFacet, int, local } from "@earendil-works/chord/next"; + +const Counter = local("example.counter", { version: 1, params: {}, members: { start: int() } }); + +export default defineFacet((host) => host.serve(Counter, () => ({ start: constant(1) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json new file mode 100644 index 000000000..04a79d9ce --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/forecast", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts new file mode 100644 index 000000000..54a92d46f --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts @@ -0,0 +1,13 @@ +// The forecast plugin's server facet: a consumer of the weather plugin's `Weather`. + +import { defineFacet, local, str } from "@earendil-works/chord/next"; +import { Weather } from "../weather/contract.ts"; + +const Forecast = local("example.forecast", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Forecast, (_params, self) => { + const weather = self.get(Weather, {}); + return { text: self.derive([weather.celsius], ([celsius]) => `${celsius} degrees`) }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts new file mode 100644 index 000000000..bdfa4651d --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts @@ -0,0 +1,3 @@ +// The greeting's text; the example breaks it with a syntax error, then fixes it. + +export const greeting = "hello"; diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json new file mode 100644 index 000000000..8b85f433a --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/greeter", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts new file mode 100644 index 000000000..14f719119 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts @@ -0,0 +1,8 @@ +// The greeter plugin's server facet: serves `Greeting` with the text of `label.ts`. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; +import { greeting } from "./label.ts"; + +const Greeting = local("example.greeting", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Greeting, () => ({ text: constant(greeting) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts new file mode 100644 index 000000000..701bedbbb --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts @@ -0,0 +1,6 @@ +// The prices plugin's contract; the summary plugin imports (copies) it. The example moves it to version 2. + +import { int, local } from "@earendil-works/chord/next"; + +/** A price in cents. */ +export const Price = local("example.price", { version: 1, params: {}, members: { cents: int() } }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json new file mode 100644 index 000000000..257de583d --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/prices", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts new file mode 100644 index 000000000..56655bc13 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts @@ -0,0 +1,6 @@ +// The prices plugin's server facet: serves `Price`. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Price } from "./contract.ts"; + +export default defineFacet((host) => host.serve(Price, () => ({ cents: constant(250) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json new file mode 100644 index 000000000..6c54a9fa8 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/summary", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts new file mode 100644 index 000000000..3aad5f3e1 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts @@ -0,0 +1,14 @@ +// The summary plugin's server facet: a consumer of the prices plugin's `Price`, at the version its copy of the +// contract declares. + +import { defineFacet, local, str } from "@earendil-works/chord/next"; +import { Price } from "../prices/contract.ts"; + +const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Summary, (_params, self) => { + const price = self.get(Price, {}); + return { text: self.derive([price.cents], ([cents]) => `price v${Price.version}: ${cents} cents`) }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts new file mode 100644 index 000000000..9798b45f5 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts @@ -0,0 +1,6 @@ +// The weather plugin's contract; the forecast plugin imports (copies) it. + +import { int, local } from "@earendil-works/chord/next"; + +/** Today's temperature. */ +export const Weather = local("example.weather", { version: 1, params: {}, members: { celsius: int() } }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json new file mode 100644 index 000000000..d72a57935 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/weather", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts new file mode 100644 index 000000000..bbe12c2a7 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts @@ -0,0 +1,12 @@ +// The weather plugin's server facet: its setup throws (no API key configured), so the host leaves the plugin out at +// startup and runs the others. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Weather } from "./contract.ts"; + +const apiKey: string | undefined = undefined; + +export default defineFacet((host) => { + if (apiKey === undefined) throw new Error("weather: no API key configured"); + host.serve(Weather, () => ({ celsius: constant(21) })); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/app/index.ts b/packages/chord/test/next/examples/37-plugin-imports/app/index.ts new file mode 100644 index 000000000..b1182878a --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/app/index.ts @@ -0,0 +1,10 @@ +// The app's own code (base code): given to plugins as the host package `@example/app` (an ES module: a default export +// and named ones). A plugin imports it by that name, never by its path. + +/** The app's name and version. */ +export default { name: "example app", version: 3 }; + +/** Cents as a price. */ +export function currency(cents: number): string { + return `EUR ${(cents / 100).toFixed(2)}`; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs b/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs new file mode 100644 index 000000000..a0c39787c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs @@ -0,0 +1,7 @@ +// A CommonJS module of the app: given to plugins as the host package `@example/legacy`. + +module.exports = { + pad(text, width) { + return String(text).padStart(width, "."); + }, +}; diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js new file mode 100644 index 000000000..c9e1272e1 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js @@ -0,0 +1,6 @@ +// A small npm package (local, so the example needs no network): a plugin lists it in its `dependencies`, and the build +// inlines it into the plugin's script. + +export function greet(name) { + return `hello, ${name}`; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json new file mode 100644 index 000000000..40d2288d8 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json @@ -0,0 +1,7 @@ +{ + "name": "example-greeting", + "version": "1.0.0", + "private": true, + "type": "module", + "main": "index.js" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js new file mode 100644 index 000000000..ef13c18bc --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js @@ -0,0 +1,4 @@ +// A package a plugin keeps external (`chord.external`: a native addon, a package that reads its own files): the build +// leaves its import as it is, and the facet's `require` loads it from the plugin's `node_modules` at run time. + +exports.where = () => __filename; diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json new file mode 100644 index 000000000..0e7a2555f --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json @@ -0,0 +1,7 @@ +{ + "name": "example-native", + "version": "1.0.0", + "private": true, + "type": "commonjs", + "main": "index.js" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs b/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs new file mode 100644 index 000000000..d47f18b49 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs @@ -0,0 +1,11 @@ +// A stand-in for `npm install` (the example needs no network): run in a plugin's directory, it copies every +// `file:` dependency of the plugin's package.json into its node_modules, as npm would install it. + +import { cpSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +const { dependencies = {} } = JSON.parse(readFileSync("package.json", "utf8")); +for (const [name, spec] of Object.entries(dependencies)) { + if (!spec.startsWith("file:")) throw new Error(`fake npm: ${name}@${spec} is not a local package`); + cpSync(resolve(spec.slice("file:".length)), join("node_modules", name), { recursive: true }); +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/main.ts b/packages/chord/test/next/examples/37-plugin-imports/main.ts new file mode 100644 index 000000000..55258a655 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/main.ts @@ -0,0 +1,79 @@ +// What a facet may import (slice 7c §2, slice 12 §2). The build inlines a plugin's own files, files of other plugins of +// its root by relative path (a copied contract) and its npm `dependencies`; it leaves host packages (the app's +// instances: an ES module's default and named exports, a CommonJS module), `chord.external` packages and Node builtins +// external, answered when the script runs; a plugin's own files are read through `pluginFile`. Here (`plugins/`): the report plugin uses each of these; its +// dependencies are installed first (`installDependencies`; `fake-npm.mjs` stands in for npm: no network). The ruler +// plugin copies a file that imports a package, the refused plugin imports what a script may not, and the paths plugin +// uses `__dirname` and `__filename`: they are left out with their build errors. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/37-plugin-imports/main.ts +import { cp, mkdtemp, readdir, rm } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import * as app from "./app/index.ts"; +import { Report } from "./plugins/shapes/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example installs packages into a plugin: it works on a copy of `plugins/`, `app/` and `deps/` (the plugins' +// `file:` dependencies point there), removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-37-")); +for (const dir of ["plugins", "app", "deps"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} +const root = join(copy, "plugins"); + +// 1. The component. Its host packages: Chord, the app's ES module and a CommonJS module of the app (their names kept +// external in the server kind's builds, the modules given at load). Its npm command (here a stand-in): what +// `installDependencies` runs in a plugin's directory. +const modules = { + "@earendil-works/chord/next": chordNext, + "@example/app": app, + "@example/legacy": createRequire(import.meta.url)("./app/legacy.cjs"), +}; +const plugins = createPlugins( + { + roots: [root], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + install: { npmCommand: [process.execPath, fileURLToPath(new URL("./fake-npm.mjs", import.meta.url))] }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. Startup: every plugin whose build fails is left out with its errors (one per line: file, line, column, why). The +// report plugin's dependency is not installed yet. +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); +console.log("1. left out at startup:"); +for (const { plugin, error } of loaded.failed) { + const [first, ...errors] = (error as Error).message.split("\n"); + console.log(` ${plugin}: ${first}`); + for (const line of errors) console.log(` ${line}`); +} + +// 3. The report plugin's dependencies, installed into its own node_modules; then it builds, and a reload adds it. +await plugins.installDependencies("@example/report", context); +console.log("2. installed:", (await readdir(join(root, "report", "node_modules"))).join(", ")); +const built = await plugins.build(["@example/report"], context); +for (const facet of built.facets) { + if (facet.kind !== "script") continue; + const files = facet.inputs.map(({ file }) => file.slice(root.length + 1)); + console.log(` built ${facet.plugin}/${facet.facet}: read ${files.join(", ")}; copied ${facet.copied.length} file`); +} +const report = await plugins.reload(host, built, context); +// The reload passes the complete set: the refused plugins fail again and stay out. +const stillRefused = report.failed.map(({ plugin }) => plugin).join(", "); +console.log(" reload:", JSON.stringify(report.plugins), "| still refused:", stillRefused); +const text = (await host.get(Report, {}, context)).text.value; +console.log("3. the report:", text); + +await host.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json new file mode 100644 index 000000000..fc2e85a6f --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/paths", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts new file mode 100644 index 000000000..b3170c2a4 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts @@ -0,0 +1,10 @@ +// The paths plugin's server facet: a script has no `__dirname` or `__filename` (it is evaluated with no file of its +// own); a file of the plugin is read through `pluginFile`. Each use is a build error. + +import { join } from "node:path"; +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => { + console.log(join(__dirname, "data.json")); + console.log(__filename); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts new file mode 100644 index 000000000..fbb6085ae --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts @@ -0,0 +1,3 @@ +// A file of the refused plugin, imported dynamically by its facet (refused). + +export const other = 1; diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json new file mode 100644 index 000000000..74f126ebd --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/refused", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts new file mode 100644 index 000000000..437d38020 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts @@ -0,0 +1,11 @@ +// The refused plugin's server facet: imports a facet script may not have, each a build error with its file and line. +// A script is evaluated with no module identity: no `import.meta` (a file of the plugin is read through `pluginFile`), +// no literal dynamic `import()`; and base code is reached through a host package, not by its path. + +import { defineFacet } from "@earendil-works/chord/next"; +import app from "../../app/index.ts"; + +export default defineFacet(() => { + console.log(import.meta.url, app.name); + void import("./other.ts"); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json new file mode 100644 index 000000000..ef32822cb --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json @@ -0,0 +1 @@ +{ "vat": 20 } diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts new file mode 100644 index 000000000..ac3f7fe53 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts @@ -0,0 +1,21 @@ +// Types for the report plugin's imports that its directory does not hold: the app's host packages (an app ships their +// types with its plugin template) and its dependencies before `npm install` (their own types once installed). + +declare module "@example/app" { + const app: { readonly name: string; readonly version: number }; + export default app; + export function currency(cents: number): string; +} + +declare module "@example/legacy" { + const legacy: { pad(text: string, width: number): string }; + export default legacy; +} + +declare module "example-greeting" { + export function greet(name: string): string; +} + +declare module "example-native" { + export function where(): string; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json new file mode 100644 index 000000000..751b92827 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json @@ -0,0 +1,12 @@ +{ + "name": "@example/report", + "private": true, + "type": "module", + "dependencies": { + "example-greeting": "file:../../deps/example-greeting", + "example-native": "file:../../deps/example-native" + }, + "chord": { + "external": ["example-native"] + } +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts new file mode 100644 index 000000000..3aed5ad9c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts @@ -0,0 +1,32 @@ +// The report plugin's server facet: one of each import a facet may have. + +import { readFileSync } from "node:fs"; +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { pluginFile } from "@earendil-works/chord/next/plugins"; +// Host packages: an ES module (default and named imports) and a CommonJS one, the app's own instances. +import app, { currency } from "@example/app"; +import legacy from "@example/legacy"; +// An npm dependency: inlined into this script by the build (installed first). +import { greet } from "example-greeting"; +// A `chord.external` package: left external, loaded from this plugin's node_modules when the script runs. +import { where } from "example-native"; +// A contract of another plugin of this root, by relative path: a copy built into this script. +import { Report } from "../shapes/contract.ts"; + +// A file of this plugin, read at run time by its absolute path. +const rates = JSON.parse(readFileSync(pluginFile("data/rates.json"), "utf8")) as { readonly vat: number }; + +export default defineFacet((host) => + host.serve(Report, () => ({ + text: constant( + [ + `${app.name} v${app.version}`, + currency(1250), + legacy.pad("vat", 6), + `${rates.vat}%`, + greet("plugins"), + where().split("/").slice(-3).join("/"), + ].join(" | "), + ), + })), +); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json new file mode 100644 index 000000000..49a09428c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/ruler", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts new file mode 100644 index 000000000..ec9a0dd74 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts @@ -0,0 +1,9 @@ +// The ruler plugin's server facet: imports a file of the shapes plugin that imports a package. Refused at build: a +// copied file may import only host packages and files of its own plugin. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Millimetres } from "../shapes/units.ts"; + +export default defineFacet(() => { + console.log(Millimetres); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts new file mode 100644 index 000000000..9dcc55e1c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts @@ -0,0 +1,7 @@ +// The shapes plugin's contract (a contracts-only plugin): other plugins import this file and get their own copy of it. +// A copied file may import host packages and files of its own plugin only. + +import { local, str } from "@earendil-works/chord/next"; + +/** A report: its lines. */ +export const Report = local("example.report", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json new file mode 100644 index 000000000..8bedbd5da --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/shapes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts new file mode 100644 index 000000000..c5a8a13b9 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts @@ -0,0 +1,7 @@ +// A file of the shapes plugin that imports a package (not a host package): fine for the shapes plugin itself, but a +// plugin importing this file would copy it, and a copied file may not import packages (the ruler plugin shows it). + +import { num } from "@earendil-works/pi-schema"; + +/** A length in millimetres. */ +export const Millimetres = num({ minimum: 0 }); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts b/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts new file mode 100644 index 000000000..c2c72bda8 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts @@ -0,0 +1,4 @@ +// The app's own code (base code): the plugins component watches it and marks `base` when it changes; Chord restarts +// nothing for it, the app restarts its hosts. + +export const appName = "example"; diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json new file mode 100644 index 000000000..201740a8b --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/extra", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts new file mode 100644 index 000000000..dff218b90 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts @@ -0,0 +1,8 @@ +// The extra plugin's server facet: not in plugins/ at first; the example copies the plugin in while the app runs. It +// declares its type itself (the same shape as `labelOf("extra")`). + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const Extra = local("example.label.extra", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Extra, () => ({ text: constant("extra on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/main.ts b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts new file mode 100644 index 000000000..53313cf75 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts @@ -0,0 +1,154 @@ +// The watcher's marks and rollouts (slice 7c §5). The watcher marks what changed and never applies it: `dirty` for a +// plugin whose file changed (a contract file: every plugin that copies it), `install` for a changed `package.json`, +// `set` when a plugin directory or facet file is added or removed, `base` when the app's own code changes. The app +// builds the dirty plugins and rolls the build out over its hosts in order; a rollout clears the marks of the plugins +// it applied on every step. Here: a plugin added, one removed and one dropping its worker facet in one rollout; a +// rollout stopping at a step that rejects; a failed plugin staying dirty while the others clear; an edit during a build +// keeping its mark. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/38-watch-and-rollout/main.ts +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, type Host, type ReloadReport } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, type RolloutResult, type RolloutStep } from "../../../../src/next-node/plugins/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/` and `app/`, removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-38-")); +for (const dir of ["plugins", "app"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} +const root = join(copy, "plugins"); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** Replaces `from` with `to` in `file` (under the copy). */ +async function edit(file: string, from: string, to: string): Promise { + const text = await readFile(join(copy, file), "utf8"); + await writeFile(join(copy, file), text.replace(from, to)); +} + +/** Waits until every plugin of `names` is marked dirty. */ +const dirty = (...names: string[]) => until(() => names.every((name) => plugins.state.value.dirty.includes(name))); + +/** The marks, in one line. */ +const marks = () => JSON.stringify(plugins.state.value); + +/** What a step's report says, in one line. */ +const line = (report: ReloadReport) => + JSON.stringify({ ...report.plugins, failed: report.failed.map(({ plugin }) => plugin) }); + +/** A rollout, step by step. */ +function print(result: RolloutResult): void { + for (const { name, hosts } of result.steps) { + for (const host of hosts) if ("report" in host) console.log(` ${name}:`, line(host.report)); + } + if (result.stopped !== undefined) { + console.log(` stopped at ${result.stopped.step}: ${(result.stopped.error as Error).message}`); + } +} + +// The host packages: this process's Chord, kept external in Node builds. +const modules = { "@earendil-works/chord/next": chordNext }; +const node = { platform: "node", hostPackages: Object.keys(modules) } as const; +const plugins = createPlugins( + { + roots: [root], + kinds: { server: node, worker: node }, + // The app's own code: a change marks `base`. + base: [join(copy, "app")], + // A stand-in for npm (no network; these plugins have no dependencies). + install: { npmCommand: [process.execPath, "-e", ""] }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const server = createHost({ onError: () => {} }); +const worker = createHost({ onError: () => {} }); +await plugins.load(server, { kind: "server", core: [], modules }, context); +await plugins.load(worker, { kind: "worker", core: [], modules }, context); +const steps = [plugins.step("server", server as Host), plugins.step("worker", worker as Host)]; +console.log("1. loaded:", plugins.discovery.plugins.map(({ name }) => name).join(", "), "| marks:", marks()); + +// 2. Each kind of change and its mark. +await edit("plugins/notes/label.ts", "notes v1", "notes v2"); +await dirty("@example/notes"); +console.log("2. a file of notes edited:", marks()); +await edit("plugins/contracts/contract.ts", "one type per plugin", "one type per plugin (edited)"); +await dirty("@example/shared", "@example/tags"); +console.log(" the shared contract edited:", marks()); +await edit("plugins/tags/package.json", '"private": true', '"private": true,\n\t"description": "tags"'); +await until(() => plugins.state.value.install.includes("@example/tags")); +console.log(" tags' package.json edited:", marks()); +await plugins.installDependencies("@example/tags", context); +console.log(" tags installed:", marks()); +await cp(fileURLToPath(new URL("./incoming/extra/", import.meta.url)), join(root, "extra"), { recursive: true }); +await rm(join(root, "old"), { recursive: true }); +await rm(join(root, "tags", "worker.ts")); +await until(() => plugins.state.value.set); +// Deleting old's directory deleted its package.json too: an install mark, which goes with the next discovery. +console.log(" extra added, old removed, tags' worker facet deleted:", marks()); +await edit("app/config.ts", '"example"', '"example 2"'); +await until(() => plugins.state.value.base); +console.log(" the app's own code edited:", marks()); + +// 3. One build of the dirty plugins (discovery runs again first: `set`), rolled out server then worker. The server adds +// extra and removes old; the worker leaves tags out (it has no worker facet any more). The marks clear, but `base`: +// that one is the app's to act on (it restarts its hosts). +const built = await plugins.build(plugins.state.value.dirty, context); +console.log("3. built:", built.facets.map(({ plugin, facet }) => `${plugin}/${facet}`).join(", ")); +console.log(" removed:", JSON.stringify(built.removed)); +print(await plugins.rollout(steps, built, context)); +console.log(" marks:", marks()); + +// 4. A rollout stops at a step that rejects (a host the app cannot reach): the steps after it do not run, and the marks +// stay (the plugin was not applied everywhere). The next rollout clears them. +const unreachable: RolloutStep = { + name: "edge", + kinds: ["server"], + reload: async () => { + throw new Error("the edge host is unreachable"); + }, +}; +await edit("plugins/notes/label.ts", "notes v2", "notes v3"); +await dirty("@example/notes"); +const notes = await plugins.build(plugins.state.value.dirty, context); +console.log("4. rollout over server, edge, worker:"); +print(await plugins.rollout([steps[0]!, unreachable, steps[1]!], notes, context)); +console.log(" marks:", marks()); +print(await plugins.rollout(steps, notes, context)); +console.log(" marks after the next rollout:", marks()); + +// 5. A failed plugin stays dirty, the others clear: notes gets a syntax error, tags a good edit. +await edit("plugins/notes/label.ts", '"notes v3"', '"notes v4" +'); +await edit("plugins/tags/server.ts", "tags on the server", "tags v2 on the server"); +await dirty("@example/notes", "@example/tags"); +console.log("5. notes broken, tags edited:"); +print(await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context)); +console.log(" marks:", marks()); + +// 6. An edit after the build started keeps its mark: the rollout applies the build, but the newer edit is not in it. +await edit("plugins/notes/label.ts", '"notes v4" +', '"notes v4"'); +await dirty("@example/notes"); +const early = await plugins.build(plugins.state.value.dirty, context); +await edit("plugins/notes/label.ts", '"notes v4"', '"notes v5"'); +// The edit's event arrives on its own time (the mark is set already, so no change shows it). +await new Promise((resolve) => setTimeout(resolve, 200)); +console.log("6. built before the second edit:"); +print(await plugins.rollout(steps, early, context)); +console.log(" marks:", marks()); +print(await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context)); +console.log(" marks after building again:", marks()); + +await server.close(context); +await worker.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts new file mode 100644 index 000000000..107b2fdbc --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts @@ -0,0 +1,9 @@ +// The contracts the plugins share (a contracts-only plugin): every plugin importing this file has a copy of it, so an +// edit here marks every one of them dirty. + +import { local, str } from "@earendil-works/chord/next"; + +/** The label a plugin shows, one type per plugin. */ +export function labelOf(plugin: string) { + return local(`example.label.${plugin}`, { version: 1, params: {}, members: { text: str() } }); +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json new file mode 100644 index 000000000..dfc4cefdc --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/shared", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts new file mode 100644 index 000000000..9dd0e6455 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The notes' label; the example edits it. + +export const label = "notes v1"; diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts new file mode 100644 index 000000000..23a936014 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts @@ -0,0 +1,9 @@ +// The notes plugin's server facet: its label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(labelOf("notes"), () => ({ text: constant(`${label} on the server`) })), +); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts new file mode 100644 index 000000000..e01fc1159 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts @@ -0,0 +1,9 @@ +// The notes plugin's worker facet: its label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(labelOf("notes"), () => ({ text: constant(`${label} on the worker`) })), +); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json new file mode 100644 index 000000000..2db7fdd46 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/old", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts new file mode 100644 index 000000000..afb0165f9 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts @@ -0,0 +1,6 @@ +// The old plugin's server facet; the example deletes the plugin's directory. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("old"), () => ({ text: constant("old on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json new file mode 100644 index 000000000..558ed9ed3 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/tags", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts new file mode 100644 index 000000000..8b8ce9d68 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts @@ -0,0 +1,6 @@ +// The tags plugin's server facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts new file mode 100644 index 000000000..7cbeb3ae4 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts @@ -0,0 +1,6 @@ +// The tags plugin's worker facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the worker") }))); diff --git a/packages/chord/test/next/examples/39-browser-build/main.ts b/packages/chord/test/next/examples/39-browser-build/main.ts new file mode 100644 index 000000000..f7a05af4d --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/main.ts @@ -0,0 +1,62 @@ +// A browser build (slice 7c §2): a facet of a browser kind (`web: { platform: "browser", … }`) is built for the browser. Its +// stylesheets become one CSS file (`css`), an asset imported by URL (`./logo.svg?url`) a content-hashed file whose +// import evaluates to its URL under the app's `publicPath`. The build is plain data: script, stylesheet and assets, +// what the app serves to its pages (slice 11 ships it); a page evaluates the script with its host packages, as Node +// does here. A Node builtin is refused in a browser build. The example only reads `plugins/`: no copy. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/39-browser-build/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, evaluateFacet } from "../../../../src/next-node/plugins/index.ts"; + +const context = BACKGROUND_CONTEXT; +const root = fileURLToPath(new URL("./plugins/", import.meta.url)); + +// 1. The component: `web` is a browser kind; its assets are served under `/plugin-assets/`. +const hostPackages = { "@earendil-works/chord/next": chordNext }; +const plugins = createPlugins( + { + roots: [root], + kinds: { web: { platform: "browser", hostPackages: Object.keys(hostPackages) } }, + publicPath: "/plugin-assets/", + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. The first build: every discovered plugin. +const built = await plugins.build([], context); +const gallery = built.facets.find(({ plugin }) => plugin === "@example/gallery")!; +const bad = built.facets.find(({ plugin }) => plugin === "@example/bad-web")!; +if (gallery.kind !== "script" || bad.kind !== "failed") throw new Error("unexpected build results"); +console.log("1. @example/gallery/web: script of", gallery.source.length, "bytes, digest", gallery.digest.slice(0, 12)); +console.log(" stylesheet:", gallery.css?.name, JSON.stringify(gallery.css?.text.split("\n")[0])); +for (const asset of gallery.assets) console.log(" asset:", asset.name, `(${asset.bytes.length} bytes)`); +console.log(" the asset's URL in the script:", /\/plugin-assets\/[^"]+/.exec(gallery.source)?.[0]); +console.log(" read:", gallery.inputs.map(({ file }) => file.slice(root.length)).join(", ")); +console.log("2. @example/bad-web/web:", bad.errors.join("; ")); + +// 3. Plain data: a structured clone (what a message to a page or a worker carries) is the same build. +const shipped = structuredClone(built); +const copy = shipped.facets.find(({ plugin }) => plugin === "@example/gallery")!; +console.log( + "3. cloned:", + copy.kind === "script" && copy.digest === gallery.digest, + "| its plugins:", + Object.keys(shipped.plugins).join(", "), +); + +// 4. Where it runs (a page; Node stands in here), the script is evaluated with the host packages and loaded. +const Gallery = local("example.gallery", { version: 1, params: {}, members: { logo: str() } }); +if (copy.kind === "script") { + const facet = evaluateFacet(copy, { hostPackages, plugin: shipped.plugins["@example/gallery"]! }); + const page = createHost({ onError: () => {} }); + await page.load([{ name: "@example/gallery", facet }], context); + console.log("4. the gallery's logo on the page:", (await page.get(Gallery, {}, context)).logo.value); + await page.close(context); +} + +plugins.close(); diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json new file mode 100644 index 000000000..6196d6fa7 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/bad-web", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts new file mode 100644 index 000000000..8ae46e262 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts @@ -0,0 +1,8 @@ +// The bad-web plugin's web facet: a Node builtin has no place in a browser build (refused at build). + +import { join } from "node:path"; +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => { + console.log(join("a", "b")); +}); diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts new file mode 100644 index 000000000..808a676b6 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts @@ -0,0 +1,9 @@ +// Types for what a browser build imports besides code: a stylesheet (collected into the build's one CSS file) and an +// asset by URL (`./file.svg?url`: the URL the app serves it under, as a string). + +declare module "*.css" {} + +declare module "*?url" { + const url: string; + export default url; +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg new file mode 100644 index 000000000..27de953fd --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg @@ -0,0 +1 @@ + diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css new file mode 100644 index 000000000..cbc3f629a --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css @@ -0,0 +1,8 @@ +.gallery { + display: grid; + gap: 8px; +} + +.gallery img { + width: 64px; +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json new file mode 100644 index 000000000..618ee3124 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/gallery", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts new file mode 100644 index 000000000..3755d990c --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts @@ -0,0 +1,10 @@ +// The gallery plugin's web facet, built for a browser: its stylesheet is collected into one CSS file, its logo is an +// asset imported by URL (content-hashed, under the app's `publicPath`). + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; +import "./gallery.css"; +import logo from "./assets/logo.svg?url"; + +const Gallery = local("example.gallery", { version: 1, params: {}, members: { logo: str() } }); + +export default defineFacet((host) => host.serve(Gallery, () => ({ logo: constant(logo) }))); diff --git a/packages/chord/test/next/examples/40-http/main.ts b/packages/chord/test/next/examples/40-http/main.ts new file mode 100644 index 000000000..78dbca808 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/main.ts @@ -0,0 +1,56 @@ +// HTTP (slice 10): a server answers HTTP requests with routes that plugins contribute. The notes plugin (`plugins/`) +// contributes a page, a JSON route and a file download; the server's own code only builds the component, says where +// Chord's own listener listens, and loads the plugins. Every answer carries the security baseline's headers. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/40-http/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +// 1. A server host and its HTTP component; its settings are arguments: Chord's own listener listens on a free +// loopback port. Nobody signs in here: no authenticator, every request is anonymous. +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); + +// 2. The plugins: the component's facet (its registries and the listener) goes first, then every plugin's server +// facet. The listener opens once they are published. +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +// 3. Some requests, printed like curl would. +async function curl(path: string, headers: Record = {}): Promise { + const response = await fetch(`http://127.0.0.1:${port}${path}`, { headers }); + const range = headers.Range === undefined ? "" : ` (Range: ${headers.Range})`; + console.log(`GET ${path}${range} -> ${response.status} ${response.headers.get("content-type")}`); + const download = response.headers.get("content-disposition"); + if (download !== null) console.log(` ${download}`); + console.log(` ${JSON.stringify((await response.text()).replace(/nonce="[^"]+"/, 'nonce="…"'))}`); +} + +await curl("/"); +await curl("/notes/hello"); +await curl("/notes/hello/download", { Range: "bytes=0-6" }); +await curl("/notes/nope/download"); +await curl("/nowhere"); +const baseline = await fetch(`http://127.0.0.1:${port}/notes/hello`); +for (const name of ["cache-control", "x-content-type-options", "x-frame-options"]) { + console.log(` ${name}: ${baseline.headers.get(name)}`); +} + +// 4. Close the component first: it refuses new requests and waits for running ones (the host's close would seal them +// at once). Then the host. +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md b/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md new file mode 100644 index 000000000..13cbb3050 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md @@ -0,0 +1,3 @@ +# Hello + +A note served as a download. diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/package.json b/packages/chord/test/next/examples/40-http/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/server.ts b/packages/chord/test/next/examples/40-http/plugins/notes/server.ts new file mode 100644 index 000000000..48e6b9388 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/server.ts @@ -0,0 +1,29 @@ +// The notes plugin's server facet: a page, a JSON route and a file download, contributed to the HTTP component's +// registry. + +import { defineFacet } from "@earendil-works/chord/next"; +import { fileResponse, Http } from "@earendil-works/chord/next/http"; +import { pluginFile } from "@earendil-works/chord/next/plugins"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + // A page: a link from another site may open it. Its inline script carries the server's nonce (the CSP). + "GET /": { + page: true, + handler: async (request) => ({ + kind: "html", + body: `

Notes

`, + }), + }, + // `:name` is one path segment, decoded. + "GET /notes/:name": async (request) => ({ kind: "json", value: { note: request.params.name } }), + // A file of this plugin, saved under its name, with ranges. + "GET /notes/:name/download": async (request) => + fileResponse(pluginFile(`files/${request.params.name}.md`), { + download: `${request.params.name}.md`, + ranges: true, + }), + }, + }), +); diff --git a/packages/chord/test/next/examples/41-workers/main.ts b/packages/chord/test/next/examples/41-workers/main.ts new file mode 100644 index 000000000..35460577b --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/main.ts @@ -0,0 +1,68 @@ +// Workers (slice 10): the server routes a session's address to its own worker process. The first reference to a +// session starts its worker; once nobody uses it for `idleMs`, the worker stops. The worker's code is the sessions +// plugin (`plugins/`), built by the server and shipped to the worker. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/41-workers/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Greeting, Session } from "./plugins/sessions/contract.ts"; + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); + +// 1. The server, and its workers: one forked process per session, stopped after 300 ms without use. +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-41:${process.pid}:${address.key}`, + idleMs: () => 300, + plugins, + onError: console.error, + }, + context, +); +await server.load([], context); + +// 2. Every change of the workers' state, printed. +const watch = await workers.state.watch(context); +watch.start(async (call) => { + if (call.kind !== "value") return; + const entries = Object.entries(call.value).map(([key, state]) => `${key} ${state.status}`); + console.log(` workers: ${entries.join(", ") || "none"}`); +}); + +// 3. The first reference starts the worker; the greeting comes from that process. +console.log("1. getting S1's greeting"); +const greeting = await server.get(Greeting, { id: "S1" }, context); +console.log(`2. "${greeting.text.value}" (worker pid ${greeting.pid.value}, server pid ${process.pid})`); + +// 4. Releasing it leaves the worker idle: 300 ms later it stops. +greeting.release(); +console.log("3. released; waiting for the idle stop"); +while (Object.keys(workers.state.value).length > 0) { + await new Promise((resolve) => setTimeout(resolve, 50)); +} +console.log("4. stopped"); + +await watch.stop(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts new file mode 100644 index 000000000..0ef89bfff --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's greeting, served by the worker at the session's address. */ +export const Greeting = remote("example.greeting", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str(), pid: int() } } }, +}); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts new file mode 100644 index 000000000..79ee9efca --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts @@ -0,0 +1,11 @@ +// The sessions plugin's worker facet: each worker greets with its process id. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Greeting } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Greeting, ({ id }) => ({ + variant: "main", + members: { text: constant(`hello from session ${id}`), pid: constant(process.pid) }, + })), +); diff --git a/packages/chord/test/next/examples/41-workers/worker.ts b/packages/chord/test/next/examples/41-workers/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/42-worker-crash/main.ts b/packages/chord/test/next/examples/42-worker-crash/main.ts new file mode 100644 index 000000000..37297f924 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/main.ts @@ -0,0 +1,67 @@ +// A worker crash (slice 10): workers never restart by themselves. A crashed worker's address stays failed, its +// holders see `failed` with the error, and nothing forks until the app retries (a Retry button) or a rollout ships new +// code. Here: a session crashes, stays failed for a while, then Retry brings it back. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/42-worker-crash/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Process, Session } from "./plugins/sessions/contract.ts"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: () => {} }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-42:${process.pid}:${address.key}`, + plugins, + onError: (error) => console.log(` reported: ${error.kind}`), + }, + context, +); +await server.load([], context); + +// 1. A held session (a tab showing it): its worker runs. +const session = server.hold(Process, { id: "S1" }, context); +const state = () => session.state.value.kind; +while (state() !== "live") await sleep(10); +console.log(`1. S1 runs in process ${session.pid.value}`); + +// 2. It crashes. The address is failed, and the holder sees why; it stays on the address's path. +await session.crash(context); +while (state() !== "failed") await sleep(10); +const failed = workers.state.value["session|S1"]; +console.log(`2. crashed: ${failed?.status === "failed" ? failed.error.message : failed?.status}`); + +// 3. No restart loop: a second later it is still failed, and no other worker was started. +await sleep(1_000); +console.log(`3. a second later: ${workers.state.value["session|S1"]?.status}, the holder: ${state()}`); + +// 4. Retry (the app's button): the worker starts again and the same holder binds to it. +await workers.retry(Session({ id: "S1" }), context); +while (state() !== "live") await sleep(10); +console.log(`4. after Retry: S1 runs in process ${session.pid.value}`); + +session.release(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts new file mode 100644 index 000000000..0a995c248 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, method, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's process: its pid, and a way to make it crash. */ +export const Process = remote("example.process", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { pid: int() }, methods: { crash: method() } } }, +}); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts new file mode 100644 index 000000000..cb8cae63c --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts @@ -0,0 +1,17 @@ +// The sessions plugin's worker facet: its process id, and `crash`, which ends the process (a bug, an out-of-memory). + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Process } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Process, () => ({ + variant: "main", + members: { + pid: constant(process.pid), + crash: async () => { + setTimeout(() => process.exit(13), 10); + return null; + }, + }, + })), +); diff --git a/packages/chord/test/next/examples/42-worker-crash/worker.ts b/packages/chord/test/next/examples/42-worker-crash/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/43-rollout/main.ts b/packages/chord/test/next/examples/43-rollout/main.ts new file mode 100644 index 000000000..5fd443756 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/main.ts @@ -0,0 +1,85 @@ +// A rollout across the server and its workers (slice 10): an edit of a plugin is built once by the server and rolled +// out step by step, the server first, then every running worker; each host reports what changed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/43-rollout/main.ts +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, type RolloutHost } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Banner, Label, Session } from "./plugins/notes/contract.ts"; + +/** Waits until `done()` holds (a watcher's mark, a frame from a worker), checking every 10 ms. */ +async function waitUntil(done: () => boolean): Promise { + while (!done()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** One line for what a rollout step did on one of its hosts. */ +function describe(host: RolloutHost): string { + if ("report" in host) return `changed ${host.report.plugins.changed.join(", ")}`; + if ("stopped" in host) return "stopped"; + return "failed"; +} + +// The example edits a plugin: it works on a copy of `plugins/`. +const root = await mkdtemp(join(tmpdir(), "chord-example-43-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +const node = { platform: "node", hostPackages: ["@earendil-works/chord/next"] } as const; +const plugins = createPlugins( + { roots: [root], kinds: { server: node, worker: node }, onError: console.error }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-43:${process.pid}:${address.key}`, + plugins, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [], modules: { "@earendil-works/chord/next": chordNext } }, context); + +// 1. The server's banner and two running sessions. +const banner = await server.get(Banner, {}, context); +const s1 = await server.get(Label, { id: "S1" }, context); +const s2 = await server.get(Label, { id: "S2" }, context); +console.log(`1. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); + +// 2. An edit: the watcher marks the plugin; one build serves both kinds. +await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); +await waitUntil(() => plugins.state.value.dirty.includes("@example/notes")); +const built = await plugins.build(plugins.state.value.dirty, context); + +// 3. The rollout: the server, then the workers (each running one reloads in parallel). +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, context); +for (const step of result.steps) { + for (const host of step.hosts) console.log(`2. ${step.name}: ${host.host} ${describe(host)}`); +} + +// 4. Every host shows the new label (the workers' frames arrive over their links). +await waitUntil(() => s2.text.value === "S2: v2"); +console.log(`3. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); +console.log(`4. still dirty: ${JSON.stringify(plugins.state.value.dirty)}`); + +banner.release(); +s1.release(); +s2.release(); +await workers.close(context); +await server.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts new file mode 100644 index 000000000..86d192d7e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts @@ -0,0 +1,16 @@ +// The notes' contracts: a label on the server, and one in each session's worker. + +import { hostKind, local, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** The label as the server shows it. */ +export const Banner = local("example.banner", { version: 1, params: {}, members: { text: str() } }); + +/** The label as a session's worker shows it. */ +export const Label = remote("example.label", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str() } } }, +}); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts new file mode 100644 index 000000000..c8c560a6e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The label every host shows; the example edits it. + +export const label = "v1"; diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts new file mode 100644 index 000000000..ca8c22e00 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// The notes plugin's server facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Banner } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Banner, () => ({ text: constant(`server: ${label}`) }))); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts new file mode 100644 index 000000000..972e91925 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts @@ -0,0 +1,9 @@ +// The notes plugin's worker facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Label } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(Label, ({ id }) => ({ variant: "main", members: { text: constant(`${id}: ${label}`) } })), +); diff --git a/packages/chord/test/next/examples/43-rollout/worker.ts b/packages/chord/test/next/examples/43-rollout/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/44-http-wraps/main.ts b/packages/chord/test/next/examples/44-http-wraps/main.ts new file mode 100644 index 000000000..af970636c --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/main.ts @@ -0,0 +1,37 @@ +// Wraps (slice 10): plugins wrap routes, their own or everyone's, in phases (`observe`, `headers`, `transform`, +// outermost first). Here the notes plugin wraps its own route to add a header, and an audit plugin wraps every route +// to log it; the audit plugin needs no route of its own. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/44-http-wraps/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +for (const path of ["/notes", "/missing"]) { + console.log(`GET ${path}`); + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(` -> ${response.status}, Cache-Control: ${response.headers.get("cache-control")}`); +} +// The 404 has no route, so no wrap ran for it: the audit log shows only /notes, and it keeps the default +// `Cache-Control: no-store` that the notes wrap replaced on its own route. + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json new file mode 100644 index 000000000..10a68118f --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/audit", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts new file mode 100644 index 000000000..e4259d470 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts @@ -0,0 +1,24 @@ +// The audit plugin: no routes of its own, one wrap around every route (`scope: "all"`), outermost (`observe`). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + wraps: [ + { + name: "log", + phase: "observe", + scope: "all", + wrap: async (request, next, ctx) => { + const response = await next(request); + const status = "status" in response ? response.status : 200; + console.log( + ` audit: ${ctx.principal === undefined ? "anonymous" : "someone"} ${request.method} ${request.path} ${status}`, + ); + return response; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts new file mode 100644 index 000000000..0c2a83103 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts @@ -0,0 +1,20 @@ +// The notes plugin: a route, and its own wrap (around its own routes only) adding a header. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { "GET /notes": async () => ({ kind: "json", value: ["buy milk", "call mom"] }) }, + wraps: [ + { + name: "cache", + phase: "headers", + wrap: async (request, next) => { + const response = await next(request); + return { ...response, headers: { ...response.headers, "Cache-Control": "max-age=60" } }; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/45-plugin-socket/main.ts b/packages/chord/test/next/examples/45-plugin-socket/main.ts new file mode 100644 index 000000000..9f10cd8b8 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/main.ts @@ -0,0 +1,64 @@ +// A plugin socket (slice 10): a plugin contributes a WebSocket endpoint for its own protocol (a terminal, voice). The +// server checks the origin and authenticates first: without a principal the upgrade is refused before plugin code. +// Here the app's authenticator reads a principal from a header (a real app: a signed cookie, `readCookie`). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/45-plugin-socket/main.ts +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +// The app's core facet: who someone is. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Authenticators, (request) => { + const name = request.headers.get("x-user"); + return name === null ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); + +// 1. Without a principal: refused (403), the plugin never sees it. +const refused = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`); +const [, response] = await once(refused, "unexpected-response"); +console.log(`1. anonymous: refused ${response.statusCode}`); + +// 2. As ada: the plugin's handler greets, then echoes. +const socket = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`, { headers: { "x-user": "ada" } }); +const messages: string[] = []; +socket.on("message", (data) => messages.push(String(data))); +await once(socket, "open"); +socket.send("hello"); +// The greeting and the echo. +while (messages.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); +} +for (const message of messages) console.log(`2. ${message}`); + +// 3. Closing the component closes its sockets (1001, going away); then the host. +const closed = once(socket, "close"); +await http.close(context); +await server.close(context); +const [code] = await closed; +console.log(`3. closed with ${code}`); +plugins.close(); diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json new file mode 100644 index 000000000..ed606991b --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/echo", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts new file mode 100644 index 000000000..abf014eb3 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts @@ -0,0 +1,16 @@ +// The echo plugin: a WebSocket of its own protocol at `/echo/:room`, after the server checked who is connecting. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + sockets: { + "/echo/:room": (socket, request, ctx) => { + const who = JSON.stringify(ctx.principal); + socket.send(`welcome to ${request.params.room}, ${who}`); + socket.on("message", (data) => socket.send(`echo: ${String(data)}`)); + }, + }, + }), +); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/main.ts b/packages/chord/test/next/examples/46-allowed-hosts/main.ts new file mode 100644 index 000000000..67c346516 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/main.ts @@ -0,0 +1,61 @@ +// Allowed hosts (slice 10): the server answers only requests on loopback or through a host a plugin contributes. A +// plugin can allow a host at runtime, a tunnel naming its public host, and take it back; links and sockets through a +// host no longer allowed are terminated at once. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/46-allowed-hosts/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { local, method, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +/** The tunnel plugin's object, as the app's code declares it (the same contract). */ +const Tunnel = local("example.tunnel", { version: 1, params: {}, members: { open: method(str()), close: method() } }); + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +/** GET /hello through `host` (as a tunnel would forward it), printed. */ +async function hello(host: string): Promise { + const answer = await new Promise((resolve) => { + const req = request({ host: "127.0.0.1", port, path: "/hello", headers: { host } }, (res) => { + let body = ""; + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => resolve(`${res.statusCode} ${body}`)); + }); + req.end(); + }); + console.log(` GET /hello via ${host} -> ${answer}`); +} + +const tunnel = await server.get(Tunnel, {}, context); +console.log("1. no tunnel yet"); +await hello(`localhost:${port}`); +await hello("notes.example.net"); +console.log("2. the tunnel opens"); +await tunnel.open("notes.example.net", context); +await hello("notes.example.net"); +console.log("3. the tunnel closes"); +await tunnel.close(context); +await hello("notes.example.net"); + +tunnel.release(); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json new file mode 100644 index 000000000..7681eec3f --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/tunnel", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts new file mode 100644 index 000000000..2e1239c10 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts @@ -0,0 +1,31 @@ +// The tunnel plugin: when its tunnel opens, it allows the tunnel's public host (requests through it are https); when +// it closes, it takes the host back. Its contribution is a writable, so the router follows each change. + +import { defineFacet, local, method, str } from "@earendil-works/chord/next"; +import { Http, type HttpContribution } from "@earendil-works/chord/next/http"; + +export const Tunnel = local("example.tunnel", { + version: 1, + params: {}, + members: { open: method(str()), close: method() }, +}); + +export default defineFacet((host, self) => { + self.contribute(Http, { + routes: { "GET /hello": async (request) => ({ kind: "json", value: { origin: request.origin } }) }, + }); + host.serve(Tunnel, (_params, tunnel) => { + const allowed = tunnel.writable({ hosts: [] }); + tunnel.contribute(Http, allowed); + return { + open: async (name) => { + allowed.set({ hosts: [name] }); + return null; + }, + close: async () => { + allowed.set({ hosts: [] }); + return null; + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/47-http-own-server/main.ts b/packages/chord/test/next/examples/47-http-own-server/main.ts new file mode 100644 index 000000000..6595b2517 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/main.ts @@ -0,0 +1,70 @@ +// Your own server (slice 10): the HTTP component is a Web-standard `fetch` handler, so an app with a server of its own +// mounts it instead of Chord's listener. Here a plain Node server converts every request with `nodeRequest` (the +// listener's checks: a malformed target refused, a second `Host` kept for admission to refuse; and the connection: +// loopback requests count as local only with it), passes it to `http.fetch`, and every upgrade to `http.upgrade`. +// With Hono the same is one line: `app.all("/*", (c) => http.fetch(c.req.raw, connection))`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/47-http-own-server/main.ts +import { createServer, request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +// 1. The component, without a listener: no `listen` setting. +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); + +// 2. The app's own server: a Web `Request` in, a Web `Response` out (each header line kept: several `Set-Cookie`). A +// real app adds its own routes around it. +const own = createServer(async (message, res) => { + const converted = chordHttp.nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); +}); +own.on("upgrade", http.upgrade); +await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); +const address = own.address(); +const port = typeof address === "object" && address !== null ? address.port : 0; + +// 3. Requests through the app's server get the same routes, checks and headers. +for (const path of ["/notes", "/nowhere"]) { + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(`GET ${path} -> ${response.status} ${await response.text()}`); + console.log(` x-content-type-options: ${response.headers.get("x-content-type-options")}`); +} + +// 4. What a Web `Request` cannot carry is refused at the conversion, as Chord's listener refuses it; the app's server +// lives on (`TRACE`: 501; a `Host` with userinfo: 403). +for (const [method, host] of [ + ["TRACE", `127.0.0.1:${port}`], + ["GET", `attacker@127.0.0.1:${port}`], +] as const) { + const status = await new Promise((resolve) => { + request({ host: "127.0.0.1", port, method, path: "/notes", headers: { host } }, (response) => { + response.resume(); + resolve(response.statusCode); + }).end(); + }); + console.log(`${method} /notes (Host: ${host}) -> ${status}`); +} + +// 5. The app closes its server and the component (it refuses, then waits for running requests), then the host. +await new Promise((resolve) => own.close(resolve)); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts new file mode 100644 index 000000000..2b7a0abd9 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts @@ -0,0 +1,12 @@ +// The notes plugin's server facet: one route, the same whichever server runs it. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /notes": async (request) => ({ kind: "json", value: { notes: ["buy milk"], via: request.origin } }), + }, + }), +); diff --git a/packages/chord/test/next/examples/48-http-baseline/main.ts b/packages/chord/test/next/examples/48-http-baseline/main.ts new file mode 100644 index 000000000..764fcce21 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/main.ts @@ -0,0 +1,84 @@ +// The security baseline (slice 10): on by default, the app decides only who someone is. Here the app signs people in +// with a signed cookie (`cookie`, `readCookie`) and the notes plugin contributes a page, routes and a webhook. Requests +// another site's page makes in a browser (Fetch Metadata says so) are never authenticated, a foreign `Origin` is +// refused, but a link to a page still opens it signed in, and a cross-site webhook runs anonymous. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/48-http-baseline/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const secret = "a long random secret"; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); + +// 1. The app's core facet: a sign-in route setting the cookie, and an authenticator reading it back. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Http, { + routes: { + "POST /sign-in": async (request) => { + const set = chordHttp.cookie("user", String(await request.json()), { secret, request, sameSite: "Lax" }); + return { kind: "json", value: "signed in", headers: { "Set-Cookie": set } }; + }, + }, + }); + self.contribute(chordHttp.Authenticators, (request) => { + const name = chordHttp.readCookie(request, "user", secret); + return name === undefined ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); +const base = `http://127.0.0.1:${port}`; + +// 2. Ada signs in; her browser keeps the cookie. +const signIn = await fetch(`${base}/sign-in`, { method: "POST", body: '"ada"' }); +const cookie = signIn.headers.get("set-cookie")!.split(";")[0]!; +console.log(`1. signed in: ${cookie.slice(0, 12)}…`); + +/** A request from Ada's browser: her cookie, and what Fetch Metadata says about who made it (`fetch` sets its own). */ +async function browser(label: string, method: string, path: string, headers: Record = {}, body = "") { + const answer = await new Promise((resolve) => { + const sent = request({ host: "127.0.0.1", port, method, path, headers: { cookie, ...headers } }, (response) => { + let text = ""; + response.on("data", (chunk) => { + text += chunk; + }); + response.on("end", () => resolve(`${response.statusCode} ${text}`)); + }); + sent.end(body); + }); + console.log(`${label}: ${answer}`); +} + +const ours = { "sec-fetch-site": "same-origin", origin: base }; +const evil = { "sec-fetch-site": "cross-site", origin: "https://evil.example" }; +await browser("2. our page reads the notes", "GET", "/notes", ours); +await browser("3. our page adds a note", "POST", "/notes", ours, '"call mom"'); +await browser("4. evil.example posts a note", "POST", "/notes", evil, '"pwned"'); +await browser("5. evil.example reads the notes", "GET", "/notes", { "sec-fetch-site": "cross-site" }); +const link = { "sec-fetch-site": "cross-site", "sec-fetch-mode": "navigate", "sec-fetch-dest": "document" }; +await browser("6. a link on evil.example to the page", "GET", "/", link); +await browser("7. a build server's webhook", "POST", "/hooks/build", { origin: "https://ci.example" }); + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts new file mode 100644 index 000000000..0980b62f1 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts @@ -0,0 +1,33 @@ +// The notes plugin's server facet: the notes page (a link from another site may open it), the notes as JSON, adding +// a note, and a webhook another site calls (cross-site: it always runs anonymous). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +const notes = ["buy milk"]; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /": { + page: true, + handler: async (_request, ctx) => ({ kind: "html", body: `hi ${who(ctx.principal)}` }), + }, + "GET /notes": async (_request, ctx) => ({ kind: "json", value: { notes, as: who(ctx.principal) } }), + "POST /notes": async (request, ctx) => { + if (ctx.principal === undefined) return { kind: "error", status: 401 }; + notes.push(String(await request.json())); + return { kind: "json", value: notes }; + }, + "POST /hooks/build": { + crossSite: true, + handler: async (_request, ctx) => ({ kind: "json", value: who(ctx.principal) }), + }, + }, + }), +); + +/** Who a principal is, for the answers. */ +function who(principal: unknown): string { + return principal === undefined ? "anonymous" : JSON.stringify(principal); +} diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v2.ts b/packages/chord/test/next/examples/49-plugin-api/app/v2.ts new file mode 100644 index 000000000..9b1da645e --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v2.ts @@ -0,0 +1,4 @@ +// Acme's API, version 2 (`@acme/app/v2`): still provided, as a thin layer over v3. Version 2 called the greetings +// registry `Hellos`; the layer keeps that name, so plugins written against v2 build and run unchanged. + +export { constant, defineFacet, Greetings as Hellos, local, str } from "./v3.ts"; diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3.ts new file mode 100644 index 000000000..149c4274a --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3.ts @@ -0,0 +1,23 @@ +// Acme's API, version 3, shared by every host kind (`@acme/app/v3`): what a plugin imports from its host. It re-exports +// the part of Chord Acme supports (plugins never import Chord directly) next to Acme's own part: the `Greetings` +// registry every kind has. Examples 49 to 52 build against it; a real app keeps it in `src/api/v3/`. + +import { defineRegistry } from "../../../../../src/next/index.ts"; + +export { + constant, + defineFacet, + defineRegistry, + defineState, + int, + local, + method, + obj, + str, +} from "../../../../../src/next/index.ts"; + +/** Lines of greeting, one per contributing plugin; the app's start-data facet serves it (composing them in order). */ +export const Greetings = defineRegistry("acme.greetings", { version: 1 }); + +/** The API's own version, for plugins that print it. */ +export const API_VERSION = "v3"; diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts new file mode 100644 index 000000000..445c8647c --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for server facets only (`@acme/app/v3/server`): what exists on the server and nowhere else. + +/** The server's name. */ +export function serverName(): string { + return "the acme server"; +} diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts new file mode 100644 index 000000000..23e38cb0a --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for web facets only (`@acme/app/v3/web`): the panels a page shows. + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** A panel's title, one per contributing plugin. */ +export const Panels = defineRegistry<{ readonly title: string }, readonly string[]>("acme.panels", { version: 1 }); diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts new file mode 100644 index 000000000..aa51aae44 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for worker facets only (`@acme/app/v3/worker`): a registry of the workers' indexers (example 52). + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** Word indexers a worker runs over a project's notes: each contributes its name. */ +export const Indexers = defineRegistry<{ readonly name: string }, readonly string[]>("acme.indexers", { version: 1 }); diff --git a/packages/chord/test/next/examples/49-plugin-api/main.ts b/packages/chord/test/next/examples/49-plugin-api/main.ts new file mode 100644 index 000000000..145cf1f93 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/main.ts @@ -0,0 +1,80 @@ +// The app's versioned API (slice 12 §1, decision "Plugins import only the app's versioned API"). Plugins import their +// host only through the app's API modules, a version in the path: `@acme/app/v3` (shared by every kind) and its +// per-kind modules (`@acme/app/v3/server`, `/v3/web`); the app lists the versions it still provides (`api.versions`) +// and each kind's modules (`kinds..hostPackages`). Here (`plugins/`, the API in `app/`): a v3 plugin and a v2 +// plugin both load (v2 is a thin layer over v3); refused at build, each with its error: a v1 plugin (no longer +// provided), a facet mixing v2 and v3, a direct import of Chord, a server facet importing the web's module. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/49-plugin-api/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import { createPlugins, evaluateFacet } from "../../../../src/next-node/plugins/index.ts"; +import * as v2 from "./app/v2.ts"; +import * as v3server from "./app/v3/server.ts"; +import * as v3web from "./app/v3/web.ts"; +import * as v3 from "./app/v3.ts"; + +const context = BACKGROUND_CONTEXT; +const root = fileURLToPath(new URL("./plugins/", import.meta.url)); + +// 1. The component with the app's API: its name and versions (newest first), and per kind the modules its facets may +// import. Chord is no host package of plugins: the API re-exports what Acme supports of it. +const plugins = createPlugins( + { + roots: [{ path: root, required: false }], + api: { name: "@acme/app", versions: ["v3", "v2"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server", "@acme/app/v2"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web", "@acme/app/v2"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. The server's modules (this process's instances) and its own start-data facet, base code: it serves the +// `Greetings` registry, composing the lines in plugin-name order. +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server, "@acme/app/v2": v2 }; +const core = defineFacet((host) => host.serve(v3.Greetings, { compose: (table) => table.map(({ value }) => value) })); +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load( + host, + { kind: "server", core: [{ name: "@acme/core", facet: core }], modules }, + context, +); + +// 3. Who loaded, and who was refused (each build error names the file, line and rule). +console.log("1. greetings:"); +for (const line of (await host.registry(v3.Greetings, context)).value) console.log(` ${line}`); +console.log("2. refused at build:"); +for (const { plugin, error } of loaded.failed) { + const [, ...errors] = (error as Error).message.split("\n"); + console.log(` ${plugin}: ${errors.join("; ")}`); +} + +// 4. The web kind provides `/v3/web`: the panel plugin's web facet builds, and a page (here a host in this process, +// given the web's modules) evaluates and loads it. +const built = await plugins.build(["@acme/panel"], context); +console.log("3. the panel plugin:", built.facets.map((facet) => `${facet.facet} ${facet.kind}`).join(", ")); +const web = built.facets.find((facet) => facet.facet === "web" && facet.kind === "script"); +if (web?.kind === "script") { + const webModules = { "@acme/app/v3": v3, "@acme/app/v3/web": v3web, "@acme/app/v2": v2 }; + const panels = defineFacet((h) => + h.serve(v3web.Panels, { compose: (table) => table.map(({ value }) => value.title) }), + ); + const page = createHost({ onError: () => {} }); + const facet = evaluateFacet(web, { hostPackages: webModules, plugin: built.plugins["@acme/panel"]! }); + await page.load( + [ + { name: "@acme/page", facet: panels }, + { name: "@acme/panel", facet }, + ], + context, + ); + console.log(" panels on the page:", (await page.registry(v3web.Panels, context)).value.join(", ")); + await page.close(context); +} + +await host.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts new file mode 100644 index 000000000..4b2598837 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts @@ -0,0 +1,9 @@ +// Types of the API version this plugin was written against: the app no longer provides it (no module to type-check +// against, the build refuses the import), so the plugin's last author left these. + +declare module "@acme/app/v1" { + export function defineFacet( + setup: (host: unknown, self: { contribute(registry: unknown, value: string): void }) => void, + ): unknown; + export const Greetings: unknown; +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json new file mode 100644 index 000000000..fdfe3f61d --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/ancient", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts new file mode 100644 index 000000000..b1b42e926 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts @@ -0,0 +1,5 @@ +// A plugin written against v1, which the app no longer provides: refused at build, naming the versions it has. + +import { defineFacet, Greetings } from "@acme/app/v1"; + +export default defineFacet((_host, self) => self.contribute(Greetings, "hello from v1")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json new file mode 100644 index 000000000..00bf91b51 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/direct", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts new file mode 100644 index 000000000..13a290169 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts @@ -0,0 +1,5 @@ +// A plugin importing Chord directly: refused, pointing to the app's API (which re-exports what plugins may use). + +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => {}); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json new file mode 100644 index 000000000..46fb32844 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/hello", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts new file mode 100644 index 000000000..2fee479bf --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts @@ -0,0 +1,6 @@ +// A plugin written against v3: the shared module and the server's own module, one version for the whole facet. + +import { defineFacet, Greetings } from "@acme/app/v3"; +import { serverName } from "@acme/app/v3/server"; + +export default defineFacet((_host, self) => self.contribute(Greetings, `hello from a v3 plugin on ${serverName()}`)); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json new file mode 100644 index 000000000..e35b0f9c6 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/legacy", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts new file mode 100644 index 000000000..360ac3775 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts @@ -0,0 +1,5 @@ +// A plugin written against v2 a year ago: the app still lists v2, so it builds and runs, through v2's names. + +import { defineFacet, Hellos } from "@acme/app/v2"; + +export default defineFacet((_host, self) => self.contribute(Hellos, "hello from a v2 plugin")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json new file mode 100644 index 000000000..922f1ac5c --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/mixed", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts new file mode 100644 index 000000000..7d84e0f69 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts @@ -0,0 +1,6 @@ +// A facet importing two versions of the API: refused (v2's and v3's registries may be different ones). + +import { Hellos } from "@acme/app/v2"; +import { defineFacet } from "@acme/app/v3"; + +export default defineFacet((_host, self) => self.contribute(Hellos, "hello from two versions")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json new file mode 100644 index 000000000..c5fb2d9ab --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/panel", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts new file mode 100644 index 000000000..d26507ccf --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts @@ -0,0 +1,7 @@ +// A server facet importing the web's module of the API: refused, naming the kind (a page's registry has no place on +// the server). + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "a panel" })); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts new file mode 100644 index 000000000..50abf6434 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts @@ -0,0 +1,6 @@ +// The panel plugin's web facet: the web's module of the API is for exactly this kind. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "word count" })); diff --git a/packages/chord/test/next/examples/50-plugin-imports/main.ts b/packages/chord/test/next/examples/50-plugin-imports/main.ts new file mode 100644 index 000000000..36b1b6af7 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/main.ts @@ -0,0 +1,96 @@ +// Imports between plugins (slice 12 §2, §3; decision "Contracts come from where the plugin's code came from"). A plugin +// carries the contract it was written against: plugins of one root (one repository, one commit) import each other's +// contract files by relative path, copied into the importer's script; a plugin of another repository takes them as a +// dependency pinned by an npm version or a git ref, inlined at the installed commit. Per-kind contract files +// (`contract.web.ts`) are for their kind only. Here: the app's root (`plugins/`, required) with notes, search (its server +// facet using notes' shared contract, its web facet notes' web file) and peek (a server facet reaching the web file +// through a helper: refused); the user's root (`user/`) with Bob's stats plugin, whose `@bob/words` dependency is pinned +// to the first commit of a local git repository (`words/` committed twice; no network), and reach (a relative path +// out of its root: refused). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/50-plugin-imports/main.ts +import { execFileSync } from "node:child_process"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { int, local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins, PluginError } from "../../../../src/next-node/plugins/index.ts"; +import * as v3server from "../49-plugin-api/app/v3/server.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example installs into a plugin: it works on a copy of its directories, removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-50-")); +for (const dir of ["plugins", "user", "words"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} + +// 1. `@bob/words` in its own git repository: version 1 committed, then version 2. Bob pinned the first commit. +const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.email=bob@example.com", "-c", "user.name=bob", ...args], { cwd, encoding: "utf8" }); +const words = join(copy, "words"); +git(words, "init", "--quiet"); +git(words, "add", "--all"); +git(words, "commit", "--quiet", "--message", "words 1"); +const pinned = git(words, "rev-parse", "HEAD").trim(); +const source = await readFile(join(words, "words.ts"), "utf8"); +await writeFile(join(words, "words.ts"), source.replace('VERSION = "1"', 'VERSION = "2"')); +git(words, "commit", "--quiet", "--all", "--message", "words 2"); +git(copy, "clone", "--quiet", "--bare", words, join(copy, "words.git")); +const stats = join(copy, "user", "stats", "package.json"); +const manifest = JSON.parse(await readFile(stats, "utf8")) as { dependencies: Record }; +manifest.dependencies["@bob/words"] = `git+file://${join(copy, "words.git")}#${pinned}`; +await writeFile(stats, JSON.stringify(manifest, null, "\t")); +console.log("1. @bob/stats depends on", manifest.dependencies["@bob/words"].replace(copy, "…")); + +// 2. The component: the app's root required, the user's not; the app's API (example 49's) for both kinds. +const plugins = createPlugins( + { + roots: [{ path: join(copy, "plugins"), required: true }, join(copy, "user")], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 3. Bob's dependencies, installed by npm into his plugin's node_modules (a git dependency at a ref is npm's own). +await plugins.installDependencies("@bob/stats", context); + +// 4. The server loads both roots. The peek plugin of the app's root fails: so the required root's load rejects (a +// `PluginError`, after the load settled: the others are loaded), and the app decides (here: it reports and goes on). +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server }; +const host = createHost({ onError: () => {} }); +try { + await plugins.load(host, { kind: "server", core: [], modules }, context); +} catch (error) { + if (!(error instanceof PluginError)) throw error; + console.log("2. the load rejects:", error.name, "for", error.required.map(({ plugin }) => plugin).join(", ")); + for (const { plugin, error: cause } of error.failed) { + console.log(` ${plugin}: ${(cause as Error).message.split("\n").slice(1).join("; ")}`); + } +} +const Search = local("acme.search", { version: 1, params: {}, members: { hits: int() } }); +const Stats = local("bob.stats", { version: 1, params: {}, members: { text: str() } }); +console.log("3. search hits:", (await host.get(Search, {}, context)).hits.value); +console.log(" stats:", (await host.get(Stats, {}, context)).text.value); + +// 5. The search plugin's builds: what each copied from the notes plugin (its server facet the shared file, its web +// facet the web file too). +const built = await plugins.build(["@acme/search"], context); +for (const facet of built.facets) { + if (facet.kind !== "script") continue; + const copied = facet.copied.map((file) => file.slice(copy.length + 1)); + console.log(`4. @acme/search ${facet.facet} copied: ${copied.join(", ")}`); +} + +await host.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts new file mode 100644 index 000000000..d7a17cf0d --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts @@ -0,0 +1,7 @@ +// The notes plugin's shared contract: data only, importable from every kind. Other plugins of this root import it by +// relative path; the build copies it into their scripts. + +import { local, str } from "@acme/app/v3"; + +/** The notes, as one text. */ +export const Notes = local("acme.notes", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts new file mode 100644 index 000000000..f5f6806b9 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts @@ -0,0 +1,5 @@ +// The notes plugin's web contract file (a file of the web kind: `.web.` before its extension): only web facets may +// import it, directly or through other files. + +/** The panel the notes plugin offers pages that list notes. */ +export const notesPanel = { title: "notes" } as const; diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json new file mode 100644 index 000000000..1c76a10b8 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts new file mode 100644 index 000000000..4f4eafc17 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts @@ -0,0 +1,6 @@ +// The notes plugin's server facet: serves `Notes`. + +import { constant, defineFacet } from "@acme/app/v3"; +import { Notes } from "./contract.ts"; + +export default defineFacet((host) => host.serve(Notes, () => ({ text: constant("buy milk, call bob, ship it") }))); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json new file mode 100644 index 000000000..14832d7dd --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/peek", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts new file mode 100644 index 000000000..dae94d9d7 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts @@ -0,0 +1,6 @@ +// The peek plugin's server facet reaches a web file through its helper: refused at build, naming the chain. + +import { defineFacet } from "@acme/app/v3"; +import { notesPanel } from "./view.ts"; + +export default defineFacet(() => console.log(notesPanel)); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts new file mode 100644 index 000000000..d984e1271 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts @@ -0,0 +1,3 @@ +// A helper of the peek plugin that re-exports the notes plugin's web contract file. + +export { notesPanel } from "../notes/contract.web.ts"; diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json new file mode 100644 index 000000000..6f7be6d36 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/search", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts new file mode 100644 index 000000000..5b28876fd --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts @@ -0,0 +1,14 @@ +// The search plugin's server facet: uses the notes plugin's contract, imported by relative path (same root, same +// commit: a copy of that file in this script). + +import { defineFacet, int, local } from "@acme/app/v3"; +import { Notes } from "../notes/contract.ts"; + +const Search = local("acme.search", { version: 1, params: {}, members: { hits: int() } }); + +export default defineFacet((host) => + host.serve(Search, (_params, self) => { + const notes = self.get(Notes, {}); + return { hits: self.derive([notes.text], ([text]) => text.split(",").length) }; + }), +); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts new file mode 100644 index 000000000..7f000cfdb --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts @@ -0,0 +1,7 @@ +// The search plugin's web facet: imports the notes plugin's web contract file, which web facets may. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; +import { notesPanel } from "../notes/contract.web.ts"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: `search ${notesPanel.title}` })); diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json b/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json new file mode 100644 index 000000000..d64b400a3 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json @@ -0,0 +1,5 @@ +{ + "name": "@bob/reach", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts b/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts new file mode 100644 index 000000000..3606563ba --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts @@ -0,0 +1,7 @@ +// A user's plugin (another root) importing the app's notes contract by its path: refused, a relative path leaving its +// root; a plugin of another repository takes contracts as a dependency (or from the app's API). + +import { defineFacet } from "@acme/app/v3"; +import { Notes } from "../../plugins/notes/contract.ts"; + +export default defineFacet(() => console.log(Notes.name)); diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts b/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts new file mode 100644 index 000000000..4aca7b708 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts @@ -0,0 +1,6 @@ +// Types of the stats plugin's dependency before it is installed (its own types once installed). + +declare module "@bob/words" { + export const VERSION: string; + export function words(text: string): number; +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json b/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json new file mode 100644 index 000000000..46aee125f --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json @@ -0,0 +1,8 @@ +{ + "name": "@bob/stats", + "private": true, + "type": "module", + "dependencies": { + "@bob/words": "git+file:///path/to/words.git#" + } +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts b/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts new file mode 100644 index 000000000..6d6982abe --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts @@ -0,0 +1,11 @@ +// Bob's stats plugin (the user's root): counts words with `@bob/words`, a dependency pinned to a git commit. The build +// inlines the package at the installed commit, whatever the repository holds since. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; +import { VERSION, words } from "@bob/words"; + +const Stats = local("bob.stats", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Stats, () => ({ text: constant(`${words("buy milk, call bob")} words (@bob/words ${VERSION})`) })), +); diff --git a/packages/chord/test/next/examples/50-plugin-imports/words/package.json b/packages/chord/test/next/examples/50-plugin-imports/words/package.json new file mode 100644 index 000000000..5fee19049 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/words/package.json @@ -0,0 +1,6 @@ +{ + "name": "@bob/words", + "version": "1.0.0", + "type": "module", + "exports": { ".": "./words.ts" } +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/words/words.ts b/packages/chord/test/next/examples/50-plugin-imports/words/words.ts new file mode 100644 index 000000000..de1226fed --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/words/words.ts @@ -0,0 +1,9 @@ +// `@bob/words`: a package in its own git repository (the example commits it twice: this file, then a version 2). + +/** The version of this commit. */ +export const VERSION = "1"; + +/** The words in `text`. */ +export function words(text: string): number { + return text.split(/\s+/).filter(Boolean).length; +} diff --git a/packages/chord/test/next/examples/51-single-file-plugin/main.ts b/packages/chord/test/next/examples/51-single-file-plugin/main.ts new file mode 100644 index 000000000..3003288e9 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/main.ts @@ -0,0 +1,88 @@ +// Single-file plugins (slice 12 §4, decision "Single-file plugins"). A file at a root's top named +// `..ts(x)` is a facet of the plugin ``; files with one first name are one plugin. It has no +// `package.json`, so no dependencies: it imports the app's API only, defines its contracts inline, serves objects and +// contributes to registries. When it needs more, it becomes a directory with the same name, which a reload treats as +// the same plugin. Here (`plugins/`): `hello` (a server and a web file), `clock`, and `bad`, which imports another file +// and is refused; then `hello` becomes a directory with a helper file: the watcher marks the root, a build and a +// reload change the plugin `hello` in place. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/51-single-file-plugin/main.ts +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import * as v3server from "../49-plugin-api/app/v3/server.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example turns a file into a directory: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-51-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +// 1. The component, the server loaded: discovery found three single-file plugins; `bad` fails its build. +const plugins = createPlugins( + { + roots: [root], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server }; +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); +for (const plugin of plugins.discovery.plugins) { + const files = Object.values(plugin.facets).map((file) => file.slice(root.length + 1)); + console.log(`1. ${plugin.name} (${plugin.layout}): ${files.join(", ")}`); +} +for (const { plugin, error } of loaded.failed) { + console.log(` ${plugin} refused: ${(error as Error).message.split("\n").slice(1).join("; ")}`); +} +const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } }); +const Clock = local("example.clock", { version: 1, params: {}, members: { text: str() } }); +const hello = await host.get(Hello, {}, context); +console.log("2. hello:", hello.text.value, "| clock:", (await host.get(Clock, {}, context)).text.value); + +// 2. `hello` needs a second file: it becomes the directory `hello/` (a package.json named `hello`), its files removed. +await mkdir(join(root, "hello")); +await writeFile(join(root, "hello", "package.json"), JSON.stringify({ name: "hello", type: "module" })); +await writeFile(join(root, "hello", "greeting.ts"), 'export const greeting = "hello from a directory";\n'); +await writeFile( + join(root, "hello", "server.ts"), + [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'import { greeting } from "./greeting.ts";', + 'const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } });', + "export default defineFacet((host) => host.serve(Hello, () => ({ text: constant(greeting) })));", + ].join("\n"), +); +await rm(join(root, "hello.server.ts")); +await rm(join(root, "hello.web.ts")); +await until(() => plugins.state.value.set && plugins.state.value.dirty.includes("hello")); +console.log("3. marks:", JSON.stringify(plugins.state.value)); + +// 3. A build discovers again; the reload changes `hello` in place (the same plugin, its object swapped), a holder +// sees the new text. +const built = await plugins.build(plugins.state.value.dirty, context); +const report = await plugins.reload(host, built, context); +console.log("4. reload:", JSON.stringify(report.plugins), "| hello:", hello.text.value); +console.log(" hello is now:", plugins.discovery.plugins.find(({ name }) => name === "hello")?.layout); + +hello.release(); +await host.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts new file mode 100644 index 000000000..82f4fb859 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts @@ -0,0 +1,6 @@ +// A single-file plugin importing another file: refused at build, saying to make it a directory. + +import { defineFacet } from "@acme/app/v3"; +import { helper } from "./helper.ts"; + +export default defineFacet(() => console.log(helper)); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts new file mode 100644 index 000000000..49ec01059 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts @@ -0,0 +1,8 @@ +// Another single-file plugin: a line naming the server, through the server's module of the API. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; +import { serverName } from "@acme/app/v3/server"; + +const Clock = local("example.clock", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Clock, () => ({ text: constant(`ticking on ${serverName()}`) }))); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts new file mode 100644 index 000000000..79f4cffc1 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts @@ -0,0 +1,8 @@ +// A single-file plugin named `hello` (its file name: `..ts`), its server facet: it defines its contract +// inline and serves it. No directory, no package.json: it imports only the app's API. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; + +const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Hello, () => ({ text: constant("hello from one file") }))); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts new file mode 100644 index 000000000..ad2790e7f --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts @@ -0,0 +1,6 @@ +// The same plugin's web facet: a second file with the same first name. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "hello" })); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts new file mode 100644 index 000000000..13e787649 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts @@ -0,0 +1,3 @@ +// A plain file at the root's top: not a plugin (its name has no kind), and no single-file plugin may import it. + +export const helper = "a helper"; diff --git a/packages/chord/test/next/examples/52-test-plugin/main.ts b/packages/chord/test/next/examples/52-test-plugin/main.ts new file mode 100644 index 000000000..e5c5dfc9c --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/main.ts @@ -0,0 +1,68 @@ +// Testing a plugin alone (slice 12 §6, decision "Testing a plugin alone"). `testPlugin(dir, options, ctx)` builds the +// plugin as production does (the same build, the app's API checked), evaluates it with the app's real API modules and +// loads it on an in-memory host (a `:memory:` store) beside stubs: ordinary facets serving the contracts it uses with +// fake data. Registries it contributes to that nobody serves are served by the helper, which collects the entries; +// contracts its requests wait for unserved are listed. Here (`plugins/`): the stats plugin, with a stub for the notes +// plugin, its indexer read from the collected `Indexers`, and the `Tags` contract it waits for listed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/52-test-plugin/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { testPlugin } from "../../../../src/next-node/plugins/index.ts"; +import * as v3worker from "../49-plugin-api/app/v3/worker.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The contracts the test uses, declared again with the API's builders (contracts are data: the same name, version and +// schema bind). The plugins' own files import `@acme/app/v3`, a host package this process has no module path for. +const Notes = v3.local("acme.notes", { version: 1, params: { project: v3.str() }, members: { text: v3.str() } }); +const Stats = v3.local("acme.stats", { version: 1, params: { project: v3.str() }, members: { words: v3.int() } }); +const TagLine = v3.local("acme.tag-line", { version: 1, params: { project: v3.str() }, members: { text: v3.str() } }); + +/** The worker host's principal type, and a user's. */ +type Principal = { readonly kind: "worker"; readonly project: string } | { readonly kind: "user"; readonly id: string }; + +// 1. A stub of the notes plugin: an ordinary facet serving `Notes` with a fixed text for every project. +const notesStub = v3.defineFacet((host) => + host.serve(Notes, ({ project }) => ({ text: v3.constant(`${project}: buy milk, call bob, ship it`) })), +); + +// 2. The plugin on a worker host of its own, acting as the worker's principal; the app's API as production has it. +const t = await testPlugin( + fileURLToPath(new URL("./plugins/stats/", import.meta.url)), + { + kind: "worker", + kinds: ["server", "worker", "web"], + api: { name: "@acme/app", versions: ["v3"] }, + modules: { "@acme/app/v3": v3, "@acme/app/v3/worker": v3worker }, + stubs: [{ name: "@acme/notes", facet: notesStub }], + principal: { kind: "worker", project: "p1" }, + }, + context, +); + +// 3. Its objects are live: get them, read their state. +const stats = await t.host.get(Stats, { project: "p1" }, context); +console.log("1. words in p1:", stats.words.value); + +// 4. What it contributed to `Indexers`, a registry of the app no stub serves: collected by the helper. +console.log( + "2. indexers:", + t.contributions(v3worker.Indexers).map(({ plugin, value }) => `${value.name} (${plugin})`), +); + +// 5. A contract nobody serves: its requests wait, and the helper names it, so the test knows what to stub next. +const tags = t.host.hold(TagLine, { project: "p1" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("3. waiting for:", t.unserved(), "| tag line:", tags.state.value.kind); + +// 6. Code outside objects as another principal (a user), and as the host's own. +const who = (principal: Principal | undefined) => (principal === undefined ? "nobody" : JSON.stringify(principal)); +console.log("4. root as the host:", who(await t.root(async (ctx) => ctx.principal, context))); +const user = { kind: "user", id: "u1" } as const; +console.log(" root as a user:", who(await t.root(async (ctx) => ctx.principal, context, { principal: user }))); + +stats.release(); +tags.release(); +await t.close(context); diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts new file mode 100644 index 000000000..964c8de65 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts @@ -0,0 +1,6 @@ +// The notes plugin's contract (its facets are not part of this example: a test of the stats plugin stubs them). + +import { local, str } from "@acme/app/v3"; + +/** A project's notes, as one text. */ +export const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json new file mode 100644 index 000000000..1c76a10b8 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json new file mode 100644 index 000000000..173f09134 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/stats", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts new file mode 100644 index 000000000..a3c88b119 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts @@ -0,0 +1,27 @@ +// The stats plugin's worker facet: words per project over the notes plugin's `Notes`, an indexer contributed to the +// workers' `Indexers` registry, and tags over a `Tags` contract (the tags plugin's; this example's test stubs none). + +import { defineFacet, int, local, str } from "@acme/app/v3"; +import { Indexers } from "@acme/app/v3/worker"; +import { Notes } from "../notes/contract.ts"; + +/** Words per project. */ +const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** The tags plugin's contract, as this plugin uses it. */ +const Tags = local("acme.tags", { version: 1, params: { project: str() }, members: { list: str() } }); + +/** The tags of a project, as this plugin shows them. */ +const TagLine = local("acme.tag-line", { version: 1, params: { project: str() }, members: { text: str() } }); + +export default defineFacet((host, self) => { + self.contribute(Indexers, { name: "word count" }); + host.serve(Stats, ({ project }, self) => { + const notes = self.get(Notes, { project }); + return { words: self.derive([notes.text], ([text]) => text.split(/\s+/).filter(Boolean).length) }; + }); + host.serve(TagLine, ({ project }, self) => { + const tags = self.get(Tags, { project }); + return { text: self.derive([tags.list], ([list]) => `tags: ${list}`) }; + }); +}); diff --git a/packages/chord/test/next/host/activity.test.ts b/packages/chord/test/next/host/activity.test.ts new file mode 100644 index 000000000..63ac47af6 --- /dev/null +++ b/packages/chord/test/next/host/activity.test.ts @@ -0,0 +1,164 @@ +// Tests of \`host.activity\` (§3, R26): the counts per kind, one publication per microtask turn with changes, a slow +// watcher, derives over it, nothing scheduled while nobody follows it, lingering. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, local, method, remote } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { fakeTimers, flushed, framesOf, linked, loaded, Notes, Server, serveNotes, transported } from "../support.ts"; + +const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +describe("host.activity", () => { + it("counts holds arriving over links and served here, not this host's own references", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + expect(server.activity.value).toEqual({ references: 0, calls: 0, jobs: 0, lingering: 0 }); + const notes = await client.get(Notes, { session: "S1" }, context); + await server.get(Notes, { session: "S2" }, context); + expect(server.activity.value.references).toBe(1); + notes.release(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("counts a hold placed here from its arrival, bound or not: a missing provider keeps the worker alive (M§10)", async () => { + const { server, client } = await linked(() => {}); + const missing = client.hold(Notes, { session: "S1" }, context); + await transported(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(server.activity.value.references).toBe(1); + // Presence lists bound references only. + expect(server.references(Notes).value).toEqual([]); + missing.release(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("counts method calls running on this host, local and remote, until they settle", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const remoteRef = await client.get(Notes, { session: "S1" }, context); + const localRef = await server.get(Notes, { session: "S1" }, context); + const calls = [remoteRef.wait(context), localRef.wait(context)]; + await transported(); + expect(server.activity.value.calls).toBe(2); + remoteRef.release(); + localRef.release(); + await Promise.allSettled(calls); + await transported(); + expect(server.activity.value.calls).toBe(0); + }); + + it("publishes once per microtask turn with changes; a slow watcher gets the newest counts", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method() } }); + const { host } = await loaded((h) => h.serve(Counter, () => ({ bump: async () => null }))); + const counter = await host.get(Counter, {}, context); + const watch = await host.activity.watch(context); + const frames = framesOf(watch); + // Ten calls started in one turn: ten changes, one publication; they settle in a later turn: one more. + const bumps = Array.from({ length: 10 }, () => counter.bump(context)); + await Promise.all(bumps); + await flushed(); + const counts = (calls: number) => ({ references: 0, calls, jobs: 0, lingering: 0 }); + expect(frames).toEqual([ + [counts(10), [["r", counts(10)]]], + [counts(0), [["r", counts(0)]]], + ]); + }); + + it("gives a slow watcher the newest counts once it is free: its pending frames fold", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method() } }); + const { host } = await loaded((h) => h.serve(Counter, () => ({ bump: async () => null }))); + const counter = await host.get(Counter, {}, context); + const watch = await host.activity.watch(context); + const gate = Promise.withResolvers(); + const seen: unknown[] = []; + watch.start(async (call) => { + if (call.kind !== "value") return; + seen.push(call.value); + if (seen.length === 1) await gate.promise; + }); + // Three more publications while the watcher is busy with the first. + for (let i = 0; i < 2; i++) { + await counter.bump(context); + await flushed(); + } + gate.resolve(); + await flushed(); + const counts = (calls: number) => ({ references: 0, calls, jobs: 0, lingering: 0 }); + expect(seen).toEqual([counts(1), counts(0)]); + }); + + it("schedules nothing while nobody watches or derives from it; a later watch reads the counts then", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method(), wait: method() } }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, () => ({ bump: async () => null, wait: () => gate.promise })), + ); + const counter = await host.get(Counter, {}, context); + const scheduled = vi.spyOn(globalThis, "queueMicrotask"); + try { + await counter.bump(context); + const waiting = counter.wait(context); + await flushed(); + expect(scheduled).not.toHaveBeenCalled(); + const watch = await host.activity.watch(context); + expect(watch.value).toMatchObject({ calls: 1 }); + gate.resolve(null); + await waiting; + expect(scheduled).toHaveBeenCalledTimes(1); + } finally { + scheduled.mockRestore(); + } + }); + + it("feeds derives (base code holding the host): they recompute in the publishing batch", async () => { + const Busy = local("app.busy", { version: 1, params: {}, members: { busy: bool(), work: method() } }); + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Busy, (_, self) => ({ + busy: self.derive([host.activity], ([{ calls }]) => calls > 0), + work: async () => null, + })), + ); + await host.load([{ name: "@test/core", facet }], context); + const busy = await host.get(Busy, {}, context); + const values = framesOf(await busy.busy.watch(context)); + await busy.work(context); + await flushed(); + expect(values.map(([value]) => value)).toEqual([true, false]); + expect(busy.busy.value).toBe(false); + }); + + describe("lingering", () => { + fakeTimers(); + + it("counts live objects kept by their type's linger after their last reference, until they end", async () => { + const { server, client } = await linked((h) => + h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 1000 }), + ); + const kept = await client.get(Kept, {}, context); + kept.release(); + await transported(); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + // A new reference within the linger takes it back. + const again = await client.get(Kept, {}, context); + expect(server.activity.value).toMatchObject({ references: 1, lingering: 0 }); + again.release(); + await transported(); + expect(server.activity.value.lingering).toBe(1); + await vi.advanceTimersByTimeAsync(1000); + expect(server.activity.value.lingering).toBe(0); + }); + }); + + it("counts no jobs before slice 5", async () => { + const { host } = await loaded(() => {}); + expect(host.activity.value.jobs).toBe(0); + }); +}); diff --git a/packages/chord/test/next/host/adapted.test.ts b/packages/chord/test/next/host/adapted.test.ts new file mode 100644 index 000000000..c95e83b11 --- /dev/null +++ b/packages/chord/test/next/host/adapted.test.ts @@ -0,0 +1,1499 @@ +// Tests of adapted members (Q10, §22, slice 7b §7): consumers bound at an older version a provider accepts read its +// state mapped (diffs per commit), call it with mapped arguments and results or its replacement of a removed method, +// read keyed members mapped per key, and are refused a member the provider refuses. + +import { describe, expect, it } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, evolve, int, keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { AdaptedMembers } from "../../../src/next/host/adapted.ts"; +import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + changesOf, + delivered, + expectCollected, + flushed, + instancesOf, + metered, + rejection, + valuesOf, +} from "../support.ts"; + +/** v1: a list of texts, `add` taking a text and returning the count, `clear`, keyed notes of a title. */ +const NotesV1 = local("app.notes", { + version: 1, + params: {}, + members: { + items: arr(str()), + add: method(obj({ text: str() }), int()), + clear: method(), + notes: keyed(obj({ title: str() })), + }, +}); + +/** v2: items carry a tag, `add` takes one and returns an object, `clear` is gone, notes have a body. */ +const NotesV2 = evolve(NotesV1, (e) => + e + .state("items", arr(obj({ text: str(), tag: str() }))) + .method("add", obj({ text: str(), tag: str() }), obj({ count: int() })) + .removeMethod("clear") + .keyed("notes", obj({ title: str(), body: str() })), +); + +const Items = defineState("items", { + schema: obj({ items: arr(obj({ text: str(), tag: str() })) }), + version: 1, + initial: () => ({ items: [] }), +}); + +const Notes = defineState("notes", { + schema: obj({ title: str(), body: str() }), + version: 1, + initial: () => ({ title: "", body: "" }), + keyed: true, +}); + +/** Serves `NotesV2`, accepting v1 consumers (`clear` replaced, or refused with `refuse`). */ +function serveV2(h: FacetHost, refuse = false): void { + const clears: string[] = []; + h.serve( + NotesV2, + (_, self) => { + const items = self.state(Items); + const notes = self.state(Notes); + return { + items: self.derive([items], ([value]) => value.items), + notes, + add: async ({ text, tag }: { text: string; tag: string }, ctx: Context) => { + await items.change((draft) => void draft.items.push({ text, tag }), ctx); + await notes.change( + text, + (draft) => { + draft.title = text; + draft.body = tag; + }, + ctx, + ); + return { count: items.value.items.length }; + }, + }; + }, + { + accepts: { + 1: { + state: { items: (current) => current.map(({ text }) => text) }, + methods: { + add: { args: (old) => ({ text: old.text, tag: "v1" }), result: (current) => current.count }, + clear: refuse + ? "refuse" + : { + call: async () => { + clears.push("clear"); + return null; + }, + }, + }, + keyed: { notes: refuse ? "refuse" : (current) => ({ title: current.title }) }, + }, + }, + }, + ); +} + +describe("an older consumer of a provider's accepted version", () => { + it("reads state mapped, with diffs per commit; its calls are mapped forwards and their results back", async () => { + const { telemetry, counts } = await metered(); + const host = createHost({ telemetry, onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const old = await host.get(NotesV1, {}, context); + const items = changesOf(await old.items.watch(context)); + expect(await old.add({ text: "a" }, context)).toBe(1); + expect(await old.add({ text: "b" }, context)).toBe(2); + await delivered(); + expect(items).toEqual([ + { kind: "value", value: ["a"], ops: [["p", [], 0, 0, ["a"]]], retired: false }, + { kind: "value", value: ["a", "b"], ops: [["p", [], 1, 0, ["b"]]], retired: false }, + ]); + // Once for the watch's first value, then once per commit: never per consumer or per read. + expect(counts()["chord.contract.adapts"]).toBe(3); + expect(old.items.value).toEqual(["a", "b"]); + expect(counts()["chord.contract.adapts"]).toBe(3); + await host.close(context); + }); + + it("calls the provider's replacement of a removed method; a refused one fails contract_mismatch", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const old = await host.get(NotesV1, {}, context); + expect(await old.clear(context)).toBeNull(); + await host.close(context); + }); + + it("is refused a binding whose member the provider refuses: it waits contract_mismatch", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h, true)) }], context); + const old = host.hold(NotesV1, {}, context); + await flushed(); + expect(old.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + expect(await rejection(old.clear(context))).toMatchObject({ code: "contract_mismatch" }); + await host.close(context); + }); + + it("reads keyed members mapped per key: get, watch and a factory's at(key)", async () => { + const View = local("app.view", { version: 1, params: { key: str() }, members: { title: str() } }); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: defineFacet((h) => serveV2(h)) }, + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, ({ key }, self) => { + const note = self.get(NotesV1, {}).notes.at(key); + return { title: self.derive([note], ([value]) => value.title) }; + }), + ), + }, + ], + context, + ); + const old = await host.get(NotesV1, {}, context); + await old.add({ text: "a" }, context); + expect(await old.notes.get("a", context)).toEqual({ title: "a" }); + const watch = (await old.notes.watch("a", context))!; + expect(watch.value).toEqual({ title: "a" }); + const view = await host.get(View, { key: "a" }, context); + expect(view.title.value).toBe("a"); + await host.close(context); + }); +}); + +describe("reload: adapted members follow the provider's topology (fix round 1)", () => { + it("rebuilds a changed-member adapter as a whole-state adapter when a later version removes it", async () => { + const V1 = local("astra.adapt", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() }))); + const V3 = evolve(V2, (e) => e.removeState("n").addState("m", int())); + const host = createHost({ onError: () => {} }); + const first = defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value: 5 }) }), { accepts: { 1: { state: { n: (n) => n.value } } } }), + ); + const second = defineFacet((h) => + h.serve(V3, () => ({ m: constant(9) }), { accepts: { 1: { state: { n: (s) => s.m } } } }), + ); + await host.load([{ name: "@astra/adapt", facet: first }], context); + const old = await host.get(V1, {}, context); + expect(old.n.value).toBe(5); + await host.reload([{ name: "@astra/adapt", facet: second }], context); + await flushed(); + let actual: unknown; + try { + actual = old.n.value; + } catch (error) { + actual = error; + } + old.release(); + await host.close(context); + expect(actual).toBe(9); + }); + + it("maps existing local watches when the consumer's current version becomes an accepted older one", async () => { + const V1 = local("sol.state", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [{ name: "@sol/state", facet: defineFacet((h) => h.serve(V1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 7 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(valuesOf(calls)).toEqual([7]); + } finally { + await host.close(context); + } + }); + + it("rebuilds an existing adapted member's dependencies when a later provider removes that member", async () => { + const V1 = local("sol.removed-member", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.removeState("x").addState("y", int())); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 2 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + const old = await host.get(V1, {}, context); + expect(old.x.value).toBe(2); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V3, () => ({ y: constant(7) }), { + accepts: { + 1: { state: { x: (state) => state.y } }, + 2: { state: { x: (state) => ({ n: state.y }) } }, + }, + }), + ), + }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + expect(old.state.value.kind).toBe("live"); + expect(old.x.value).toBe(7); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: adapted members publish one coherent frame (fix round 1)", () => { + it("does not publish the new mapping applied to the old attempt's state", async () => { + const V1 = local("astra.atomic", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", int())); + const facet = (n: number, factor: number) => + defineFacet((h) => + h.serve(V2, () => ({ n: constant(n) }), { accepts: { 1: { state: { n: (v) => v * factor } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/atomic", facet: facet(2, 2) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.n.watch(context); + const changes = changesOf(watch); + await delivered(); + expect(old.n.value).toBe(4); + await host.reload([{ name: "@astra/atomic", facet: facet(3, 3) }], context); + await flushed(); + await delivered(); + const values = changes.map((call) => (call.kind === "value" ? call.value : "failed")); + await watch.stop(); + old.release(); + await host.close(context); + expect(values).toEqual([9]); + }); + + it("never computes a new accepted-version mapping over the old provider shape", async () => { + const V1 = local("sol.mapping-publication", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.state("x", obj({ wrapped: obj({ n: int() }) }))); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 2 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V3, () => ({ x: constant({ wrapped: { n: 7 } }) }), { + accepts: { + 1: { state: { x: (value) => value.wrapped.n } }, + 2: { state: { x: (value) => value.wrapped } }, + }, + }), + ), + }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + await delivered(); + expect(valuesOf(calls)).toEqual([7]); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: an older consumer across a removal (fix round 1)", () => { + it("keeps an older watch open and heals it after the type is served again", async () => { + const V1 = local("sol.older-removal", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const facet = (n: number) => + defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load([{ name: "@sol/older", facet: facet(1) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.x.watch(context); + const calls = changesOf(watch); + await host.reload([], context); + await host.reload([{ name: "@sol/older", facet: facet(7) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + // The removal's failure once, then the re-served value, whole. + expect(valuesOf(calls)).toEqual([{ failed: "sol.older-removal is not served here" }, 7]); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: keyed members of an older version follow the provider (fix round 1)", () => { + it("updates an open old-version watch when only its mapping changes", async () => { + const V1 = local("astra.keyed", { version: 1, params: {}, members: { rows: keyed(int()), set: method(int()) } }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + keyed: true, + }); + const facet = (factor: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const rows = self.state(Rows); + return { + rows, + set: async (n, ctx) => { + await rows.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/keyed", facet: facet(2) }], context); + const old = await host.get(V1, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + expect(watch.value).toBe(6); + watch.start(async () => {}); + await host.reload([{ name: "@astra/keyed", facet: facet(4) }], context); + await flushed(); + await old.set(5, context); + await delivered(); + const actual = watch.value; + const fresh = await old.rows.get("a", context); + await watch.stop(); + old.release(); + await host.close(context); + expect({ actual, fresh }).toEqual({ actual: 20, fresh: 20 }); + }); + + it("updates mappings of an existing older watch and factory pin", async () => { + const V1 = local("sol.keyed", { + version: 1, + params: {}, + members: { entries: keyed(obj({ old: int() })), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("entries", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.keyed("entries", obj({ n: int() }))); + const Doc = defineState("entries", { + version: 1, + schema: obj({ n: int() }), + keyed: true, + initial: () => ({ n: 0 }), + }); + const View = local("sol.keyed.view", { version: 1, params: {}, members: { old: int() } }); + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(V1, {}).entries.at("k"); + return { old: self.derive([pin], ([value]) => value.old) }; + }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/keyed", + facet: defineFacet((h) => + h.serve( + V2, + (_, self) => { + const entries = self.state(Doc); + return { + entries, + set: async (n: number, ctx: Context) => { + await entries.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { accepts: { 1: { keyed: { entries: (value) => ({ old: value.n }) } } } }, + ), + ), + }, + { name: "@sol/view", facet: view }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(1, context); + const watch = (await old.entries.watch("k", context))!; + const calls = changesOf(watch); + await host.reload( + [ + { + name: "@sol/keyed", + facet: defineFacet((h) => + h.serve( + V3, + (_, self) => { + const entries = self.state(Doc); + return { + entries, + set: async (n: number, ctx: Context) => { + await entries.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { + accepts: { + 1: { keyed: { entries: (value) => ({ old: value.n * 10 }) } }, + 2: { keyed: { entries: (value) => value } }, + }, + }, + ), + ), + }, + { name: "@sol/view", facet: view }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + await old.set(2, context); + await delivered(); + expect(valuesOf(calls).at(-1)).toEqual({ old: 20 }); + expect(reader.old.value).toBe(20); + } finally { + await host.close(context); + } + }); +}); + +describe("adapted members' lifetime (fix round 1)", () => { + const adapted = instancesOf(AdaptedMembers, "member"); + + it("drops an older version's adapted members when its last consumer leaves, while the object lives on", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const current = await host.get(NotesV2, {}, context); + const from = adapted.length; + const old = await host.get(NotesV1, {}, context); + expect(old.items.value).toEqual([]); + expect(adapted.length).toBeGreaterThan(from); + // The older version's: those of the version shown stay while the object lives (the mappings read them). + const older = adapted.slice(from).filter((ref) => ref.deref()?.version === 1); + expect(older.length).toBeGreaterThan(0); + old.release(); + await flushed(); + await expectCollected(...older); + expect(current.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("reload: keyed members unchanged at a consumer's version follow a later mapping (fix round 1)", () => { + const RowsV1 = local("app.rows", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), set: method(int()) }, + }); + /** v2 adds a member: `rows` is unchanged for v1 consumers. */ + const RowsV2 = evolve(RowsV1, (e) => e.addState("extra", int())); + /** v3 changes `rows` (same shape): v1 and v2 consumers read it mapped. */ + const RowsV3 = evolve(RowsV2, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const View = local("app.rows.view", { version: 1, params: {}, members: { n: int() } }); + const ten = (value: { readonly n: number }) => ({ n: value.n * 10 }); + + /** Serves `contract` with its keyed rows, `set` writing row "k", and `accepts`. */ + function rows(contract: typeof RowsV1 | typeof RowsV2 | typeof RowsV3, accepts: object) { + return defineFacet((h) => + h.serve( + contract as typeof RowsV3, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return (contract === RowsV1 ? { rows: docs, set } : { rows: docs, set, extra: constant(0) }) as never; + }, + { accepts } as never, + ), + ); + } + + /** A v1 consumer's factory pin of row "k". */ + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(RowsV1, {}).rows.at("k"); + return { n: self.derive([pin], ([value]) => value.n) }; + }), + ); + + /** Loads `first`, watches and pins row "k" as a v1 consumer, reloads to `second`, sets 2 and reads all three. */ + async function across(first: ReturnType, second: ReturnType) { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@app/rows", facet: first }, + { name: "@app/view", facet: view }, + ], + context, + ); + const old = await host.get(RowsV1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(1, context); + const watch = (await old.rows.watch("k", context))!; + watch.start(async () => {}); + await host.reload( + [ + { name: "@app/rows", facet: second }, + { name: "@app/view", facet: view }, + ], + context, + ); + await flushed(); + await old.set(2, context); + await delivered(); + const result = { watch: watch.value, get: await old.rows.get("k", context), pin: reader.n.value }; + await watch.stop(); + old.release(); + reader.release(); + await host.close(context); + return result; + } + + it("re-maps a key watched and pinned at the provider's current version once a later provider maps it", async () => { + const result = await across( + rows(RowsV1, {}), + rows(RowsV3, { 1: { keyed: { rows: ten } }, 2: { keyed: { rows: ten } } }), + ); + expect(result).toEqual({ watch: { n: 20 }, get: { n: 20 }, pin: 20 }); + }); + + it("re-maps a key of an older version its provider left unchanged once a later provider maps it", async () => { + const result = await across( + rows(RowsV2, { 1: {} }), + rows(RowsV3, { 1: { keyed: { rows: ten } }, 2: { keyed: { rows: ten } } }), + ); + expect(result).toEqual({ watch: { n: 20 }, get: { n: 20 }, pin: 20 }); + }); +}); + +describe("reload: keyed sources of an older version through a removal (fix round 2)", () => { + const V1 = local("app.keyed.removal", { + version: 1, + params: {}, + members: { rows: keyed(int()), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const View = local("app.keyed.removal.view", { version: 1, params: {}, members: { n: int() } }); + const keyMappings = instancesOf(AdaptedMembers, "key"); + + /** Serves `V2`, mapping its rows to v1 times `factor`; `set` writes row "a". */ + const rows = (factor: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows: docs, set }; + }, + { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + + /** A v1 consumer's factory pin of row "a". */ + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(V1, {}).rows.at("a"); + return { n: self.derive([pin], ([value]) => value) }; + }), + ); + + it("re-maps an open older keyed watch and a factory pin through the re-served provider (Astra, Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@app/rows", facet: rows(2) }, + { name: "@app/view", facet: view }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + const calls = changesOf(watch); + expect([watch.value, reader.n.value]).toEqual([6, 6]); + await host.reload([{ name: "@app/view", facet: view }], context); + await flushed(); + // Frozen while the type is gone: the last mapped value. + expect(watch.value).toBe(6); + await host.reload( + [ + { name: "@app/rows", facet: rows(4) }, + { name: "@app/view", facet: view }, + ], + context, + ); + await flushed(); + await old.set(5, context); + await delivered(); + expect({ watch: watch.value, get: await old.rows.get("a", context), pin: reader.n.value }).toEqual({ + watch: 20, + get: 20, + pin: 20, + }); + expect(valuesOf(calls).at(-1)).toBe(20); + await watch.stop(); + old.release(); + reader.release(); + await host.close(context); + }); + + it("keeps no removed instance's mappings once the reference moved, its watch open (decision: references only)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const from = keyMappings.length; + const old = await host.get(V1, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + watch.start(async () => {}); + // The removed instance's mapping of the key: nothing of it outlives it through the reference. + const removed = keyMappings.slice(from); + expect(removed.length).toBeGreaterThan(0); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + // The re-served object's documents start over (no store here). + await old.set(3, context); + await delivered(); + expect(watch.value).toBe(12); + await expectCollected(...removed); + await watch.stop(); + old.release(); + await host.close(context); + }); +}); + +describe("reload: a version refusal keeps the waiting reference's last values (fix round 2)", () => { + const V1 = local("app.version.return", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() }))); + const accepted = (value: number) => + defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value }) }), { accepts: { 1: { state: { n: (v) => v.value } } } }), + ); + const refused = defineFacet((h) => h.serve(V2, () => ({ n: constant({ value: 2 }) }))); + + it("keeps an older watch open while its version is refused and heals it when it is accepted again (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/version", facet: accepted(4) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.n.watch(context); + const calls = changesOf(watch); + let closed = false; + void watch.closed.then(() => { + closed = true; + }); + await host.reload([{ name: "@app/version", facet: refused }], context); + await delivered(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // Its last mapped value, read directly too. + expect([watch.value, old.n.value]).toEqual([4, 4]); + await host.reload([{ name: "@app/version", facet: accepted(9) }], context); + await delivered(); + expect({ closed, watch: watch.value, read: old.n.value, last: valuesOf(calls).at(-1) }).toEqual({ + closed: false, + watch: 9, + read: 9, + last: 9, + }); + old.release(); + await host.close(context); + }); + + it("heals a watch of the previous current version once a later provider accepts it (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/version", facet: defineFacet((h) => h.serve(V1, () => ({ n: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.n.watch(context)); + await host.reload([{ name: "@app/version", facet: refused }], context); + (await host.get(V2, {}, context)).release(); + expect(old.state.value.kind).toBe("failed"); + await host.reload([{ name: "@app/version", facet: accepted(3) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(old.state.value.kind).toBe("live"); + expect(valuesOf(calls).at(-1)).toBe(3); + old.release(); + await host.close(context); + }); +}); + +describe("reload: watches of one version across a rollback (fix round 2)", () => { + it("keeps both generations of v1 watches after v1 to v2 to v1 to v2 (Sol)", async () => { + const V1 = local("app.rollback", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const v1 = (n: number) => defineFacet((h) => h.serve(V1, () => ({ x: constant(n) }))); + const v2 = (n: number) => + defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rollback", facet: v1(1) }], context); + const first = await host.get(V1, {}, context); + const firstCalls = changesOf(await first.x.watch(context)); + await host.reload([{ name: "@app/rollback", facet: v2(2) }], context); + (await host.get(V2, {}, context)).release(); + await host.reload([{ name: "@app/rollback", facet: v1(3) }], context); + const second = await host.get(V1, {}, context); + const secondCalls = changesOf(await second.x.watch(context)); + await host.reload([{ name: "@app/rollback", facet: v2(4) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(valuesOf(firstCalls)).toEqual([2, 3, 4]); + expect(valuesOf(secondCalls)).toEqual([4]); + expect(second.x.value).toBe(4); + first.release(); + second.release(); + await host.close(context); + }); +}); + +describe("reload: an older consumer's reads while its type is removed (fix round 2)", () => { + const V1 = local("app.removed.value", { version: 1, params: {}, members: { n: int(), gone: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() })).removeState("gone")); + const facet = defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value: 5 }) }), { + accepts: { 1: { state: { n: (v) => v.value, gone: (state) => state.n.value + 1 } } }, + }), + ); + + it("reads its last mapped value, not the provider's raw shape, a member read before or not (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/removed", facet }], context); + const old = await host.get(V1, {}, context); + expect(old.n.value).toBe(5); + await host.reload([], context); + // `gone` was never read before the removal: the current version has no such member. + expect({ n: old.n.value, gone: old.gone.value }).toEqual({ n: 5, gone: 6 }); + old.release(); + await host.close(context); + }); +}); + +describe("reload: an older consumer's first read during a parked swap (fix round 2)", () => { + it("maps with the provider whose attempt is shown until the new one goes live (Sol)", async () => { + const V1 = local("app.lazy", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.lazy.missing", { version: 1, params: {}, members: {} }); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/lazy", facet: defineFacet((h) => h.serve(V1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + // Nothing read before the swap: the first read comes while the new attempt parks on a missing dependency. + const v2 = defineFacet((h) => + h.serve( + V2, + (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 7 }) }; + }, + { accepts: { 1: { state: { x: (value) => value.n } } } }, + ), + ); + await host.reload([{ name: "@app/lazy", facet: v2 }], context); + await delivered(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(old.x.value).toBe(1); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { name: "@app/lazy", facet: v2 }, + { name: "@app/missing", facet: defineFacet((h) => h.serve(Missing, () => ({}))) }, + ], + context, + ); + await delivered(); + expect(old.state.value).toEqual({ kind: "live" }); + expect([old.x.value, valuesOf(calls).at(-1)]).toEqual([7, 7]); + old.release(); + await host.close(context); + }); +}); + +describe("reload: references keep their last values through a removal, a refusal and an end while they wait (fix round 3)", () => { + const V1 = local("app.waiting", { + version: 1, + params: { id: int() }, + members: { rows: keyed(int()), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const keyMappings = instancesOf(AdaptedMembers, "key"); + + /** Serves `V2`, `set` writing row "a"; mapping v1's rows times `factor`, or refusing v1 without one. */ + const rows = (factor?: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows: docs, set }; + }, + factor === undefined ? {} : { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + + /** A v1 reference to object `id` with row "a" at 3 and an open, started watch of it. */ + async function watched(host: Host, id: number) { + const old = await host.get(V1, { id }, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + const calls = changesOf(watch); + let closed = false; + void watch.closed.then(() => { + closed = true; + }); + return { old, watch, calls, closed: () => closed }; + } + + it("keeps an older keyed watch when the refusal idles its object, which ends (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const { old, watch, closed } = await watched(host, 1); + expect(watch.value).toBe(6); + // v1 refused: its only holder waits, so the object (with params) ends; its pinned keys wait too. + await host.reload([{ name: "@app/rows", facet: rows() }], context); + await flushed(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(5, context); + await delivered(); + expect({ closed: closed(), watch: watch.value, get: await old.rows.get("a", context) }).toEqual({ + closed: false, + watch: 20, + get: 20, + }); + old.release(); + await host.close(context); + }); + + it("keeps an older keyed watch through a refusal and then a removal of its type (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const { old, calls, closed } = await watched(host, 1); + const current = await host.get(V2, { id: 1 }, context); + await host.reload([{ name: "@app/rows", facet: rows() }], context); + await delivered(); + expect(old.state.value.kind).toBe("failed"); + current.release(); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(3, context); + await delivered(); + expect([closed(), valuesOf(calls).at(-1)]).toEqual([false, 12]); + old.release(); + await host.close(context); + }); + + it("keeps no removed instance's mappings while its keyed watch stays open (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const from = keyMappings.length; + const { old, watch } = await watched(host, 1); + const removed = keyMappings.slice(from); + expect(removed.length).toBeGreaterThan(0); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(3, context); + await delivered(); + expect(watch.value).toBe(12); + // The watch is still open: what it follows now is the re-served object's instance. + await expectCollected(...removed); + old.release(); + await host.close(context); + }); + + it("reads an older member never read before at its last mapped value after a refusal (Astra)", async () => { + const State1 = local("app.waiting.state", { version: 1, params: {}, members: { n: int() } }); + const State2 = evolve(State1, (e) => e.state("n", obj({ value: int() }))); + const serve = (value: number, accepts: boolean) => + defineFacet((h) => + h.serve( + State2, + () => ({ n: constant({ value }) }), + accepts ? { accepts: { 1: { state: { n: (v) => v.value } } } } : {}, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/state", facet: serve(4, true) }], context); + const old = await host.get(State1, {}, context); + await host.reload([{ name: "@app/state", facet: serve(9, false) }], context); + await flushed(); + expect(old.n.value).toBe(4); + old.release(); + await host.close(context); + }); + + it("follows a newer consumer through two rollbacks, its last value kept while refused (Sol)", async () => { + const X1 = local("app.waiting.rollback", { version: 1, params: {}, members: { x: int() } }); + const X2 = evolve(X1, (e) => e.state("x", obj({ n: int() }))); + const v1 = (n: number) => defineFacet((h) => h.serve(X1, () => ({ x: constant(n) }))); + const v2 = (n: number) => + defineFacet((h) => + h.serve(X2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rollback", facet: v2(1) }], context); + const held = await host.get(X2, {}, context); + const calls = changesOf(await held.x.watch(context)); + for (const [back, ahead] of [ + [2, 3], + [4, 5], + ]) { + // A v1 provider cannot serve the v2 consumer: it waits with its source, then takes it back. + await host.reload([{ name: "@app/rollback", facet: v1(back!) }], context); + await delivered(); + expect(held.state.value.kind).toBe("failed"); + await host.reload([{ name: "@app/rollback", facet: v2(ahead!) }], context); + (await host.get(X2, {}, context)).release(); + await delivered(); + } + // Each refusal's failure once, then each return's value, whole. + const refused = { failed: "app.waiting.rollback v2 is not served here" }; + expect([held.x.value, valuesOf(calls)]).toEqual([{ n: 5 }, [refused, { n: 3 }, refused, { n: 5 }]]); + held.release(); + await host.close(context); + }); +}); + +describe("reload: a released reference's last values go with it while another version waits (fix round 4)", () => { + it("drops a removed version's mapped value when its last request leaves while another version waits (Astra)", async () => { + const H1 = local("app.waiting.versions", { + version: 1, + params: { id: int() }, + members: { n: obj({ mapped: int() }) }, + }); + const H2 = evolve(H1, (e) => e.state("n", obj({ raw: int() }))); + const facet = defineFacet((h) => + h.serve(H2, () => ({ n: constant({ raw: 7 }) }), { + accepts: { 1: { state: { n: (v) => ({ mapped: v.raw }) } } }, + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/versions", facet }], context); + const older = await host.get(H1, { id: 1 }, context); + const newer = await host.get(H2, { id: 1 }, context); + const mapped = new WeakRef(older.n.value); + // Both versions wait with their last values; v1's reference is released, v2's stays. + await host.reload([], context); + older.release(); + await flushed(); + await expectCollected(mapped); + newer.release(); + await host.close(context); + }); + + it("drops a refused version's snapshot when its last request leaves while an older one waits (Sol)", async () => { + const G1 = local("app.waiting.chain", { version: 1, params: {}, members: { x: obj({ n: int() }) } }); + const G2 = evolve(G1, (e) => e.state("x", obj({ n: int() }))); + const G3 = evolve(G2, (e) => e.state("x", obj({ n: int() }))); + const serve = (contract: typeof G1 | typeof G2 | typeof G3, n: number) => + defineFacet((h) => h.serve(contract as typeof G3, () => ({ x: constant({ n }) }))); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/chain", facet: serve(G1, 1) }], context); + const first = await host.get(G1, {}, context); + await host.reload([{ name: "@app/chain", facet: serve(G2, 2) }], context); + const second = await host.get(G2, {}, context); + const snapshot = new WeakRef(second.x.value); + // v3 accepts neither: both wait; v2's last request leaves, v1's stays. + await host.reload([{ name: "@app/chain", facet: serve(G3, 3) }], context); + (await host.get(G3, {}, context)).release(); + second.release(); + await delivered(); + expect(first.state.value.kind).toBe("failed"); + await expectCollected(snapshot); + first.release(); + await host.close(context); + }); +}); + +describe("reload: stopping a keyed watch drops its pin and last value while its reference waits (fix round 4)", () => { + it("drops a stopped key's document while its reference still waits (Sol)", async () => { + const K1 = local("app.waiting.key", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), extra: int(), set: method(int()) }, + }); + const K2 = evolve(K1, (e) => e.state("extra", int())); + const Doc = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 1 }), + keyed: true, + }); + const facet = defineFacet((h) => + h.serve( + K2, + (_, self) => { + const rows = self.state(Doc); + const set = async (n: number, ctx: Context) => { + await rows.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows, extra: constant(0), set }; + }, + { accepts: { 1: { state: { extra: (value) => value } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/key", facet }], context); + const ref = await host.get(K1, {}, context); + await ref.set(3, context); + // A watch of the key while the type is removed (the reference waits, its key with its last value), then + // stopped; the watch itself goes out of scope here. + const watched = async () => { + const watch = (await ref.rows.watch("k", context))!; + watch.start(async () => {}); + const value = new WeakRef(watch.value); + await host.reload([], context); + await watch.stop(); + return value; + }; + const document = await watched(); + expect(ref.state.value.kind).toBe("failed"); + await expectCollected(document); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: a reference that never had values reads none during a parked swap (fix round 4)", () => { + it("has a newer reference read nothing during a parked swap, its watch waiting, then the new instance's values (Sol)", async () => { + const N1 = local("app.unshown", { version: 1, params: {}, members: { x: int() } }); + const N2 = evolve(N1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.unshown.missing", { version: 1, params: {}, members: {} }); + const v2 = defineFacet((h) => + h.serve( + N2, + (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 7 }) }; + }, + { accepts: { 1: { state: { x: (value) => value.n } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/unshown", facet: defineFacet((h) => h.serve(N1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(N1, {}, context); + await host.reload([{ name: "@app/unshown", facet: v2 }], context); + const newer = host.hold(N2, {}, context); + expect(newer.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // The swap parks: the shown values are v1's; a v2 reference never had values, never v1's shape: its read throws + // the parked swap's failure. + expect(old.x.value).toBe(1); + expect(() => newer.x.value).toThrow(expect.objectContaining({ code: "unavailable" })); + // A watch waits for its first values through the failure (Opus G3). + const waiting = newer.x.watch(context); + await host.reload( + [ + { name: "@app/unshown", facet: v2 }, + { name: "@app/missing", facet: defineFacet((h) => h.serve(Missing, () => ({}))) }, + ], + context, + ); + await delivered(); + expect(newer.state.value).toEqual({ kind: "live" }); + expect([newer.x.value, old.x.value]).toEqual([{ n: 7 }, 7]); + expect([(await waiting).value, (await newer.x.watch(context)).value]).toEqual([{ n: 7 }, { n: 7 }]); + newer.release(); + old.release(); + await host.close(context); + }); +}); + +describe("reload: a reference reads its last value while its version's return parks (fix round 5)", () => { + it("reads the last value its surviving watch shows after v2 to v1 to a parked v2 (Sol)", async () => { + const R1 = local("app.return", { version: 1, params: {}, members: { x: int() } }); + const R2 = evolve(R1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.return.missing", { version: 1, params: {}, members: {} }); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/return", facet: defineFacet((h) => h.serve(R2, () => ({ x: constant({ n: 7 }) }))) }], + context, + ); + const ref = await host.get(R2, {}, context); + const watch = await ref.x.watch(context); + watch.start(async () => {}); + // A v1 provider cannot serve v2: the reference waits with its last value. + await host.reload( + [{ name: "@app/return", facet: defineFacet((h) => h.serve(R1, () => ({ x: constant(1) }))) }], + context, + ); + expect(ref.x.value).toEqual({ n: 7 }); + // v2 is served again but its swap parks: the sources taken back are v2's own, shown before. + const parked = defineFacet((h) => + h.serve(R2, (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 9 }) }; + }), + ); + await host.reload([{ name: "@app/return", facet: parked }], context); + await delivered(); + expect([watch.value, ref.x.value, (await ref.x.watch(context)).value]).toEqual([{ n: 7 }, { n: 7 }, { n: 7 }]); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: one rule for reads and watches across a new object (fix round 6)", () => { + const Type = local("app.successor", { version: 1, params: {}, members: { x: int() } }); + const Missing = local("app.successor.missing", { version: 1, params: {}, members: {} }); + + it("reads the last value its watch shows while the re-served object parks (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/successor", facet: defineFacet((h) => h.serve(Type, () => ({ x: constant(7) }))) }], + context, + ); + const ref = await host.get(Type, {}, context); + const watch = await ref.x.watch(context); + watch.start(async () => {}); + await host.reload([], context); + expect(ref.x.value).toBe(7); + // Re-served: a new object, which parks before any instance of it went live: the reference keeps its last value. + const parked = defineFacet((h) => + h.serve(Type, (_, self) => { + self.get(Missing, {}); + return { x: constant(9) }; + }), + ); + await host.reload([{ name: "@app/successor", facet: parked }], context); + await delivered(); + expect(ref.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect([watch.value, ref.x.value]).toEqual([7, 7]); + ref.release(); + await host.close(context); + }); + + it("opens a new watch at its last value while the re-served object's native opens (Sol)", async () => { + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/successor", facet: defineFacet((h) => h.serve(Type, () => ({ x: constant(7) }))) }], + context, + ); + const ref = await host.get(Type, {}, context); + await host.reload([], context); + const opening = defineFacet((h) => + h.serve(Type, (_, self) => { + self.open( + async () => gate.promise, + () => {}, + ); + return { x: constant(9) }; + }), + ); + const reloading = host.reload([{ name: "@app/successor", facet: opening }], context); + await delivered(); + expect(ref.state.value.kind).toBe("loading"); + expect(ref.x.value).toBe(7); + // At once, not after hydration: the source of its version exists. + const watch = await ref.x.watch(context); + expect(watch.value).toBe(7); + const calls = changesOf(watch); + gate.resolve(1); + await reloading; + await delivered(); + expect(valuesOf(calls).at(-1)).toBe(9); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: an older version only a pending provider maps has no values yet (fix round 7)", () => { + const A1 = local("app.pending.accept", { version: 1, params: {}, members: { x: int() } }); + const A2 = evolve(A1, (e) => e.state("x", obj({ n: int() }))); + const A3 = evolve(A2, (e) => e.state("x", obj({ wrapped: obj({ n: int() }) }))); + + /** v2 shown (no accepts for v1), a reload to v3 (accepting v1 and v2) gated on a native; a new v1 hold, loading. */ + async function pending() { + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/accept", facet: defineFacet((h) => h.serve(A2, () => ({ x: constant({ n: 2 }) }))) }], + context, + ); + const v3 = defineFacet((h) => + h.serve( + A3, + (_, self) => { + self.open( + async () => gate.promise, + () => {}, + ); + return { x: constant({ wrapped: { n: 7 } }) }; + }, + { + accepts: { + 1: { state: { x: (value) => value.wrapped.n } }, + 2: { state: { x: (value) => value.wrapped } }, + }, + }, + ), + ); + const reloading = host.reload([{ name: "@app/accept", facet: v3 }], context); + await delivered(); + return { host, ref: host.hold(A1, {}, context), gate, reloading }; + } + + it("reads not_ready rather than asking the shown provider for a mapping it lacks (Sol)", async () => { + const { host, ref, gate, reloading } = await pending(); + expect(ref.state.value.kind).toBe("loading"); + expect(() => ref.x.value).toThrow(expect.objectContaining({ code: "not_ready" })); + gate.resolve(1); + await reloading; + await delivered(); + expect(ref.x.value).toBe(7); + ref.release(); + await host.close(context); + }); + + it("opens a watch once the accepting provider is live, not before (Sol)", async () => { + const { host, ref, gate, reloading } = await pending(); + let opened = false; + const opening = ref.x.watch(context).then((watch) => { + opened = true; + return watch; + }); + await delivered(); + expect(opened).toBe(false); + gate.resolve(1); + await reloading; + expect((await opening).value).toBe(7); + ref.release(); + await host.close(context); + }); +}); + +describe('an older consumer keeps nothing of its instance\'s mapping (decision "References are the only indirection")', () => { + const mappings = instancesOf(AdaptedMembers, "member"); + + it("lets an ended instance's mapping derives go while the older reference waits with an open watch", async () => { + const M1 = local("app.mapped.gc", { version: 1, params: {}, members: { n: int() } }); + const M2 = evolve(M1, (e) => e.state("n", obj({ value: int() }))); + const facet = defineFacet((h) => + h.serve(M2, () => ({ n: constant({ value: 4 }) }), { accepts: { 1: { state: { n: (v) => v.value } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/mapped", facet }], context); + const from = mappings.length; + const old = await host.get(M1, {}, context); + const calls = changesOf(await old.n.watch(context)); + const used = mappings.slice(from); + expect(used.length).toBeGreaterThan(0); + await host.reload([], context); + await delivered(); + // Its last mapped value and the failure once; the instance's mapping went with it. + expect([old.n.value, valuesOf(calls)]).toEqual([4, [{ failed: "app.mapped.gc is not served here" }]]); + await expectCollected(...used); + old.release(); + await host.close(context); + }); +}); + +describe("an instance's mapping detaches only what it made (final review)", () => { + it("keeps an unchanged member, the instance's own derive, when the last older reference leaves (Astra)", async () => { + const D1 = local("app.shared-derive", { version: 1, params: {}, members: { n: int(), set: method(int()) } }); + const D2 = evolve(D1, (e) => e.addState("extra", int())); + const Counter = defineState("counter", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 1 }) }); + const facet = defineFacet((h) => + h.serve( + D2, + (_, self) => { + const counter = self.state(Counter); + const set = async (n: number, ctx: Context) => { + await counter.change((draft) => { + draft.n = n; + }, ctx); + return null; + }; + return { n: self.derive([counter], ([c]) => c.n * 2), extra: constant(0), set }; + }, + { accepts: { 1: {} } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/derive", facet }], context); + const newer = await host.get(D2, {}, context); + const older = await host.get(D1, {}, context); + // v1 reads `n` unchanged: the instance's own derive, not a mapping. + expect(older.n.value).toBe(2); + const calls = changesOf(await newer.n.watch(context)); + older.release(); + await newer.set(3, context); + await delivered(); + expect([newer.n.value, valuesOf(calls)]).toEqual([6, [6]]); + newer.release(); + await host.close(context); + }); +}); + +describe("keyed mappings per member (final review)", () => { + it("maps one document exposed as two keyed members through each member's own mapping (Astra)", async () => { + const K1 = local("app.keyed-alias", { + version: 1, + params: {}, + members: { first: keyed(int()), second: keyed(int()), set: method(int()) }, + }); + const K2 = evolve(K1, (e) => e.keyed("first", obj({ n: int() })).keyed("second", obj({ n: int() }))); + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + keyed: true, + }); + const facet = defineFacet((h) => + h.serve( + K2, + (_, self) => { + const rows = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await rows.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { first: rows, second: rows, set }; + }, + { accepts: { 1: { keyed: { first: (value) => value.n, second: (value) => value.n * 10 } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/keyed-alias", facet }], context); + const ref = await host.get(K1, {}, context); + await ref.set(5, context); + const first = (await ref.first.watch("a", context))!; + const second = (await ref.second.watch("a", context))!; + expect({ first: first.value, second: second.value, read: await ref.second.get("a", context) }).toEqual({ + first: 5, + second: 50, + read: 50, + }); + await first.stop(); + await second.stop(); + ref.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts new file mode 100644 index 000000000..9f7e7e46a --- /dev/null +++ b/packages/chord/test/next/host/attempt.test.ts @@ -0,0 +1,993 @@ +import { createHook } from "node:async_hooks"; +import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, opt, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type HostError, + type Reference, + type Self, + type StoreSelf, +} from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { + failLive, + failures, + fakeTimers, + flushed, + instancesOf, + loaded, + memoryStore, + metered, + rejection, + Server, + tableOwner, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); + +describe("failure kinds", () => { + const Broken = local("app.broken", { version: 1, params: { id: int() }, members: { label: str() } }); + const Bad = defineState("bad", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: "x" }) as never }); + + it.each<[string, (self: Self) => unknown, string]>([ + [ + "a throwing factory", + () => { + throw new Error("factory threw"); + }, + "app.broken(1): factory threw", + ], + ["a missing member", () => ({}), "app.broken(1): label: a state member is a JSON watchable"], + ["an unknown member", () => ({ label: constant(""), extra: 1 }), "app.broken(1): extra: not a member"], + [ + "a native watchable as state (Q14)", + () => ({ label: { value: "x", watch: async () => ({}) } }), + "app.broken(1): label: a state member is a JSON watchable", + ], + [ + "a throwing first derive", + (self) => ({ + label: self.derive([], () => { + throw new Error("derive threw"); + }), + }), + "app.broken(1): derive threw", + ], + [ + "an initial() that fails its schema", + (self) => ({ label: self.derive([self.state(Bad)], () => "") }), + "app.broken(1): bad: initial() does not match the schema", + ], + [ + "an open that throws broken(error)", + (self) => { + self.open( + async () => { + throw broken(new Error("no remote configured")); + }, + () => {}, + ); + return { label: constant("") }; + }, + "app.broken(1): no remote configured", + ], + ])( + "%s fails broken: reported, the get rejects at once with a ChordError, nothing is left", + async (_, factory, message) => { + const served = vi.fn((_: { id: number }, self: Self) => factory(self) as never); + const { host, errors } = await loaded((h) => h.serve(Broken, served)); + const error = await host.get(Broken, { id: 1 }, context).catch((error: unknown) => error); + expect(isChordError(error) && error.message).toBe(message); + await expect(host.get(Broken, { id: 1 }, context)).rejects.toThrow(message); + expect(served).toHaveBeenCalledTimes(2); + expect(failures(errors)).toEqual(["app.broken(1): broken", "app.broken(1): broken"]); + // One report per failure: a derive throwing while its attempt activates reports no derive_failed. + expect(errors.map(({ kind }) => kind)).toEqual(["activation_failed", "activation_failed"]); + }, + ); + + it("an open that throws fails transient: reported, the get waits through retries", async () => { + const { host, errors } = await loaded((h) => + h.serve(Broken, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { label: constant("") }; + }), + ); + expect(await rejection(host.get(Broken, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.broken(1): offline", + }); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.broken(1)", + failure: { kind: "transient", error: { code: "unavailable" } }, + }, + ]); + await host.close(context); + }); + + describe("an attempt not live within readyTimeout", () => { + fakeTimers(); + + it("is abandoned and fails transient: the opener's context aborted, what opened closed", async () => { + let opener: Context | undefined; + const closes: string[] = []; + const { host, errors } = await loaded( + (h) => + h.serve(Broken, (_, self) => { + self.open( + async () => "a", + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + self.open( + async (ctx) => { + opener = ctx; + return new Promise(() => {}); + }, + () => {}, + ); + return { label: constant("") }; + }), + { readyTimeout: 100 }, + ); + const got = rejection(host.get(Broken, { id: 1 }, context, { readyTimeout: 150 })); + await vi.advanceTimersByTimeAsync(100); + expect([opener?.abortSignal?.aborted, closes]).toEqual([true, ["a end"]]); + const notLive = { code: "unavailable", message: "app.broken(1): did not become live within 100 ms" }; + expect(errors).toMatchObject([{ kind: "activation_failed", failure: { kind: "transient", error: notLive } }]); + await vi.advanceTimersByTimeAsync(50); + expect(await got).toEqual(notLive); + }); + }); + + describe("an attempt waiting for its previous instance's teardown", () => { + fakeTimers(); + + it("is bounded by readyTimeout too: abandoned transient without running its factory, retried after it", async () => { + const closing = Promise.withResolvers(); + let factories = 0; + const { host, errors } = await loaded( + (h) => + h.serve(Page, (_, self) => { + factories++; + self.open( + async () => "native", + () => closing.promise, + ); + return { label: constant("p"), slow: async () => null }; + }), + { readyTimeout: 100, closeTimeout: 10_000 }, + ); + (await host.get(Page, { id: 1 }, context)).release(); + const next = host.hold(Page, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(100); + expect(factories).toBe(1); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + failure: { kind: "transient", error: { message: "app.page(1): did not become live within 100 ms" } }, + }, + ]); + closing.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + expect([factories, next.state.value]).toEqual([2, { kind: "live" }]); + }); + }); + + it("a keyed document's initial() failing its schema fails that transaction, not the object", async () => { + const Keys = defineState("keys", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: "x" }) as never, + keyed: true, + }); + const { host, errors } = await loaded((h) => + h.serve(Page, (_, self) => { + const keys = self.state(Keys); + return { + label: constant("live"), + slow: async (ctx) => { + await keys.change("a", () => {}, ctx); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + await expect(page.slow(context)).rejects.toThrow("keys: initial() does not match the schema"); + expect([page.label.value, errors]).toEqual(["live", []]); + }); + + it("releases the references it acquired", async () => { + const pages = vi.fn((_: { id: number }) => ({ label: constant("p"), slow: async () => null })); + const { host } = await loaded((h) => { + h.serve(Broken, (_, self) => { + const page = self.get(Page, { id: 7 }); + const label = self.derive([page.label], (): string => { + throw new Error("late"); + }); + return { label }; + }); + h.serve(Page, pages); + }); + await expect(host.get(Broken, { id: 1 }, context)).rejects.toThrow("late"); + expect(pages).toHaveBeenCalledTimes(1); + await host.get(Page, { id: 7 }, context); + expect(pages).toHaveBeenCalledTimes(2); + }); +}); + +describe("ending", () => { + it("rejects their callers at once; the handler runs on with its context aborted and cannot commit", async () => { + const started = Promise.withResolvers(); + const proceed = Promise.withResolvers(); + const finished = Promise.withResolvers<{ aborted?: boolean; error?: unknown }>(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + const list = self.state(List); + return { + label: constant(""), + slow: async (ctx) => { + started.resolve(); + await proceed.promise; + const tx = self.tx(async (tx) => (await tx.draft(list)).items.push("x"), context); + finished.resolve({ aborted: ctx.abortSignal?.aborted, error: await rejection(tx) }); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await started.promise; + page.release(); + const released = { code: "released", reason: "ended", message: "app.page(1): ended" }; + expect(await rejection(call)).toEqual(released); + proceed.resolve(); + expect(await finished.promise).toEqual({ aborted: true, error: released }); + }); + + it("rejects a call whose caller's context is already aborted, before the handler runs", async () => { + const slow = vi.fn(async () => null); + const { host } = await loaded((h) => h.serve(Page, () => ({ label: constant(""), slow }))); + const page = await host.get(Page, { id: 1 }, context); + const { context: cancelled, cancel } = withCancel(context); + cancel(new Error("gone")); + await expect(page.slow(cancelled)).rejects.toThrow("gone"); + expect(slow).not.toHaveBeenCalled(); + }); + + it("waits for the running handlers' own promises (the drain) before the natives close", async () => { + const events: string[] = []; + const started = Promise.withResolvers(); + const proceed = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + self.open( + async () => "native", + (value, reason) => void events.push(`close ${value} ${reason.kind}`), + ); + return { + label: constant(""), + slow: async () => { + started.resolve(); + await proceed.promise; + events.push("handler settled"); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await started.promise; + page.release(); + await expect(call).rejects.toThrow("app.page(1): ended"); + await flushed(); + expect(events).toEqual([]); + proceed.resolve(); + await flushed(); + expect(events).toEqual(["handler settled", "close native end"]); + }); + + it("drains a handler that releases its object's last reference before it returns its promise", async () => { + const events: string[] = []; + const proceed = Promise.withResolvers(); + let page: Reference | undefined; + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + self.open( + async () => null, + () => void events.push("closed"), + ); + return { + label: constant(""), + // Not async: the release runs before the handler has a promise to return. + slow: () => { + page!.release(); + return proceed.promise.then(() => { + events.push("handler settled"); + return null; + }); + }, + }; + }), + ); + page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await expect(call).rejects.toThrow("app.page(1): ended"); + await flushed(); + expect(events).toEqual([]); + proceed.resolve(); + await flushed(); + expect(events).toEqual(["handler settled", "closed"]); + }); + + it("refuses transactions at once; documents stay readable until the closes are done", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const seen = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + const handle = self.state(List); + list = handle; + self.open( + async () => null, + async () => seen.resolve([handle.value, await rejection(handle.change(() => {}, context))]), + ); + return { label: constant(""), slow: async () => null }; + }), + ); + (await host.get(Page, { id: 1 }, context)).release(); + expect(await seen.promise).toEqual([ + { items: [] }, + { code: "released", reason: "ended", message: "app.page(1): ended" }, + ]); + await flushed(); + expect(() => list!.value).toThrow("app.page(1): ended"); + }); + + describe("the drain", () => { + fakeTimers(); + + it("is bounded by closeTimeout: a handler still running then sees its natives close, which is reported", async () => { + const closed = vi.fn(); + const { host, errors } = await loaded( + (h) => + h.serve(Page, (_, self) => { + self.open(async () => null, closed); + return { label: constant(""), slow: () => new Promise(() => {}) }; + }), + { closeTimeout: 1000 }, + ); + const page = await host.get(Page, { id: 1 }, context); + void page.slow(context).catch(() => {}); + page.release(); + await vi.advanceTimersByTimeAsync(999); + expect(closed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(closed).toHaveBeenCalledTimes(1); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.page(1)" }]); + }); + + it("a handler stuck past it no longer counts as a call nor holds its ctx.get references", async () => { + const Item = local("app.item", { version: 1, params: { id: int() }, members: { label: str() } }); + const got = Promise.withResolvers(); + const { host } = await loaded( + (h) => { + h.serve(Item, () => ({ label: constant("item") })); + h.serve(Page, () => ({ + label: constant(""), + slow: async (ctx) => { + await ctx.get(Item, { id: 7 }); + got.resolve(); + return new Promise(() => {}); + }, + })); + }, + { closeTimeout: 1000 }, + ); + const items = host.references(Item); + const page = await host.get(Page, { id: 1 }, context); + void page.slow(context).catch(() => {}); + await got.promise; + expect(host.activity.value.calls).toBe(1); + expect(items.value).toHaveLength(1); + page.release(); + await vi.advanceTimersByTimeAsync(1000); + expect(host.activity.value.calls).toBe(0); + expect(items.value).toEqual([]); + }); + }); +}); + +describe("declarations", () => { + it("are valid only while the factory runs", async () => { + let later: Self | undefined; + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + later = self; + return { label: constant(""), slow: async () => null }; + }), + ); + await host.get(Page, { id: 1 }, context); + expect(() => later!.state(List)).toThrow("app.page(1): self.state is valid only in the factory"); + expect(() => later!.derive([], () => 1)).toThrow("self.derive is valid only in the factory"); + expect(() => later!.get(Page, { id: 2 })).toThrow("self.get is valid only in the factory"); + expect(() => + later!.open( + async () => null, + () => {}, + ), + ).toThrow("self.open is valid only in the factory"); + const Names = defineRegistry("app.names", { version: 1 }); + expect(() => later!.registry(Names)).toThrow("self.registry is valid only in the factory"); + expect(() => later!.writable(1)).toThrow("self.writable is valid only in the factory"); + expect(() => later!.retryOn()).toThrow("self.retryOn is valid only in the factory"); + }); + + it("contribute only in host-lifetime objects (R21): `Self` has no `contribute`, a declaration error at runtime", async () => { + expectTypeOf().not.toHaveProperty("contribute"); + expectTypeOf().toHaveProperty("contribute"); + const Names = defineRegistry("app.names", { version: 1 }); + const Store = local("app.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded((h) => { + h.serve(Names, { compose: (table) => table.length }); + h.serve(Page, (_, self) => { + (self as unknown as StoreSelf).contribute(Names, "page"); + return { label: constant(""), slow: async () => null }; + }); + h.serve(Store, (_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + return {}; + }); + }); + await expect(host.get(Page, { id: 1 }, context)).rejects.toThrow( + "app.page(1): self.contribute is valid only in a local type with empty params", + ); + await expect(host.get(Store, {}, context)).rejects.toThrow( + "app.store: registry app.names is contributed to twice", + ); + await host.close(context); + }); + + it("refuse a document definition declared twice", async () => { + const Twice = local("app.twice", { version: 1, params: { n: int() }, members: {} }); + const { host } = await loaded((h) => + h.serve(Twice, (_, self) => { + [self.state(List), self.state(List)]; + return {}; + }), + ); + await expect(host.get(Twice, { n: 2 }, context)).rejects.toThrow('app.twice(2): state "list" is declared twice'); + }); +}); + +describe("calls", () => { + it("check arguments with L1 and pass arguments and results by reference", async () => { + const seen: unknown[] = []; + const result = { items: [] }; + const Echo = local("app.echo", { + version: 1, + params: {}, + members: { echo: method(obj({ text: str() }), obj({ items: arr(str()) })), none: method() }, + }); + const { host } = await loaded((h) => + h.serve(Echo, () => ({ + echo: async (args) => { + seen.push(args); + return result; + }, + // A method without arguments gets only the context. + none: async (...args) => { + seen.push(args.length); + return null; + }, + })), + ); + const echo = await host.get(Echo, {}, context); + const args = { text: "t" }; + expect(await echo.echo(args, context)).toBe(result); + expect(await echo.none(context)).toBeNull(); + expect(seen).toEqual([args, 1]); + expect(seen[0]).toBe(args); + // @ts-expect-error: an unknown field + expect(await rejection(echo.echo({ text: "t", extra: 1 }, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("take a plain value or a synchronous throw from a handler that is no async function", async () => { + const Plain = local("app.plain", { + version: 1, + params: {}, + members: { value: method(undefined, int()), fail: method() }, + }); + const { host } = await loaded((h) => + // Plugin code is bundled without type checks. + h.serve(Plain, () => ({ + value: (() => 7) as never, + fail: (() => { + throw new Error("sync throw"); + }) as never, + })), + ); + const plain = await host.get(Plain, {}, context); + expect(await plain.value(context)).toBe(7); + await expect(plain.fail(context)).rejects.toThrow("sync throw"); + }); + + it("cost one validation per call and one compilation per method", async () => { + const telemetry = await metered(); + // A fresh declaration: compiled checks are cached per contract. + const Fresh = local("app.store", { version: 1, params: {}, members: { add: method(obj({ text: str() })) } }); + const { host } = await loaded((h) => h.serve(Fresh, () => ({ add: async () => null }))); + const store = await host.get(Fresh, {}, context); + await store.add({ text: "a" }, telemetry.context); + await store.add({ text: "b" }, telemetry.context); + expect(telemetry.counts()).toEqual({ "chord.data.compilations": 1, "chord.data.validations": 2 }); + }); +}); + +describe("the call path's cost", () => { + it("makes at most 19 promises per local call (a guard: the drain and the cleanups share one reaction)", async () => { + const Touch = local("app.touch", { version: 1, params: {}, members: { touch: method() } }); + const { host } = await loaded((h) => h.serve(Touch, () => ({ touch: async () => null }))); + const ref = await host.get(Touch, {}, context); + await ref.touch(context); + let promises = 0; + // Every `then` makes a promise: one more reaction per call shows here (HEAD before the cleanups made 20). + const hook = createHook({ + init: (_id, type) => { + if (type === "PROMISE") promises++; + }, + }); + hook.enable(); + try { + for (let call = 0; call < 100; call++) await ref.touch(context); + } finally { + hook.disable(); + } + expect(promises / 100).toBeLessThanOrEqual(19); + await host.close(context); + }); +}); + +describe("remote variants", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { n: int() } } }, + }); + const owner = { kind: "owner" } as const; + + /** A host acting as `owner` whose factory for `Two` returns `implementation`. */ + async function two(implementation: unknown) { + const errors: HostError[] = []; + const host = createHost({ + address: Server(), + principal: owner, + onError: (error) => errors.push(error), + }); + const facet = defineFacet((h) => h.serve(Two, () => implementation as never)); + await host.load([{ name: "@test/app", facet }], context); + return { host, errors }; + } + + it("the factory chooses one; the reference narrows on its variant", async () => { + const { host } = await two({ variant: "view", members: { n: constant(1) } }); + const ref = await host.get(Two, {}, context); + expect(ref.variant).toBe("view"); + if (ref.variant === "view") expect(ref.n.value).toBe(1); + }); + + it.each([ + ["an unknown variant", { variant: "other", members: {} }, 'variant "other": not a variant of app.two'], + ["a missing member", { variant: "full", members: { n: constant(1) } }, "bump: a method is a function"], + [ + "an extra member", + { variant: "view", members: { n: constant(1), bump: async () => null } }, + "bump: not a member", + ], + ])("%s fails the activation broken", async (_, implementation, message) => { + const { host, errors } = await two(implementation); + expect(await rejection(host.get(Two, {}, context))).toEqual({ + code: "unavailable", + message: `app.two: ${message}`, + }); + expect(failures(errors)).toEqual(["app.two: broken"]); + }); + + it("a member outside the bound variant fails invalid/member", async () => { + const { host } = await two({ variant: "view", members: { n: constant(1) } }); + const ref = (await host.get(Two, {}, context)) as unknown as { bump(ctx: Context): Promise }; + expect(await rejection(ref.bump(context))).toEqual({ + code: "invalid", + reason: "member", + message: 'app.two (variant view) has no method "bump"', + }); + }); +}); + +describe("tables", () => { + const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), at: int() }, + key: ["id"], + indexes: { by_at: ["at"] }, + }); + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + + it("a fresh table is created at activation at its version, with its indexes, in one commit", async () => { + const store = await memoryStore({ outbox: true }); + const { host } = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await host.get(Store, {}, context); + expect(store.table("app.store:edits")).toMatchObject({ + version: 1, + state: "ready", + indexes: { by_at: ["at"] }, + }); + expect((await store.outbox!.read(0, 10, context)).map(({ writes }) => writes.map(({ kind }) => kind))).toEqual([ + ["table", "index"], + ]); + await host.close(context); + const again = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await again.host.get(Store, {}, context); + expect(store.lastSeq).toBe(1); + await again.host.close(context); + await store.close(context); + }); + + it.each<[string, (self: StoreSelf) => void, string]>([ + [ + "declared twice", + (self) => { + self.table(Edits); + self.table(Edits); + }, + 'table "edits" is declared twice', + ], + [ + "another version recorded", + (self) => self.table(defineTable("edits", { version: 2, columns: { id: str() }, key: ["id"] })), + "older than the history's base 2", + ], + [ + "another definition at the same version", + (self) => self.table(defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] })), + "recorded with another definition", + ], + ])("fails the activation broken: %s", async (_, declare, message) => { + const store = await memoryStore(); + const first = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await first.host.get(Store, {}, context); + await first.host.close(context); + const { host } = await loaded( + (h) => + h.serve(Store, (_, self) => { + declare(self); + return { n: constant(1) }; + }), + { store }, + ); + expect(await rejection(host.get(Store, {}, context))).toMatchObject({ + message: expect.stringContaining(message), + }); + await host.close(context); + await store.close(context); + }); + + it("needs a local type with empty params (P3) and a host with a store", async () => { + const { host } = await loaded((h) => { + h.serve(Page, (_, self) => { + // @ts-expect-error: only a local type with empty params has tables (P3) + const declare = () => self.table(Edits); + void declare; + (self as unknown as StoreSelf).table(Edits); + return { label: constant(""), slow: async () => null }; + }); + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }); + }); + expect(await rejection(host.get(Page, { id: 1 }, context))).toMatchObject({ + message: expect.stringContaining("persist needs a stable identity"), + }); + expect(await rejection(host.get(Store, {}, context))).toMatchObject({ + message: expect.stringContaining("persist needs a store"), + }); + await host.close(context); + }); +}); + +describe("eager migrations at activation (slice 6a §9)", () => { + fakeTimers(); + // The clock is faked, but the zero-delay timers a migration yields with between chunks run every real turn. + let pumping = true; + beforeEach(() => { + pumping = true; + const pump = (): void => { + if (!pumping) return; + vi.advanceTimersByTime(0); + setImmediate(pump); + }; + setImmediate(pump); + }); + afterEach(() => { + pumping = false; + }); + const TABLE = "app.store:edits"; + const OPEN = { admit: () => true, adopt: () => {} }; + const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); + + /** A store whose edits table is at version 1 with `count` rows. */ + async function seeded(count: number): Promise { + const store = await memoryStore(); + await prepareTables([{ definition: EditsV1, table: TABLE }], store, OPEN, "app.store", context); + const staged: StagedWrite[] = []; + const changes = new TableChanges(new TableHandle(EditsV1, TABLE, store, tableOwner()), staged); + for (let i = 0; i < count; i++) changes.put({ id: `e${String(i).padStart(3, "0")}`, at: i }); + await store.commit(staged, OPEN, context); + return store; + } + + /** Waits (real turns of the event loop, the clock faked) until `done`. */ + async function until(done: () => boolean): Promise { + for (let turn = 0; turn < 1_000 && !done(); turn++) await flushed(); + expect(done()).toBe(true); + } + + /** Version 2 adding `note`, whose `rows` runs `each` per row first. */ + function evolved(each: (id: string) => void) { + return evolveTable(EditsV1, (t) => + t.add("note", opt(str())).rows((row) => { + each(row.id); + return { note: `#${row.at}` }; + }), + ); + } + + /** A native whose open never succeeds: it rejects when its context is aborted (the deadline). */ + function hanging(ctx: Context): Promise { + return new Promise((_, reject) => ctx.abortSignal!.addEventListener("abort", () => reject(new Error("aborted")))); + } + + it("pause the readyTimeout deadline and resume it with the budget left", async () => { + const store = await seeded(250); + // Each row takes 10 ms of the fake clock: 2.5 s of migration against a 1 s deadline. + const Edits = evolved(() => vi.advanceTimersByTime(10)); + const loading = loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + // The factory takes 600 ms of the budget; the native after the migration never opens. + vi.advanceTimersByTime(600); + self.open(hanging, () => {}); + return { n: constant(1) }; + }), + { store, readyTimeout: 1_000 }, + ); + await until(() => store.table(TABLE)?.version === 2 && store.table(TABLE)?.state === "ready"); + expect(Date.now()).toBe(3_100); + await vi.advanceTimersByTimeAsync(399); + await vi.advanceTimersByTimeAsync(1); + const { host, errors } = await loading; + expect(Date.now()).toBe(3_500); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + failure: { kind: "transient", error: { message: "app.store: did not become live within 1000 ms" } }, + }, + ]); + await host.close(context); + await store.close(context); + }); + + it("a timer firing during a long migration: another activation's deadline keeps its bound", async () => { + const store = await seeded(250); + const Edits = evolved(() => vi.advanceTimersByTime(10)); + // Both types have empty params: the host activates both at load, the start barrier waiting for both. + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open(hanging, () => {}); + return { n: constant(1) }; + }); + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }); + }, + { store, readyTimeout: 1_000 }, + ); + expect(Date.now()).toBe(2_500); + expect(errors).toMatchObject([{ kind: "activation_failed", object: "app.slow", failure: { kind: "transient" } }]); + expect((await host.get(Store, {}, context)).n.value).toBe(1); + await host.close(context); + await store.close(context); + }); + + it("a throwing rows fails the object broken, the table migrating; a retry resumes after the committed chunks", async () => { + const store = await seeded(250); + let crash = true; + const patched: string[] = []; + const Edits = evolved((id) => { + if (crash && id === "e150") throw new Error("crash"); + patched.push(id); + }); + const { host, errors } = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { store }, + ); + const held = host.hold(Store, {}, context); + await until(() => held.state.value.kind === "failed"); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { message: expect.stringContaining("crash") } }, + }); + expect(errors).toMatchObject([{ kind: "activation_failed", failure: { kind: "broken" } }]); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + crash = false; + patched.length = 0; + held.retry(); + await until(() => held.state.value.kind === "live"); + expect(patched).toHaveLength(150); + expect(patched[0]).toBe("e100"); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + held.release(); + await host.close(context); + await store.close(context); + }); +}); + +describe("parked attempts (dependency model §3)", () => { + fakeTimers(); + const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + /** A view with params: no host lifetime, so `load` does not wait for it. */ + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + + it("run no timer while parked but parked_long's: no dependency deadline, no backoff, the factory run once", async () => { + const factories = vi.fn(); + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + factories(); + return { n: self.get(Missing, {}).n }; + }), + ); + const view = host.hold(View, {}, context); + await flushed(); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.missing: app.missing is not served here" } }, + retryAt: undefined, + }); + // `parked_long`'s one timer fired (once, after readyTimeout); nothing else runs. + expect(vi.getTimerCount()).toBe(0); + expect([factories.mock.calls.length, errors]).toEqual([ + 1, + [{ kind: "parked_long", object: "app.view", waitingFor: "app.missing", state: "failed", since: 0 }], + ]); + view.release(); + }); + + it("pause the readyTimeout deadline while parked: the own steps get the budget left", async () => { + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open( + () => new Promise((resolve) => setTimeout(resolve, 5_000)), + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const slow = self.get(Slow, { id: 1 }); + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: slow.n }; + }); + }, + { readyTimeout: 10_000 }, + ); + const view = host.hold(Viewer, { id: 1 }, context); + // Parked 5 s on its dependency; then its own open hangs: it fails 10 s after the boundary, not 10 s after start. + await vi.advanceTimersByTimeAsync(5_000 + 9_999); + expect(failures(errors)).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(failures(errors)).toEqual(["app.viewer(1): transient"]); + view.release(); + }); + + it("go on past the dependency check: a dependency leaving live during the own steps changes nothing (R18)", async () => { + const Dependency = local("app.dependency", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + let dependencySelf: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Dependency, (_, self) => { + dependencySelf = self; + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const dependency = self.get(Dependency, {}); + self.open( + () => gate.promise, + () => {}, + ); + return { n: dependency.n }; + }); + }); + const view = host.hold(Viewer, { id: 1 }, context); + await flushed(); + // The view opens its own native; its dependency fails live meanwhile. + failLive(objects, dependencySelf!, new ChordError({ code: "unavailable", message: "offline" })); + await flushed(); + gate.resolve(); + await flushed(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(failures(errors)).toEqual([]); + view.release(); + }); +}); diff --git a/packages/chord/test/next/host/backoff.test.ts b/packages/chord/test/next/host/backoff.test.ts new file mode 100644 index 000000000..f53bdd426 --- /dev/null +++ b/packages/chord/test/next/host/backoff.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from "vitest"; +import { Backoff } from "../../../src/next/host/backoff.ts"; +import { fakeTimers } from "../support.ts"; + +/** The length of the window `failed` opens for `identity`, measured until `due` runs. */ +function window(backoff: Backoff, due: ReturnType, identity: string): number { + const calls = due.mock.calls.length; + const start = Date.now(); + backoff.failed(identity); + while (due.mock.calls.length === calls) vi.advanceTimersByTime(100); + return Date.now() - start; +} + +describe("backoff", () => { + fakeTimers(); + + it("opens a window of 1 s after the first failure, doubling up to 60 s, and calls due at its end", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + const windows = Array.from({ length: 9 }, () => window(backoff, due, "a")); + expect(windows).toEqual([1000, 2000, 4000, 8000, 16000, 32000, 60000, 60000, 60000]); + expect(due.mock.calls.every(([identity]) => identity === "a")).toBe(true); + }); + + it("tells when a running window ends: a failed reference's retryAt", () => { + const backoff = new Backoff(() => {}); + vi.advanceTimersByTime(5_000); + backoff.failed("a"); + expect([backoff.until("a"), backoff.until("b")]).toEqual([6_000, undefined]); + vi.advanceTimersByTime(1_000); + expect(backoff.until("a")).toBeUndefined(); + backoff.failed("a"); + expect(backoff.until("a")).toBe(8_000); + backoff.skip("a"); + expect(backoff.until("a")).toBeUndefined(); + }); + + it("is waiting only inside a window", () => { + const backoff = new Backoff(() => {}); + backoff.failed("a"); + expect([backoff.waiting("a"), backoff.waiting("b")]).toEqual([true, false]); + vi.advanceTimersByTime(1000); + expect(backoff.waiting("a")).toBe(false); + }); + + it("skip ends a window without due and keeps its delay; delete starts over", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.skip("a"); + vi.advanceTimersByTime(5000); + expect([backoff.waiting("a"), due.mock.calls.length]).toEqual([false, 0]); + expect(window(backoff, due, "a")).toBe(2000); + backoff.delete("a"); + expect(window(backoff, due, "a")).toBe(1000); + }); + + it("ended keeps a running window and drops a passed one", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.ended("a"); + expect(backoff.waiting("a")).toBe(true); + vi.advanceTimersByTime(1000); + backoff.ended("a"); + expect(window(backoff, due, "a")).toBe(1000); + }); + + it("a failure inside a running window replaces it: one due, after the doubled delay", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.failed("a"); + vi.advanceTimersByTime(1999); + expect([due.mock.calls.length, vi.getTimerCount()]).toEqual([0, 1]); + vi.advanceTimersByTime(1); + expect(due).toHaveBeenCalledTimes(1); + }); + + it("clear forgets every identity with its timer", () => { + const backoff = new Backoff(() => {}); + backoff.failed("a"); + backoff.failed("b"); + backoff.clear(); + expect([backoff.waiting("a"), backoff.waiting("b"), vi.getTimerCount()]).toEqual([false, false, 0]); + }); +}); diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts new file mode 100644 index 000000000..bf4744b76 --- /dev/null +++ b/packages/chord/test/next/host/ctx.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { withLinkedAbortSignal } from "../../../src/next/host/ctx.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { Ctx, Host, Reference } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + flushed, + incarnationOf, + linked, + loaded, + rawParent, + rejection, + Server, + type TestPrincipal, +} from "../support.ts"; + +const Item = local("app.item", { version: 1, params: { id: int() }, members: { id: int() } }); +const Desk = local("app.desk", { + version: 1, + params: {}, + members: { use: method(undefined, int()), leak: method(), late: method(), who: method(undefined, str()) }, +}); + +/** A desk whose handlers use `ctx.get`; what they got and their contexts are recorded. */ +async function desk() { + const seen: { ctx?: Ctx; item?: Reference; late?: Promise; factories: number } = { + factories: 0, + }; + const { host } = await loaded((h) => { + h.serve(Item, ({ id }) => { + seen.factories++; + return { id: constant(id) }; + }); + h.serve(Desk, () => ({ + use: async (ctx) => { + const item = await ctx.get(Item, { id: 1 }); + return item.id.value; + }, + leak: async (ctx) => { + seen.ctx = ctx; + seen.item = await ctx.get(Item, { id: 2 }); + return null; + }, + late: async (ctx) => { + seen.late = ctx.get(Item, { id: 3 }).catch((error: unknown) => error); + return null; + }, + who: async (ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`, + })); + }); + return { host, seen, desk: await host.get(Desk, {}, context) }; +} + +describe("ctx", () => { + it("ctx.get references are released when the handler settles", async () => { + const { desk: ref, seen } = await desk(); + expect(await ref.use(context)).toBe(1); + await ref.leak(context); + expect(() => seen.item!.id.value).toThrow("app.item: released"); + // The object ended with its last reference: a new get creates it anew. + await ref.use(context); + expect(seen.factories).toBe(3); + }); + + it("ctx.get after the handler settled throws released", async () => { + const { desk: ref, seen } = await desk(); + await ref.leak(context); + expect(await rejection(seen.ctx!.get(Item, { id: 1 }))).toEqual({ + code: "released", + reason: "settled", + message: "app.item: ctx.get after its call settled", + }); + }); + + it("ctx.get of a handler whose call was aborted checks the signal first: nothing is created", async () => { + let factories = 0; + let got: Promise | undefined; + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => { + h.serve(Item, ({ id }) => { + factories++; + return { id: constant(id) }; + }); + h.serve(Desk, () => ({ + use: async (ctx) => { + await gate.promise; + got = ctx.get(Item, { id: 1 }).catch((error: unknown) => error); + return 0; + }, + leak: async () => null, + late: async () => null, + who: async () => "", + })); + }); + const ref = await host.get(Desk, {}, context); + const caller = new AbortController(); + const call = ref.use(withAbortSignal(caller.signal, context)).catch(() => {}); + await flushed(); + caller.abort(new Error("caller gave up")); + gate.resolve(); + await call; + await flushed(); + expect(await got).toEqual(new Error("caller gave up")); + expect(factories).toBe(0); + }); + + it("a ctx.get finishing after its handler settled is released at once (race 11)", async () => { + const { desk: ref, seen } = await desk(); + await ref.late(context); + expect(await seen.late).toMatchObject({ + code: "released", + reason: "settled", + message: "app.item: ctx.get after its call settled", + }); + await flushed(); + // The late reference's object ended with it. + expect(seen.factories).toBe(1); + }); + + it("a ctx.get still waiting when its handler settles gives up: its reference is released at once", async () => { + const Far = remote("app.far", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const Starter = local("app.starter", { version: 1, params: {}, members: { start: method() } }); + let pending: Promise | undefined; + const { host } = await loaded((h) => + h.serve(Starter, () => ({ + start: async (ctx) => { + pending = ctx.get(Far, {}).catch((error: unknown) => error); + return null; + }, + })), + ); + // A host of any principal type takes a parent link. + const parent = await rawParent(host as Host); + await (await host.get(Starter, {}, context)).start(context); + const { hold } = await parent.next("hold"); + expect(await parent.next("release")).toMatchObject({ hold }); + expect(await pending).toMatchObject({ + code: "released", + reason: "settled", + message: "app.far: ctx.get after its call settled", + }); + }); + + it("a local handler's ctx.principal is the host's, its incarnation this host's", async () => { + const { desk: ref, host } = await desk(); + expect(await ref.who(context)).toBe(`none ${(host as ChordHost).incarnation}`); + }); +}); + +describe("local handlers act as the host (§4, audit)", () => { + it("a local object called from a remote handler's ctx.get runs with this host's principal and incarnation", async () => { + const Store = local("app.local-store", { version: 1, params: {}, members: { who: method(undefined, str()) } }); + const Gate = remote("app.gate", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { methods: { who: method(undefined, str()), through: method(undefined, str()) } } }, + }); + const who = (ctx: Ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`; + const { server, client } = await linked((h) => { + h.serve(Store, () => ({ who: async (ctx) => who(ctx) })); + h.serve(Gate, () => ({ + variant: "v", + members: { + who: async (ctx) => who(ctx), + through: async (ctx) => (await ctx.get(Store, {})).who(ctx), + }, + })); + }); + const gate = await client.get(Gate, {}, context); + // The remote handler acts as its caller; the local object it reaches acts as this host, both halves. + expect(await gate.who(context)).toBe(`owner ${incarnationOf(client)}`); + expect(await gate.through(context)).toBe(`server ${incarnationOf(server)}`); + }); +}); + +describe("linked abort signals", () => { + it("abort with the caller's signal or the call's, the caller's reason first", () => { + for (const first of ["caller", "call"] as const) { + const [caller, call] = [new AbortController(), new AbortController()]; + const { context: linked } = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + (first === "caller" ? caller : call).abort(first); + expect(linked.abortSignal?.reason).toBe(first); + } + const [caller, call] = [new AbortController(), new AbortController()]; + call.abort("call"); + caller.abort("caller"); + const { context: linked } = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + expect(linked.abortSignal?.reason).toBe("caller"); + }); + + it("detach from both sources on release: a later abort reaches nothing", () => { + const [caller, call] = [new AbortController(), new AbortController()]; + const linked = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + linked.release(); + caller.abort("late"); + call.abort("late"); + expect(linked.context.abortSignal?.aborted).toBe(false); + }); + + it("is the call's own signal without a caller's", () => { + const call = new AbortController(); + const linked = withLinkedAbortSignal(call.signal, context); + expect(linked.context.abortSignal).toBe(call.signal); + linked.release(); + }); +}); diff --git a/packages/chord/test/next/host/deadline.test.ts b/packages/chord/test/next/host/deadline.test.ts new file mode 100644 index 000000000..23b6e5093 --- /dev/null +++ b/packages/chord/test/next/host/deadline.test.ts @@ -0,0 +1,42 @@ +// Tests of an activation's deadline (slice 6a §9): it expires once after its budget; paused time does not count; a +// cleared deadline never expires. + +import { describe, expect, it, vi } from "vitest"; +import { Deadline } from "../../../src/next/host/deadline.ts"; +import { fakeTimers } from "../support.ts"; + +describe("deadline", () => { + fakeTimers(); + + it("expires once after its budget", () => { + const expired = vi.fn(); + new Deadline(100, expired); + vi.advanceTimersByTime(99); + expect(expired).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1_000); + expect(expired).toHaveBeenCalledTimes(1); + }); + + it("paused time does not count: resumed with the budget it had left", () => { + const expired = vi.fn(); + const deadline = new Deadline(100, expired); + vi.advanceTimersByTime(60); + deadline.pause(); + vi.advanceTimersByTime(1_000); + deadline.resume(); + vi.advanceTimersByTime(39); + expect(expired).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(expired).toHaveBeenCalledTimes(1); + }); + + it("a cleared deadline never expires, also when resumed", () => { + const expired = vi.fn(); + const deadline = new Deadline(100, expired); + deadline.pause(); + deadline.clear(); + deadline.resume(); + vi.advanceTimersByTime(1_000); + expect(expired).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/chord/test/next/host/dependencies.test.ts b/packages/chord/test/next/host/dependencies.test.ts new file mode 100644 index 000000000..afa5d914b --- /dev/null +++ b/packages/chord/test/next/host/dependencies.test.ts @@ -0,0 +1,385 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, keyed, local, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { + bindHold, + failures, + fakeTimers, + flushed, + loaded, + Notes, + rawParent, + rejection, + relayed, + Server, + Session, + statesOf, + type TestPrincipal, + transported, +} from "../support.ts"; + +const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + +describe("the dependency check", () => { + it("breaks a dependent off a cycle with the cycle's error: it would wait on a ring that never heals", async () => { + const Ping = local("app.ping", { version: 1, params: {}, members: { n: int() } }); + const Pong = local("app.pong", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Ping, (_, self) => ({ n: self.get(Pong, {}).n })); + h.serve(Pong, (_, self) => ({ n: self.get(Ping, {}).n })); + h.serve(View, (_, self) => ({ n: self.get(Ping, {}).n })); + }); + const view = host.hold(View, { id: 1 }, context); + await flushed(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "invalid", reason: "cycle" } }, + }); + expect(failures(errors)).toContain("app.view(1): broken"); + view.release(); + }); + + it("breaks a dependent whose dependency ended (gone): the request cannot come back", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + await client.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const revoked = { code: "forbidden", reason: "revoked", message: "revoked" } as const; + peer.send({ kind: "gone", hold, error: { kind: "chord", error: revoked } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: revoked } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("goes live once every dependency is: one parked on two, the second binding last", async () => { + const client = createHost({ onError: () => {} }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + const a = self.get(Notes, { session: "S1" }).count; + const b = self.get(Notes, { session: "S2" }).count; + return { n: self.derive([a, b], ([x, y]) => x + y) }; + }), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const first = await peer.next("hold"); + const second = await peer.next("hold"); + bindHold(peer, first.hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "loading" }); + bindHold(peer, second.hold, [{ text: "b" }, { text: "c" }]); + await transported(); + expect([view.state.value, view.n.value]).toEqual([{ kind: "live" }, 3]); + view.release(); + }); + + it("breaks a dependent whose dependency a relay's authorize refused: forbidden heals only by retry() (fix round 1, Fable)", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const forbidden = { code: "forbidden", message: "session|S1: not authorized" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error: forbidden } } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: forbidden } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("settles a waiting ring across hosts after one round: the root cause kept (fix round 1, Opus F1)", async () => { + const A = remote("app.a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const B = remote("app.b", { + version: 1, + params: {}, + hostAddress: () => Session({ id: "S1" }), + variants: { v: { state: { n: int() } } }, + }); + const X = local("app.x", { version: 1, params: {}, members: { n: int() } }); + const Kick = local("app.kick", { version: 1, params: {}, members: { n: int() } }); + const Front = local("app.front", { version: 1, params: {}, members: { n: int() } }); + const Back = local("app.back", { version: 1, params: {}, members: { n: int() } }); + const { workers } = await relayed({ + trusted: true, + server: (h) => { + // The server's host object starts the worker; its A and the worker's B then start in the same turns, so + // the activation chains miss the ring (M§7's gap). + h.serve(Kick, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(A, (_p, _principal, self) => ({ variant: "v", members: { n: self.get(B, {}).n } })); + }, + worker: (h) => { + h.serve(Front, (_p, self) => ({ n: self.get(A, {}).n })); + h.serve(Back, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(B, (_p, _principal, self) => { + const a = self.get(A, {}); + self.get(X, {}); + return { variant: "v", members: { n: a.n } }; + }); + }, + }); + await transported(); + const worker = workers.get("session|S1")!; + const b = worker.hold(B, {}, context); + for (let round = 0; round < 50; round++) await transported(); + const states = statesOf(await b.state.watch(context)); + for (let round = 0; round < 50; round++) await transported(); + expect(states).toEqual([]); + expect(b.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: expect.stringMatching(/: app\.x is not served here$/) } }, + }); + b.release(); + }); +}); + +describe("parked_long (dependency model §7, audit A11)", () => { + fakeTimers(); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + + it("is reported once after readyTimeout whatever the blocker: a missing local provider; nothing fails", async () => { + const { host, errors } = await loaded((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n })), { + readyTimeout: 1_000, + }); + const view = host.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(999); + expect(errors).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.missing", state: "failed", since: 0 }, + ]); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toHaveLength(1); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await host.close(context); + }); + + it("names the first blocker in declaration order with its state; its clock runs from parking, with no link yet too", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ + n: self.derive([self.get(Notes, { session: "S1" }).count, self.get(Missing, {}).n], ([a, b]) => a + b), + })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(1_000); + // The remote one comes first (no link holds it yet), though the attempt shows the failed one. + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", state: "loading", since: 0 }, + ]); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await client.close(context); + }); + + it("keeps declaration order across registries and references: the parked cause and the report name the first (Astra)", async () => { + const Registry = defineRegistry("app.missing-registry", { version: 1 }); + const { host, errors } = await loaded( + (h) => + h.serve(View, (_, self) => { + self.registry(Registry); + return { n: self.get(Missing, {}).n }; + }), + { readyTimeout: 1_000 }, + ); + const view = host.hold(View, { id: 1 }, context); + await flushed(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { message: expect.stringMatching(/^waiting for registry app\.missing-registry: /) }, + }, + }); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { + kind: "parked_long", + object: "app.view(1)", + waitingFor: "registry app.missing-registry", + state: "failed", + since: 0, + }, + ]); + view.release(); + await host.close(context); + }); + + it("names a reference whose pinned key still loads: the episode's clock includes the load (Sol)", async () => { + const Rows = defineState("rows", { + version: 1, + keyed: true, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + }); + const Keys = local("app.keys", { version: 1, params: {}, members: { rows: keyed(obj({ n: int() })) } }); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const pinKey = ChordObject.prototype.pinKey; + const held = vi.spyOn(ChordObject.prototype, "pinKey").mockImplementation(async function ( + this: ChordObject, + member, + key, + version, + pinContext, + ) { + entered.resolve(); + await gate.promise; + return pinKey.call(this, member, key, version, pinContext); + }); + try { + const { host, errors } = await loaded( + (h) => { + h.serve(Keys, (_, self) => ({ rows: self.state(Rows) })); + h.serve(View, (_, self) => { + const row = self.get(Keys, {}).rows.at("a"); + return { n: self.derive([row], ([value]) => value.n) }; + }); + }, + { readyTimeout: 1_000 }, + ); + const view = host.hold(View, { id: 1 }, context); + await entered.promise; + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.keys", state: "loading", since: 0 }, + ]); + gate.resolve(); + await vi.waitFor(() => expect(view.state.value).toEqual({ kind: "live" })); + view.release(); + await host.close(context); + } finally { + gate.resolve(); + held.mockRestore(); + } + }); + + it("reports a ring across hosts whose member there waits on a failed dependency (it shows failed here)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + // The provider's object is parked there on a failed dependency (the ring's member back to this host waits too). + const error = { + kind: "chord", + error: { code: "unavailable", message: "waiting for app.c: not served" }, + } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", state: "failed", since: 0 }, + ]); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await client.close(context); + }); + + it("is reported once per parking episode: again for an attempt that parks after leaving parking", async () => { + const { host, errors } = await loaded((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n })), { + readyTimeout: 1_000, + }); + const view = host.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + expect(errors.map(({ kind }) => kind)).toEqual(["parked_long"]); + // A reload of its plugin replaces the parked attempt: a new episode, its clock from its own parking. + const changed = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n }))); + await host.reload([{ name: "@test/app", facet: changed }], context); + await vi.advanceTimersByTimeAsync(999); + expect(errors).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(errors[1]).toEqual({ + kind: "parked_long", + object: "app.view(1)", + waitingFor: "app.missing", + state: "failed", + since: 5_000, + }); + view.release(); + await host.close(context); + }); + + it("is not reported for an attempt that leaves parking within readyTimeout: its timer stops", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + await vi.advanceTimersByTimeAsync(500); + bindHold(peer, hold, [{ text: "a" }]); + await vi.advanceTimersByTimeAsync(5_000); + expect(view.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + view.release(); + await client.close(context); + }); +}); + +describe("a getter of a parked object (fix round 1, Astra 6)", () => { + it("rejects with the last parked cause after its dependency fell back to loading, the newest of several", async () => { + const host = createHost({ onError: () => {} }); + const peer = await rawParent(host); + await host.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const pending = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 80 })); + const { hold } = await peer.next("hold"); + for (const message of ["first absence", "specific provider absence"]) { + const error = { kind: "chord", error: { code: "unavailable", message } } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await transported(); + } + peer.send({ kind: "loading", hold }); + await transported(); + expect(await pending).toMatchObject({ + code: "unavailable", + message: "waiting for app.notes(S1): specific provider absence", + }); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/facet.test.ts b/packages/chord/test/next/host/facet.test.ts new file mode 100644 index 000000000..619cea025 --- /dev/null +++ b/packages/chord/test/next/host/facet.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { defineFacet, type Facet, type FacetHost, type FacetSelf, type Host } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { loaded } from "../support.ts"; + +const Info = local("app.info", { version: 1, params: {}, members: { n: int() } }); + +describe("facets", () => { + it("declare only while their setup runs", async () => { + let later: FacetHost | undefined; + await loaded((host) => { + later = host; + }); + expect(() => later!.serve(Info, () => ({ n: constant(1) }))).toThrow( + "@test/app: host.serve is valid only in the facet's setup", + ); + }); + + it("only declare: a setup takes the host and a self that only contributes; no documents, routines or natives", () => { + expectTypeOf["setup"]>>().toEqualTypeOf< + [FacetHost<{ kind: "owner" }>, FacetSelf] + >(); + expectTypeOf().toEqualTypeOf<"contribute">(); + expectTypeOf().not.toHaveProperty("state"); + expectTypeOf().not.toHaveProperty("on"); + expectTypeOf().not.toHaveProperty("onLive"); + }); + + it("contribute at most one value per registry, only while their setup runs", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const twice = await loaded((_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + }); + expect(twice.failed.map(({ error }) => (error as Error).message)).toEqual([ + 'plugin "@test/app": @test/app: registry app.names is contributed to twice', + ]); + await twice.host.close(context); + let later: FacetSelf | undefined; + const { host } = await loaded((host, self) => { + host.serve(Names, { compose: (table) => table.length }); + later = self; + }); + expect(() => later!.contribute(Names, "late")).toThrow( + "@test/app: self.contribute is valid only in the facet's setup", + ); + await host.close(context); + }); + + it("are typed by exactly the app's principal type", () => { + type App = { kind: "owner" } | { kind: "guest"; session: string }; + const facet = defineFacet(() => {}); + expectTypeOf(facet).toEqualTypeOf>(); + type Load = Host["load"]; + expectTypeOf[0]>().toEqualTypeOf< + readonly { readonly name: string; readonly facet: Facet }[] + >(); + expectTypeOf>().toExtend>(); + expectTypeOf>().not.toExtend>(); + expectTypeOf>().not.toExtend>(); + }); +}); diff --git a/packages/chord/test/next/host/follow.test.ts b/packages/chord/test/next/host/follow.test.ts new file mode 100644 index 000000000..de225e098 --- /dev/null +++ b/packages/chord/test/next/host/follow.test.ts @@ -0,0 +1,345 @@ +// Tests of `self.follow` (slice 6b §3, D9, R4): a source over another runtime's synchronous observer, attached during +// activation, its calls marked in the running batch or grouped into one batch outside it, its ends. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { int, local, obj } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { FollowSource } from "../../../src/next/host/follow.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { FollowAttach, FollowObserver } from "../../../src/next/host/index.ts"; +import { Batch } from "../../../src/next/state/batch.ts"; +import type { Watchable, WatchEnd } from "../../../src/watch.ts"; +import { changesOf, delivered, expectCollected, instancesOf, loaded, valuesOf } from "../support.ts"; + +const followed = instancesOf(FollowSource, "attach"); + +type Count = { readonly n: number }; + +/** A runtime with synchronous observers, as Durable's views have them. */ +class Runtime { + value: Count = { n: 0 }; + readonly observers = new Set>(); + attachments = 0; + detached = 0; + /** Resolves the next attachment when set (an attachment that takes time). */ + gate: Promise | undefined; + absent = false; + + readonly attach: FollowAttach = async (observer) => { + this.observers.add(observer); + this.attachments++; + await this.gate; + if (this.absent) return undefined; + return { + value: this.value, + detach: () => { + this.observers.delete(observer); + this.detached++; + }, + }; + }; + + /** One change: every observer advanced with its ops. */ + set(n: number, retired = false): void { + const ops: Op[] = [["s", ["n"], n]]; + this.value = { n }; + for (const observer of this.observers) + observer.advance({ kind: "value", value: this.value, ops, retired }, context); + } + + end(end: WatchEnd): void { + for (const observer of [...this.observers]) observer.end(end); + this.observers.clear(); + } +} + +const View = local("app.view", { version: 1, params: {}, members: { a: obj({ n: int() }), sum: int() } }); + +/** Serves `View`, following `a` and `b`, its `sum` a derive over both (its computes counted). */ +async function served(a = new Runtime(), b = new Runtime()) { + let computes = 0; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + const right = self.follow(b.attach); + const sum = self.derive([left, right], ([x, y]) => { + computes++; + return x.n + y.n; + }); + return { a: left, sum }; + }), + ); + const view = await host.get(View, {}, context); + const batch = (host as unknown as ChordHost).batch; + return { host, errors, view, a, b, batch, computes: () => computes }; +} + +describe("calls", () => { + it("inside a running batch: one frame per commit, and a derive over two views computes once with both", async () => { + const { view, a, b, batch, computes } = await served(); + const calls = changesOf(await view.a.watch(context)); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + batch.commit(() => { + a.set(1); + a.set(2); + b.set(10); + }, context); + await delivered(); + expect(calls).toEqual([ + { + kind: "value", + value: { n: 2 }, + ops: [ + ["s", ["n"], 1], + ["s", ["n"], 2], + ], + retired: false, + }, + ]); + expect(valuesOf(sums)).toEqual([12]); + expect(computes() - before).toBe(1); + }); + + it("outside any batch: queued and applied in one batch on the next microtask", async () => { + const { view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1); + b.set(2); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + expect(view.a.value).toEqual({ n: 1 }); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + }); + + it("before the attachment resolved: applied in order on its value", async () => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + const serving = served(a); + await delivered(); + a.set(5); + a.set(6); + gate.resolve(); + const { view } = await serving; + expect(view.a.value).toEqual({ n: 6 }); + }); +}); + +describe("types", () => { + it("infers the value type from the attachment, without a type argument", async () => { + const runtime = new Runtime(); + // A runtime's own observe, as Durable's `observeConversation` is: typed by its value, not by Chord's observer. + const observe = (observer: FollowObserver) => runtime.attach(observer, context); + await loaded((h) => + h.serve(View, (_, self) => { + const a = self.follow((observer) => observe(observer)); + expectTypeOf(a).toEqualTypeOf>(); + return { a, sum: self.derive([a], ([x]) => x.n) }; + }), + ); + }); +}); + +describe("activation", () => { + it("an absent address fails the activation transient; a failed activation detaches what attached", async () => { + const a = new Runtime(); + const b = new Runtime(); + b.absent = true; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => ({ a: self.follow(a.attach), sum: self.derive([self.follow(b.attach)], () => 0) })), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow("followed address is absent"); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await delivered(); + expect([a.attachments, a.detached]).toEqual([a.attachments, a.attachments]); + await host.close(context); + }); +}); + +describe("ends (R4)", () => { + it("failed: the live object fails transient once, reported, and its retry attaches again", async () => { + const { host, errors, view, a, b } = await served(); + a.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + b.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + expect((view.state.value as { retryAt?: number }).retryAt).toBeGreaterThan(0); + expect(errors.filter((error) => error.kind === "object_failed")).toHaveLength(1); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + view.retry(); + await delivered(); + await delivered(); + expect(view.state.value).toEqual({ kind: "live" }); + expect([a.attachments, a.detached, b.attachments]).toEqual([2, 1, 2]); + a.set(3); + await delivered(); + expect(view.a.value).toEqual({ n: 3 }); + await host.close(context); + }); + + it("ended: final; a queued retirement reaches the watch first, the value stays, a new watch ends at once", async () => { + const { host, errors, view, a } = await served(); + const watch = await view.a.watch(context); + const calls = changesOf(watch); + a.set(4, true); + a.end({ kind: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toMatchObject([{ kind: "value", value: { n: 4 }, retired: true }]); + expect(view.a.value).toEqual({ n: 4 }); + const late = await view.a.watch(context); + expect(late.value).toEqual({ n: 4 }); + expect(await late.closed).toEqual({ kind: "ended" }); + expect(errors).toEqual([]); + expect(view.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("ends during activation and in one batch (review regressions)", () => { + it("an end before the attachment resolved keeps the attachment's value on the source", async () => { + const source = new FollowSource( + async (observer) => { + observer.end({ kind: "ended" }); + return { value: { n: 7 }, detach() {} }; + }, + { label: "early end", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + expect(source.value).toEqual({ n: 7 }); + }); + + it.each([ + ["ended", { kind: "ended" } as const], + ["failed", { kind: "failed", error: new Error("early failure") } as const], + ])("an %s end before the attachment resolved fails the activation transient", async (_, end) => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + // Loading activates the object (local, empty params): its follow attaches meanwhile. + const loading = loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + return { a: left, sum: self.derive([left], ([x]) => x.n) }; + }), + ); + await delivered(); + a.end(end); + gate.resolve(); + const { host, errors } = await loading; + await delivered(); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + expect(a.detached).toBe(a.attachments); + }); + + it("an end while the object activates (after its sources attached) fails the activation transient", async () => { + const a = new Runtime(); + let ended = false; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + // The activation computes the derives after the follows attached: the runtime ends then. + const sum = self.derive([left], ([x]) => { + if (!ended) { + ended = true; + a.end({ kind: "ended" }); + } + return x.n; + }); + return { a: left, sum }; + }), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow(); + expect(ended).toBe(true); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + }); + + it("a retirement and an end inside one batch: the watch gets the retirement, then ends", async () => { + const batch = new Batch(() => {}, undefined); + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 7 }, detach() {} }; + }, + { label: "retirement", context, ended() {} }, + batch, + ); + await source.attach(context); + const watch = await source.watch(context); + const calls = changesOf(watch); + batch.commit(() => { + observer!.advance({ kind: "value", value: { n: 7 }, ops: [], retired: true }, context); + observer!.end({ kind: "ended" }); + }, context); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toEqual([{ kind: "value", value: { n: 7 }, ops: [], retired: true }]); + }); + + it("a call and an end outside a batch with nothing watching: a later watch opens at the value and ends", async () => { + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 0 }, detach() {} }; + }, + { label: "unwatched", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + observer!.advance({ kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: true }, context); + observer!.end({ kind: "ended" }); + await delivered(); + const watch = await source.watch(context); + expect(watch.value).toEqual({ n: 1 }); + expect(await watch.closed).toEqual({ kind: "ended" }); + source.detach(); + }); + + it("an end outside a batch after calls of two views: the views of one commit are delivered together", async () => { + const { host, view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1, true); + b.set(2); + a.end({ kind: "ended" }); + await delivered(); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + await host.close(context); + }); +}); + +describe("GC", () => { + it("a source its runtime ended is still detached at the object's end, then collected", async () => { + const a = new Runtime(); + const { host } = await served(a); + a.end({ kind: "ended" }); + await delivered(); + expect(a.detached).toBe(0); + await host.close(context); + expect(a.detached).toBe(1); + await expectCollected(...followed.splice(0)); + }); + + it("an ended object's followed sources are collected, and their observers detached", async () => { + const a = new Runtime(); + const { host } = await served(a); + await host.close(context); + expect(a.observers.size).toBe(0); + await expectCollected(...followed.splice(0)); + }); +}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts new file mode 100644 index 000000000..0ae2d5361 --- /dev/null +++ b/packages/chord/test/next/host/host.test.ts @@ -0,0 +1,1719 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { + type Ctx, + createHost, + defineFacet, + type FacetHost, + type HostError, + type Self, +} from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { + failures, + fakeTimers, + faultyStore, + flushed, + forEachCodec, + linked, + loaded, + Notes, + PlacedNotes, + partValue, + RawPeer, + rejection, + relayed, + SERVER_ADDRESS, + Server, + Session, + serveNotes, + statesOf, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +const Info = local("app.info", { version: 1, params: {}, members: { title: str() } }); +const Counter = local("app.counter", { version: 1, params: {}, members: { count: int(), bump: method() } }); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { id: int() } }); +const Svc = local("app.svc", { version: 1, params: { id: int() }, members: { n: int() } }); +const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + +const noErrors = { onError: () => {} }; + +/** Serves `Svc`: one native whose open waits for `gate` and whose close records its value and reason in `closes`. */ +function serveGated(h: FacetHost, gate: Promise, closes: string[]): void { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + await gate; + return `svc ${id}`; + }, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + return { n: constant(id) }; + }); +} + +describe("load", () => { + it("runs every setup and activates the types with empty params before it resolves", async () => { + const factories = { info: vi.fn(() => ({ title: constant("t") })), page: vi.fn() }; + const facet = defineFacet((host) => { + host.serve(Info, factories.info); + host.serve(Page, factories.page); + }); + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet }], context); + expect(factories.info).toHaveBeenCalledTimes(1); + expect(factories.page).not.toHaveBeenCalled(); + await host.close(context); + }); + + it("leaves out and reports invalid and duplicate plugin names and double serves (the later); refuses a second load", async () => { + const empty = defineFacet(() => {}); + const info = defineFacet((host) => host.serve(Info, () => ({ title: constant("t") }))); + const loaded = async (facets: { readonly name: string; readonly facet: typeof empty }[]) => { + const host = createHost(noErrors); + const { failed } = await host.load(facets, context); + await host.close(context); + return failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await loaded([{ name: "Bad Name", facet: empty }])).toEqual([ + ["Bad Name", '"Bad Name" is not a valid plugin name'], + ]); + const twice = [ + { name: "a", facet: empty }, + { name: "a", facet: empty }, + ]; + expect(await loaded(twice)).toEqual([["a", 'plugin "a" is loaded twice']]); + const throwing = defineFacet(() => { + throw new Error("first failed"); + }); + // The first one's setup failing does not make the second one's name free (Sol). + const failing = createHost(noErrors); + const report = await failing.load( + [ + { name: "a", facet: throwing }, + { name: "a", facet: info }, + ], + context, + ); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["a", 'plugin "a": first failed'], + ["a", 'plugin "a" is loaded twice'], + ]); + expect(failing.hold(Info, {}, context).state.value).toMatchObject({ kind: "failed" }); + await failing.close(context); + const served = [ + { name: "a", facet: info }, + { name: "b", facet: info }, + ]; + expect(await loaded(served)).toEqual([["b", 'plugin "b": app.info is served twice (plugin "a")']]); + const host = createHost(noErrors); + await host.load([], context); + await expect(host.load([], context)).rejects.toThrow("load runs once per host"); + }); + + it("makes a get before it resolves wait for it", async () => { + const host = createHost(noErrors); + const got = host.get(Info, {}, context); + await host.load( + [{ name: "a", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + expect((await got).title.value).toBe("t"); + }); +}); + +describe("the start barrier", () => { + fakeTimers(); + + it("resolves once every host-lifetime object had its first outcome; a failed one is reported and retried", async () => { + let attempts = 0; + const { host, errors } = await loaded((h) => { + h.serve(Info, () => ({ title: constant("t") })); + h.serve(Counter, (_, self) => { + const attempt = ++attempts; + self.open( + async () => { + if (attempt === 1) throw new Error("offline"); + return null; + }, + () => {}, + ); + return { count: constant(attempt), bump: async () => null }; + }); + }); + expect(failures(errors)).toEqual(["app.counter: transient"]); + const counter = host.get(Counter, {}, context); + await vi.advanceTimersByTimeAsync(1000); + expect((await counter).count.value).toBe(2); + }); + + it("is not held by a hanging open: its attempt is abandoned at readyTimeout", async () => { + const host = createHost({ ...noErrors, readyTimeout: 100 }); + const facet = defineFacet((h) => + h.serve(Info, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { title: constant("t") }; + }), + ); + let done = false; + void host.load([{ name: "a", facet }], context).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(99); + expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(done).toBe(true); + }); + + it("waits for a local blocker's first outcome: a host object parked on another one here (CodeMode → Harness)", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const opened = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Harness, (_, self) => { + self.open( + () => opened.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + opened.resolve(); + await loading; + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "live" }); + }); + + it("ends its pass at a local blocker's own failure: the dependent parked failed { waiting } has its first outcome", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Harness, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + expect(failures(errors)).toEqual(["app.harness: transient"]); + expect(host.hold(CodeMode, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.harness: app.harness: offline" } }, + }); + }); + + it("ends its pass at a remote blocker: a wait on another host is a first outcome", async () => { + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const host = createHost(noErrors); + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(CodeMode, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await host.load([{ name: "@app/codemode", facet }], context); + expect(await parent.next("hold")).toMatchObject({ type: Notes.name }); + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "loading" }); + }); + + it("is settled by host.close: an object of its cohort that ends before its first outcome settles it as ended", async () => { + const host = createHost(noErrors); + const facet = defineFacet((h) => + h.serve(Info, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { title: constant("t") }; + }), + ); + let done = false; + const loading = host.load([{ name: "a", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + await host.close(context); + await loading; + expect(done).toBe(true); + }); + + it("waits for a local blocker parked failed { waiting } on one still in its first attempt (fix round 1, Sol 4)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const Middle = local("app.middle", { version: 1, params: { id: int() }, members: { n: int() } }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Middle, ({ id }, self) => { + const slow = self.get(Slow, { id }); + self.get(Missing, {}); + return { n: slow.n }; + }); + h.serve(Core, (_, self) => ({ n: self.get(Middle, { id: 1 }).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(host.hold(Middle, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(done).toBe(false); + gate.resolve(); + await loading; + await host.close(context); + }); + + it("waits for it whichever it declares first (fix round 1, Astra 3)", async () => { + const A = local("app.a", { version: 1, params: {}, members: {} }); + const B = local("app.b", { version: 1, params: { id: int() }, members: {} }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(A, (_, self) => { + self.get(B, { id: 1 }); + return {}; + }); + h.serve(B, (_, self) => { + self.get(Missing, {}); + self.get(Slow, { id: 1 }); + return {}; + }); + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return {}; + }); + }); + let done = false; + const loading = host.load([{ name: "@app/a", facet }], context).then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + await host.close(context); + expect(premature).toBe(false); + }); +}); + +describe("startup across hosts (dependency model §6)", () => { + // Timers never advance: nothing here may wait for a deadline. + fakeTimers(); + + /** On the server, waiting for a worker's `Agent`. */ + const Bootstrap = local("app.bootstrap", { version: 1, params: {}, members: { n: int() } }); + /** On worker S1, waiting for the server's `Catalog`. */ + const Agent = remote("app.agent", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const Catalog = remote("app.catalog", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + + it("starts two hosts that would have deadlocked (Sol's probe): an acyclic graph needs no timeout", async () => { + const { server, errors } = await relayed({ + server: (h) => { + h.serve(Bootstrap, (_, self) => ({ n: self.get(Agent, { session: "S1" }).n })); + h.serve(Catalog, () => ({ variant: "v", members: { n: constant(7) } })); + }, + worker: (h) => { + // The worker's own host object waits for the server's catalog during the worker's load. + h.serve(Agent, (_params, _principal, self) => ({ variant: "v", members: { n: self.get(Catalog, {}).n } })); + }, + }); + await transported(); + await transported(); + const bootstrap = server.hold(Bootstrap, {}, context); + expect(bootstrap.state.value).toEqual({ kind: "live" }); + expect(bootstrap.n.value).toBe(7); + expect([errors.server, errors.worker]).toEqual([[], []]); + }); + + it("places a hold arriving before the publication here, not over the parent", async () => { + const worker = createHost({ + address: Session({ id: "S1" }), + principal: { kind: "worker", session: "S1" }, + ...noErrors, + }); + const [a, b] = memoryLinkPair(); + worker.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const [c, d] = memoryLinkPair(); + worker.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const child = new RawPeer(d); + child.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + }); + const mine = worker.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(mine.state.value).toEqual({ kind: "loading" }); + await worker.load( + [{ name: "@test/w", facet: defineFacet((h) => serveNotes(h, PlacedNotes)) }], + context, + ); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + await transported(); + expect(mine.state.value).toEqual({ kind: "live" }); + expect(parent.kinds()).toEqual(["hello"]); + }); + + it("wakes a parked chain over three hosts in order, its causes passed on (tab → worker → server)", async () => { + const Chat = remote("app.chat", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const order: string[] = []; + let catalogs = 0; + const { tab } = await relayed({ + server: (h) => + h.serve(Catalog, (_params, _principal, self) => { + const attempt = ++catalogs; + self.open( + async () => { + if (attempt === 1) throw new Error("offline"); + }, + () => {}, + ); + self.onLive(async () => void order.push("catalog")); + return { variant: "v", members: { n: constant(attempt) } }; + }), + worker: (h) => + h.serve(Chat, (_params, _principal, self) => { + const catalog = self.get(Catalog, {}); + self.onLive(async () => void order.push("chat")); + return { variant: "v", members: { n: catalog.n } }; + }), + }); + const view = tab.hold(Chat, { session: "S1" }, context); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.catalog: app.catalog: offline" } }, + retryAt: undefined, + }); + // The catalog's own backoff (1 s) is the only timer: it retries, and the chain goes on in order. + await vi.advanceTimersByTimeAsync(1_000); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(order).toEqual(["catalog", "chat"]); + expect(view.n.value).toBe(2); + }); +}); + +describe("get", () => { + it("waits for an unserved type and a contract mismatch until its readyTimeout, then rejects with the cause", async () => { + const { host } = await loaded((h) => h.serve(Info, () => ({ title: constant("t") }))); + const quick = { readyTimeout: 5 }; + expect(await rejection(host.get(Page, { id: 1 }, context, quick))).toEqual({ + code: "unavailable", + message: "app.page is not served here", + }); + const InfoV2 = local("app.info", { version: 2, params: {}, members: { title: str() } }); + const Changed = local("app.info", { version: 1, params: {}, members: { title: int() } }); + expect(await rejection(host.get(InfoV2, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(Changed, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); + const Same = local("app.info", { version: 1, params: {}, members: { title: str() } }); + expect((await host.get(Same, {}, context)).title.value).toBe("t"); + }); + + it("checks params with L1 (invalid/params)", async () => { + const { host } = await loaded((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))); + // @ts-expect-error: a string is not an int + expect(await rejection(host.get(Page, { id: "x" }, context))).toMatchObject({ + code: "invalid", + reason: "params", + }); + }); +}); + +describe("deadlines of gets", () => { + fakeTimers(); + + it("reject with unavailable while no attempt has failed (the last failure: see attempt.test.ts)", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serveGated(h, gate.promise, [])); + const waiting = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + expect(await waiting).toEqual({ code: "unavailable", message: "app.svc(1): did not become live within 500 ms" }); + }); + + it("include the wait for load", async () => { + const host = createHost(noErrors); + const waiting = rejection(host.get(Info, {}, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + expect(await waiting).toEqual({ code: "unavailable", message: "app.info: did not become live within 100 ms" }); + }); + + it("are per getter: the first to pass releases only its own reference, the attempt goes on for the other", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes), { readyTimeout: 10_000 }); + const early = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 100 })); + const late = host.get(Svc, { id: 1 }, context, { readyTimeout: 1000 }); + await vi.advanceTimersByTimeAsync(100); + expect(await early).toMatchObject({ code: "unavailable" }); + gate.resolve(); + expect((await late).n.value).toBe(1); + expect(closes).toEqual([]); + }); +}); + +describe("close", () => { + it("ends every object, the last to go live first, and rejects new calls", async () => { + const Outer = local("app.outer", { version: 1, params: {}, members: { title: str() } }); + const { host } = await loaded((h) => { + h.serve(Info, () => ({ title: constant("info") })); + h.serve(Outer, (_, self) => ({ title: self.get(Info, {}).title })); + }); + const ends: string[] = []; + const outer = await host.get(Outer, {}, context); + const info = await host.get(Info, {}, context); + void (await outer.title.watch(context)).closed.then(() => ends.push("outer")); + void (await info.title.watch(context)).closed.then(() => ends.push("info")); + await host.close(context); + expect(ends).toEqual(["outer", "info"]); + expect(await rejection(host.get(Info, {}, context))).toEqual({ + code: "released", + reason: "closed", + message: "the host is closed", + }); + expect(() => info.title.value).toThrow("app.info: ended"); + }); + + it("fails a get waiting for load", async () => { + const host = createHost(noErrors); + const got = host.get(Info, {}, context); + await host.close(context); + expect(await rejection(got)).toEqual({ code: "released", reason: "closed", message: "the host is closed" }); + }); + + it("ends every object at once: calls fail released before any close ran", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, (_, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { count: constant(1), bump: async () => null }; + }), + ); + const counter = await host.get(Counter, {}, context); + const closed = host.close(context); + expect(await rejection(counter.bump(context))).toEqual({ + code: "released", + reason: "ended", + message: "app.counter: ended", + }); + closing.resolve(); + await closed; + }); + + it("aborts a pending open and does not wait for it; its late value is closed with close", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes)); + const got = rejection(host.get(Svc, { id: 1 }, context)); + await flushed(); + await host.close(context); + expect(await got).toEqual({ code: "released", reason: "ended", message: "app.svc(1): ended" }); + gate.resolve(); + await flushed(); + expect(closes).toEqual(["svc 1 close"]); + }); + + it("joins an ending running in the background", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Svc, ({ id }, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { n: constant(id) }; + }), + ); + (await host.get(Svc, { id: 1 }, context)).release(); + let closed = false; + const close = host.close(context).then(() => { + closed = true; + }); + await flushed(); + expect(closed).toBe(false); + closing.resolve(); + await close; + }); + + it("closes object by object, a dependent before its dependencies", async () => { + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + const events: string[] = []; + /** A native whose close records when it starts and, a turn later, when it ends. */ + const open = (self: Self, name: string): void => { + self.open( + async () => null, + async () => { + events.push(`${name} closing`); + await flushed(); + events.push(`${name} closed`); + }, + ); + }; + const { host } = await loaded((h) => { + h.serve(Svc, ({ id }, self) => { + open(self, `svc ${id}`); + return { n: constant(id) }; + }); + h.serve(View, ({ id }, self) => { + open(self, `view ${id}`); + return { n: self.get(Svc, { id }).n }; + }); + }); + await host.get(View, { id: 1 }, context); + await host.get(View, { id: 2 }, context); + await host.close(context); + expect(events).toEqual([ + "view 2 closing", + "view 2 closed", + "svc 2 closing", + "svc 2 closed", + "view 1 closing", + "view 1 closed", + "svc 1 closing", + "svc 1 closed", + ]); + }); +}); + +describe("close, its order and its races", () => { + it("returns one closing to every caller: a second close waits for the first", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, (_, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { count: constant(1), bump: async () => null }; + }), + ); + const first = host.close(context); + let second = false; + void host.close(context).then(() => { + second = true; + }); + await flushed(); + expect(second).toBe(false); + closing.resolve(); + await first; + await flushed(); + expect(second).toBe(true); + }); + + it("tears down what a failed dependent holds with close, and waits for it", async () => { + const Pair = local("app.pair", { version: 1, params: { id: int() }, members: { n: int() } }); + const closing = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => null, + async (_value, reason) => { + closes.push(`svc ${reason.kind}`); + await closing.promise; + }, + ); + return { n: constant(id) }; + }); + h.serve(Core, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(0) }; + }); + h.serve(Pair, ({ id }, self) => ({ + n: self.derive([self.get(Svc, { id }).n, self.get(Core, {}).n], ([n]) => n), + })); + }); + // The pair fails on the core (R18) and keeps its references: svc(1), live before the pair, held by it alone. + const svc = await host.get(Svc, { id: 1 }, context); + void host.get(Pair, { id: 1 }, context).catch(() => {}); + await flushed(); + svc.release(); + let closed = false; + const close = host.close(context).then(() => { + closed = true; + }); + await flushed(); + expect([closes, closed]).toEqual([["svc close"], false]); + closing.resolve(); + await close; + }); + + it("goes on when onError throws: failures are still classified, teardowns still finish", async () => { + const host = createHost({ + onError: () => { + throw new Error("onError threw"); + }, + }); + const facet = defineFacet((h) => + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + if (id === 2) throw broken(new Error("no config")); + return null; + }, + () => { + throw new Error("close threw"); + }, + ); + return { n: constant(id) }; + }), + ); + await host.load([{ name: "a", facet }], context); + // The background end reports its throwing close; the next instance waits for that teardown. + (await host.get(Svc, { id: 1 }, context)).release(); + expect((await host.get(Svc, { id: 1 }, context)).n.value).toBe(1); + expect(await rejection(host.get(Svc, { id: 2 }, context))).toEqual({ + code: "unavailable", + message: "app.svc(2): no config", + }); + }); + + describe("with a backoff window running", () => { + fakeTimers(); + + it("ends an object waiting for its window and clears every timer", async () => { + const { host } = await loaded((h) => + h.serve(Svc, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(0) }; + }), + ); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + await first; + const waiting = rejection(host.get(Svc, { id: 1 }, context)); + await flushed(); + await host.close(context); + expect(await waiting).toEqual({ code: "released", reason: "ended", message: "app.svc(1): ended" }); + expect(vi.getTimerCount()).toBe(0); + }); + }); +}); + +describe("getters", () => { + it("one cancelled while another waits for the same object: the other gets it, the object lives", async () => { + const { context: cancellable, cancel } = withCancel(context); + const factory = vi.fn((_: { id: number }) => { + cancel(new Error("cancelled")); + return { id: constant(1) }; + }); + const { host } = await loaded((h) => h.serve(Page, factory)); + const [cancelled, kept] = await Promise.allSettled([ + host.get(Page, { id: 1 }, cancellable), + host.get(Page, { id: 1 }, context), + ]); + expect(cancelled).toMatchObject({ status: "rejected", reason: new Error("cancelled") }); + expect(kept.status === "fulfilled" && kept.value.id.value).toBe(1); + await host.get(Page, { id: 1 }, context); + expect(factory).toHaveBeenCalledTimes(1); + }); +}); + +forEachCodec(() => { + describe("principals and where a reference goes", () => { + const Who = remote("app.who", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); + const Outer = remote("app.outer", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); + const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { who: str() } }); + const Desk = local("app.desk", { version: 1, params: {}, members: { who: method(undefined, str()) } }); + const Gate = remote("app.gate", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { methods: { who: method(undefined, str()) } } }, + }); + type P = TestPrincipal; + + /** Serves every type above: `Who` shows the principal its object is for. */ + function serveAll(h: FacetHost

, factories: string[]) { + h.serve(Who, ({ id }, principal) => { + factories.push(`${id} ${principal.kind}`); + return { variant: "v", members: { who: constant(principal.kind) } }; + }); + h.serve(Outer, (_, _principal, self) => ({ variant: "v", members: { who: self.get(Who, { id: 3 }).who } })); + h.serve(Inner, ({ id }, self) => ({ who: self.get(Who, { id }).who })); + const who = async (ctx: Ctx

) => { + const ref = await ctx.get(Who, { id: 2 }); + return ref.who.value; + }; + h.serve(Desk, () => ({ who })); + h.serve(Gate, () => ({ variant: "v", members: { who } })); + } + + it("host.get acts as the host's principal; a host without one (a client) serves no remote type", async () => { + const factories: string[] = []; + const { server } = await linked((h) => serveAll(h, factories)); + expect((await server.get(Who, { id: 1 }, context)).who.value).toBe("server"); + const client = createHost

({ onError: () => {} }); + const report = await client.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without a principal serves only local types', + }), + }, + ]); + await client.close(context); + }); + + it("a host without an address serves no remote type, whatever its principal", async () => { + const host = createHost

({ principal: { kind: "server" }, onError: () => {} }); + const report = await host.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without an address serves only local types', + }), + }, + ]); + await host.close(context); + }); + + it("ctx.get acts as ctx.principal: a guest's call gets the guest's view; a local handler's acts as the host", async () => { + const { server, client } = await linked((h) => serveAll(h, []), { child: { kind: "guest", session: "S1" } }); + expect(await (await client.get(Gate, {}, context)).who(context)).toBe("guest"); + expect(await (await server.get(Desk, {}, context)).who(context)).toBe("server"); + }); + + it("self.get acts as a remote object's principal, a local object's as the host's", async () => { + const { server, client } = await linked((h) => serveAll(h, []), { child: { kind: "guest", session: "S1" } }); + expect((await client.get(Outer, {}, context)).who.value).toBe("guest"); + expect((await server.get(Inner, { id: 4 }, context)).who.value).toBe("server"); + }); + + it("identity includes the principal key: equal keys share one object", async () => { + const factories: string[] = []; + const host = createHost

({ + address: Server(), + principal: { kind: "guest", session: "S1" }, + principalKey: (principal) => principal.kind, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, factories)) }], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "guest", session: "S2" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await host.get(Who, { id: 1 }, context); + peer.send({ + kind: "hold", + hold: 0, + type: Who.name, + version: Who.version, + fingerprint: Who.fingerprint, + params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, + }); + await peer.next("bound"); + expect(factories).toEqual(["1 guest"]); + const [c, d] = memoryLinkPair(); + host.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const owner = new RawPeer(d); + owner.send({ + kind: "hold", + hold: 0, + type: Who.name, + version: Who.version, + fingerprint: Who.fingerprint, + params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, + }); + await owner.next("bound"); + expect(factories).toEqual(["1 guest", "1 owner"]); + }); + + it("remote objects with empty params are not activated by load; they live while referenced", async () => { + const factories: string[] = []; + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const { server } = await linked((h) => + h.serve(Empty, (_, principal) => { + factories.push(principal.kind); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + expect(factories).toEqual([]); + (await server.get(Empty, {}, context)).release(); + await flushed(); + await server.get(Empty, {}, context); + expect(factories).toEqual(["server", "server"]); + }); + + it("a local type never crosses a link; a remote type without a parent link waits for one", async () => { + const { client } = await linked(() => {}); + expect(client.hold(Inner, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(await rejection(client.get(Inner, { id: 1 }, context, { readyTimeout: 5 }))).toEqual({ + code: "unavailable", + message: "app.inner is not served here", + }); + const alone = createHost

({ onError: () => {} }); + await alone.load([], context); + expect(alone.hold(Who, { id: 1 }, context).state.value).toEqual({ kind: "loading" }); + expect(await rejection(alone.get(Who, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.who(1): did not become live within 10 ms", + }); + }); + + it("connect takes one open parent link", async () => { + const { client } = await linked(() => {}); + expect(() => + client.connect(memoryLinkPair()[0], { principal: { kind: "server" }, codec: testCodec() }, context), + ).toThrow("connect: this host has an open parent link"); + }); + + it("close closes the links first: the consumer sees the link drop, no gone", async () => { + const { server, client } = await linked((h) => serveAll(h, [])); + const who = await client.get(Who, { id: 1 }, context); + const watch = await who.who.watch(context); + await server.close(context); + await transported(); + expect(who.state.value).toEqual({ kind: "loading" }); + expect(who.who.value).toBe("owner"); + expect(await Promise.race([watch.closed, transported().then(() => "open")])).toBe("open"); + }); + }); +}); + +describe("host.registry and the host's close (fix round 1)", () => { + const Absent = defineRegistry("app.absent", { version: 1 }); + + it("ends a waiting host.registry at the close with released/closed, its deadline timer with it (Sol 5, Astra 5)", async () => { + const host = createHost(noErrors); + await host.load([], context); + let outcome: unknown; + void rejection(host.registry(Absent, context, { readyTimeout: 60_000 })).then((error) => { + outcome = error; + }); + await flushed(); + await host.close(context); + await flushed(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("ends one still waiting for the publication at the close", async () => { + const host = createHost(noErrors); + const waiting = rejection(host.registry(Absent, context, { readyTimeout: 60_000 })); + await host.close(context); + expect(await waiting).toMatchObject({ code: "released", reason: "closed" }); + }); +}); + +describe("the selected provider's own checks for a request made here (fix round 1)", () => { + const Here = hostKind("app-here"); + const Elsewhere = hostKind("app-elsewhere"); + const shape = { version: 1, params: {}, variants: { view: { state: { n: int() } } } } as const; + + it("waits a same-host remote request the provider's code places elsewhere: invalid/host_address (Astra 4)", async () => { + const Consumer = remote("app.placed", { ...shape, hostAddress: () => Here() }); + const Provider = remote("app.placed", { ...shape, hostAddress: () => Elsewhere() }); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await host.load([{ name: "@app/provider", facet }], context); + const ref = host.hold(Consumer, {}, context); + await flushed(); + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "invalid", reason: "host_address" } }, + }); + ref.release(); + await host.close(context); + }); + + it("fails broken when the provider's hostAddress throws: its own code (Sol 7)", async () => { + const Consumer = remote("app.remote", { ...shape, hostAddress: () => Server() }); + const Provider = remote("app.remote", { + ...shape, + hostAddress: () => { + throw new Error("provider address bug"); + }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await server.load([{ name: "@app/provider", facet }], context); + await client.load([], context); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { kind: "server" } }, context); + client.connect(up, { principal: { kind: "server" } }, context); + const held = client.hold(Consumer, {}, context); + await transported(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { message: "app.remote: provider address bug" } }, + }); + // The same throw for a request made on the provider's host fails it broken too. + const here = server.hold(Consumer, {}, context); + expect(here.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await client.close(context); + await server.close(context); + }); +}); + +describe('a load whose setup throws (decision "Per plugin at startup and reload")', () => { + it("leaves that plugin out and reports it; the others publish, the host and its links stay open", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + const [c, d] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + host.connect(c, { principal: { kind: "server" }, codec: testCodec() }, context); + const child = new RawPeer(b); + const parent = new RawPeer(d); + const broken = defineFacet(() => { + throw new Error("setup threw"); + }); + const report = await host.load( + [ + { name: "@test/broken", facet: broken }, + { + name: "@test/app", + facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))), + }, + ], + context, + ); + expect(report.failed).toEqual([ + { plugin: "@test/broken", error: expect.objectContaining({ message: 'plugin "@test/broken": setup threw' }) }, + ]); + await transported(); + expect([child.closed, parent.closed]).toEqual([false, false]); + expect((await host.get(Info, {}, context)).title.value).toBe("t"); + // What the left-out plugin would have served waits as any missing type does. + expect(host.hold(Page, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + await host.close(context); + }); +}); + +describe("acquiring: hold and get", () => { + fakeTimers(); + + /** Serves `Svc`, whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ + function serveFlaky(h: FacetHost, failing: { left: number; broken: boolean }): void { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + throw failing.broken ? broken(new Error("offline")) : new Error("offline"); + }, + () => {}, + ); + return { n: constant(id) }; + }); + } + + const offline = { code: "unavailable", message: "app.svc(1): offline" }; + + it("invalid params at once; a local type not served here: hold waits failed, get rejects at its readyTimeout", async () => { + const host = createHost({ address: Server(), ...noErrors, principal: { kind: "server" } }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const params = { code: "invalid", reason: "params" }; + expect(() => host.hold(Notes, { session: 1 } as never, context)).toThrow("params"); + expect(await rejection(host.get(Notes, { session: 1 } as never, context))).toMatchObject(params); + const unserved = { code: "unavailable", message: "app.info is not served here" }; + expect(host.hold(Info, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: unserved }, + retryAt: undefined, + }); + const got = rejection(host.get(Info, {}, context, { readyTimeout: 5 })); + await vi.advanceTimersByTimeAsync(5); + expect(await got).toEqual(unserved); + }); + + it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type waits", async () => { + const host = createHost(noErrors); + const held = host.hold(Info, {}, context); + const unserved = host.hold(Page, { id: 1 }, context); + const got = host.get(Info, {}, context); + const refused = rejection(host.get(Page, { id: 1 }, context, { readyTimeout: 50 })); + expect([held.state.value, unserved.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); + await host.load( + [{ name: "@test/app", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + await flushed(); + expect(held.state.value).toEqual({ kind: "live" }); + expect((await got).title.value).toBe("t"); + const error = { code: "unavailable", message: "app.page is not served here" }; + expect(unserved.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error }, + retryAt: undefined, + }); + await vi.advanceTimersByTimeAsync(50); + expect(await refused).toEqual(error); + }); + + it("invalid params before load resolved: hold throws and get rejects invalid/params at once (audit, 7a)", async () => { + const host = createHost(noErrors); + expect(() => host.hold(Page, { id: "x" } as never, context)).toThrow("params"); + expect(await rejection(host.get(Page, { id: "x" } as never, context))).toMatchObject({ + code: "invalid", + reason: "params", + }); + await host.load([], context); + }); + + it("a hold released before load resolved makes nothing once it did", async () => { + const factory = vi.fn(({ id }: { id: number }) => ({ id: constant(id) })); + const host = createHost(noErrors); + host.hold(Page, { id: 1 }, context).release(); + await host.load([{ name: "@test/app", facet: defineFacet((h) => h.serve(Page, factory)) }], context); + await flushed(); + expect(factory).not.toHaveBeenCalled(); + }); + + it("through transient failures: hold is failed with retryAt, then loading; get waits, then rejects with the last cause", async () => { + const { host } = await loaded((h) => serveFlaky(h, { left: 5, broken: false })); + const held = host.hold(Svc, { id: 1 }, context); + const got = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 1_500 })); + await flushed(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "transient", error: offline }, + retryAt: 1_000, + }); + await vi.advanceTimersByTimeAsync(1_000); + await flushed(); + expect(held.state.value).toMatchObject({ kind: "failed", retryAt: 3_000 }); + await vi.advanceTimersByTimeAsync(500); + expect(await got).toEqual(offline); + // The getter released only its own reference: the hold follows the object's retries. + await vi.advanceTimersByTimeAsync(1_500); + expect(held.state.value).toMatchObject({ kind: "failed", retryAt: 7_000 }); + }); + + it("a broken failure: hold is failed without retryAt; get releases its reference and rejects at once", async () => { + const { host } = await loaded((h) => serveFlaky(h, { left: 1, broken: true })); + const held = host.hold(Svc, { id: 1 }, context); + expect(await rejection(host.get(Svc, { id: 1 }, context))).toEqual(offline); + expect(held.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" }, retryAt: undefined }); + }); + + it("a cancelled context: no effect on a hold; a get releases its reference and rejects", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes)); + const { context: cancellable, cancel } = withCancel(context); + const held = host.hold(Svc, { id: 1 }, cancellable); + const got = host.get(Svc, { id: 2 }, cancellable); + cancel(new Error("user cancelled")); + await expect(got).rejects.toThrow("user cancelled"); + gate.resolve(); + await flushed(); + expect(held.state.value).toEqual({ kind: "live" }); + // The getter's object ended with its only reference: its late value was closed. + expect(closes).toEqual(["svc 2 end"]); + }); + + it("host.close: queued calls reject released, references go released, their watches end (race 10)", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serveGated(h, gate.promise, [])); + const local = host.hold(Svc, { id: 1 }, context); + const remoteHost = createHost(noErrors); + await remoteHost.load([], context); + const remote = remoteHost.hold(Notes, { session: "S1" }, context); + const call = remote.wait(context); + const watches = [await local.state.watch(context), await remote.state.watch(context)]; + await Promise.all([host.close(context), remoteHost.close(context)]); + expect(await rejection(call)).toEqual({ code: "released", reason: "closed", message: "the host is closed" }); + expect([local.state.value, remote.state.value]).toMatchObject([{ kind: "ended" }, { kind: "ended" }]); + expect(await Promise.all(watches.map(({ closed }) => closed))).toEqual([{ kind: "ended" }, { kind: "ended" }]); + }); +}); + +forEachCodec(() => { + describe("connect after a drop", () => { + it("throws while the previous parent link is open or still closing, and takes the next once it closed (race 4)", async () => { + const { client, links } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + const next = (): void => + client.connect(memoryLinkPair()[0], { principal: { kind: "server" }, codec: testCodec() }, context); + // The server's end closes now, the client's a microtask later. + links[0].close(); + expect(next).toThrow("connect: this host has an open parent link"); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(next).not.toThrow(); + }); + }); +}); + +forEachCodec(() => { + describe("where a reference or an arriving hold goes (section 1's table)", () => { + /** A worker for session S1 serving `PlacedNotes`, a raw peer as its child, and one as its parent when `parent`. */ + async function worker(options: { readonly parent?: boolean } = {}) { + const errors: HostError[] = []; + const host = createHost({ + address: Session({ id: "S1" }), + principal: { kind: "worker", session: "S1" }, + onError: (error) => errors.push(error), + }); + let parent: RawPeer | undefined; + if (options.parent === true) { + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + parent = new RawPeer(b); + } + await host.load( + [{ name: "@test/w", facet: defineFacet((h) => serveNotes(h, PlacedNotes)) }], + context, + ); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + return { host, child: new RawPeer(b), parent, errors }; + } + + /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming the session `address`. */ + function holdOf(peer: RawPeer, session: string, address: string, type: string = PlacedNotes.name, hold = 0) { + peer.send({ + kind: "hold", + hold, + type, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session }), + address: { kind: "session", params: { id: address } }, + }); + } + + const misplaced = { + code: "invalid", + reason: "host_address", + message: "app.placed-notes: placed on session, this host is session|S1", + }; + + it("binds a hold of a type served here that names this host's address", async () => { + const { child } = await worker(); + holdOf(child, "S1", "S1"); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + + it("keeps a hold naming this host whose params the code here places elsewhere waiting: invalid/host_address (code skew, M§1)", async () => { + const { child } = await worker(); + holdOf(child, "S2", "S1"); + expect(await child.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: misplaced } }, + }); + // A refused hold is kept (a publication or a retry starts it again): the next hold takes the next id. + holdOf(child, "S2", "S1", PlacedNotes.name, 1); + expect(await child.next("failed")).toMatchObject({ hold: 1, cause: { kind: "waiting" } }); + }); + + it("places a hold by the address it names, never by the name it serves (a relay never captures it)", async () => { + const { child, parent } = await worker({ parent: true }); + holdOf(child, "S2", "S2"); + const relayed = await parent!.next("hold"); + expect(relayed).toMatchObject({ type: PlacedNotes.name, address: { kind: "session", params: { id: "S2" } } }); + holdOf(child, "S1", "S2", PlacedNotes.name, 1); + expect(await parent!.next("hold")).toMatchObject({ hold: 1 }); + }); + + it("sends a reference made here whose host address is another host over the parent link, though it serves the type", async () => { + const { host, parent } = await worker({ parent: true }); + const elsewhere = host.hold(PlacedNotes, { session: "S2" }, context); + const hold = await parent!.next("hold"); + expect(hold).toMatchObject({ type: PlacedNotes.name, address: { kind: "session", params: { id: "S2" } } }); + expect(elsewhere.state.value).toEqual({ kind: "loading" }); + expect((await host.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("editor"); + }); + + it("sends a hold it cannot serve or route over the parent link, unparsed", async () => { + const { child, parent } = await worker({ parent: true }); + holdOf(child, "S9", "S9", "app.elsewhere"); + const relayed = await parent!.next("hold"); + expect(relayed).toMatchObject({ type: "app.elsewhere" }); + expect(partValue(relayed.params)).toEqual({ session: "S9" }); + }); + + it("never sends a hold from the parent back to it: unavailable (race 8)", async () => { + const { parent } = await worker({ parent: true }); + holdOf(parent!, "S9", "S9", "app.elsewhere"); + expect(await parent!.next("failed")).toMatchObject({ + hold: 0, + cause: { + kind: "broken", + error: { kind: "chord", error: { code: "unavailable", message: "app.elsewhere is not served here" } }, + }, + }); + expect(parent!.kinds()).toEqual(["hello"]); + }); + + it("refuses a hold it cannot serve or route without a parent link: unavailable", async () => { + const { child } = await worker(); + holdOf(child, "S9", "S9", "app.elsewhere"); + expect(await child.next("failed")).toMatchObject({ + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, + }); + }); + + it("fails a reference broken with what its hostAddress throws", async () => { + const Throwing = remote("app.throwing", { + version: 1, + params: {}, + hostAddress: () => { + throw new Error("hostAddress threw"); + }, + variants: { v: { state: { n: int() } } }, + }); + const client = createHost(noErrors); + await client.load([], context); + expect(await rejection(client.get(Throwing, {}, context))).toMatchObject({ + message: "app.throwing: hostAddress threw", + }); + expect(client.hold(Throwing, {}, context).state.value).toMatchObject({ kind: "failed" }); + }); + }); +}); + +describe("the host's store", () => { + /** A store on `:memory:`, as Chord's layers see it. */ + async function memoryStore(): Promise { + return (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + } + + it("belongs to one open host at a time (rule 7)", async () => { + const store = await memoryStore(); + const first = createHost({ store, onError: () => {} }); + expect(() => createHost({ store, onError: () => {} })).toThrow("store: already attached to an open host"); + await first.close(context); + await createHost({ store, onError: () => {} }).close(context); + await store.close(context); + }); + + it("reports storage_failed once and closes the host; the app still closes the store", async () => { + const { store, fail } = await faultyStore(); + const errors: HostError[] = []; + const host = createHost({ store, onError: (error) => errors.push(error) }); + await host.load([], context); + const write: StagedWrite = { order: "write", apply: async () => [] }; + const fence = { admit: () => true, adopt: () => {} }; + // A failed `COMMIT`: an unknown outcome (D4). + fail("COMMIT"); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + // The host closed itself; the app's own close returns that closing. + expect((host as ChordHost).closing).toBe(true); + await host.close(context); + expect(errors.map(({ kind }) => kind)).toEqual(["storage_failed"]); + await store.close(context); + }); +}); + +/** A plugin's facet serving `Info` with the title `title` (a new facet value each call: a changed plugin). */ +function infoFacet(title: string) { + return defineFacet((h) => h.serve(Info, () => ({ title: constant(title) }))); +} + +describe("reload: the API (slice 7b §1)", () => { + it("throws before load resolved and rejects released/closed after close", async () => { + const host = createHost(noErrors); + expect(() => host.reload([], context)).toThrow("reload runs after load"); + await host.load([], context); + await host.close(context); + expect(await rejection(host.reload([], context))).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("reports plugins added, changed and removed; the same facet object is unchanged and untouched", async () => { + const core = infoFacet("core"); + const host = createHost(noErrors); + const kept = defineFacet((h) => h.serve(Counter, () => ({ count: constant(1), bump: async () => null }))); + await host.load( + [ + { name: "@app/core", facet: kept }, + { name: "@app/info", facet: core }, + ], + context, + ); + const counter = await host.get(Counter, {}, context); + const report = await host.reload( + [ + { name: "@app/core", facet: kept }, + { name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }, + ], + context, + ); + expect(report.plugins).toEqual({ added: ["@app/page"], changed: [], removed: ["@app/info"] }); + expect(report.objects).toEqual([{ object: "app.info", kind: "ended" }]); + // Untouched: the same reference stays live on the same object. + expect(counter.state.value).toEqual({ kind: "live" }); + expect((await host.get(Page, { id: 3 }, context)).id.value).toBe(3); + await host.close(context); + }); + + it("runs reloads one after another", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const order: string[] = []; + const first = host.reload([{ name: "@app/info", facet: infoFacet("b") }], context).then(() => order.push("b")); + const second = host.reload([{ name: "@app/info", facet: infoFacet("c") }], context).then(() => order.push("c")); + await Promise.all([first, second]); + expect(order).toEqual(["b", "c"]); + expect((await host.get(Info, {}, context)).title.value).toBe("c"); + await host.close(context); + }); + + it("an abort before the commit rejects aborted and changes nothing", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const { context: aborted, cancel } = withCancel(context); + cancel(new Error("stop")); + expect(await rejection(host.reload([{ name: "@app/info", facet: infoFacet("b") }], aborted))).toMatchObject({ + code: "aborted", + }); + expect((await host.get(Info, {}, context)).title.value).toBe("a"); + await host.close(context); + }); + + it("an abort after the commit rejects aborted naming what the commit applied, which stands", async () => { + const host = createHost(noErrors); + await host.load( + [ + { name: "@app/info", facet: infoFacet("a") }, + { name: "@app/gone", facet: defineFacet(() => {}) }, + ], + context, + ); + const info = await host.get(Info, {}, context); + const { context: aborting, cancel } = withCancel(context); + // The abort lands right after the commit (its publication batch), while the reload waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) cancel(new Error("stop")); + }); + const reloading = host.reload( + [ + { name: "@app/info", facet: infoFacet("b") }, + { name: "@app/new", facet: defineFacet(() => {}) }, + ], + aborting, + ); + const error = await rejection(reloading); + hook.mockRestore(); + expect(error).toMatchObject({ + code: "aborted", + details: { committed: { added: ["@app/new"], changed: ["@app/info"], removed: ["@app/gone"], failed: [] } }, + }); + expect(info.title.value).toBe("b"); + await host.close(context); + }); + + it("reports a setup's or a factory's throw as failed, naming the plugin, which stays on its old code", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const info = await host.get(Info, {}, context); + const throwing = defineFacet(() => { + throw new Error("setup threw"); + }); + const failedOf = async (facets: { readonly name: string; readonly facet: typeof throwing }[]) => { + const report = await host.reload(facets, context); + return report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await failedOf([{ name: "@app/info", facet: throwing }])).toEqual([ + ["@app/info", 'plugin "@app/info": setup threw'], + ]); + const factoryThrows = defineFacet((h) => + h.serve(Info, () => { + throw new Error("factory threw"); + }), + ); + expect(await failedOf([{ name: "@app/info", facet: factoryThrows }])).toEqual([ + ["@app/info", 'plugin "@app/info", app.info: factory threw'], + ]); + // A type another plugin serves already fails the plugin with new code; an added one is left out. + const twice = [ + { name: "@app/info", facet: infoFacet("b") }, + { name: "@app/other", facet: infoFacet("c") }, + ]; + const report = await host.reload(twice, context); + expect(report.plugins).toEqual({ added: [], changed: ["@app/info"], removed: [] }); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["@app/other", 'plugin "@app/other": app.info is served twice (plugin "@app/info")'], + ]); + expect(info.title.value).toBe("b"); + expect(info.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("reload: the commit (slice 7b §3)", () => { + it("serves added types, drops removed ones and their presence lists, and wakes requests per name", async () => { + const host = createHost(noErrors) as unknown as ChordHost; + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const page = host.hold(Page, { id: 1 }, context); + const other = host.hold(Core, {}, context); + await flushed(); + expect(page.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const presence = host.references(Info); + const otherStates = statesOf(await other.state.watch(context)); + await host.reload( + [ + { name: "@app/info", facet: infoFacet("a") }, + { name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }, + ], + context, + ); + await flushed(); + expect(page.state.value).toEqual({ kind: "live" }); + expect(page.id.value).toBe(1); + // Not a name the reload serves: still waiting, not checked again. + expect(other.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(otherStates).toEqual([]); + await host.reload( + [{ name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }], + context, + ); + expect(host.references(Info)).not.toBe(presence); + await host.close(context); + }); + + it("replaces a reloaded registry's provider and its plugin's contributions; a removed provider's readers wait", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const provider = (separator: string) => + defineFacet((h, self) => { + h.serve(Names, { compose: (table) => table.map(({ value }) => value).join(separator) }); + self.contribute(Names, "own"); + }); + const contributor = (name: string) => defineFacet((_h, self) => self.contribute(Names, name)); + const host = createHost(noErrors); + await host.load( + [ + { name: "@app/names", facet: provider(",") }, + { name: "@app/x", facet: contributor("x") }, + ], + context, + ); + const names = await host.registry(Names, context); + expect(names.value).toBe("own,x"); + await host.reload( + [ + { name: "@app/names", facet: provider("+") }, + { name: "@app/x", facet: contributor("y") }, + ], + context, + ); + expect(names.value).toBe("own+y"); + await host.reload([{ name: "@app/x", facet: contributor("z") }], context); + expect(() => names.value).toThrow("no plugin serves it here"); + await host.reload( + [ + { name: "@app/names", facet: provider(";") }, + { name: "@app/x", facet: contributor("z") }, + ], + context, + ); + expect(names.value).toBe("own;z"); + await host.close(context); + }); +}); + +/** A worker's title per session (v1: a string). */ +const Title = remote("app.title", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { title: str() } } }, +}); + +/** `Title` v2: an object; v1 consumers map back through `accepts`. */ +const TitleV2 = evolve(Title, (e) => e.state("v", "title", obj({ text: str() }))); + +describe("reload: registries' older readers (slice 7b §3)", () => { + it("a reader of an older version follows the new provider's accepts[old].value; one it no longer serves waits", async () => { + const NamesV1 = defineRegistry("app.names", { version: 1 }); + const NamesV2 = defineRegistry("app.names", { version: 2 }); + const provider = (map: ((names: string[]) => string) | undefined) => + defineFacet((h, self) => { + h.serve(NamesV2, { + compose: (table) => table.map(({ value }) => value), + ...(map === undefined ? {} : { accepts: { 1: { value: map } } }), + }); + self.contribute(NamesV2, "a"); + }); + const host = createHost(noErrors); + await host.load([{ name: "@app/names", facet: provider((names) => names.join(",")) }], context); + const old = await host.registry(NamesV1, context); + expect(old.value).toBe("a"); + await host.reload([{ name: "@app/names", facet: provider((names) => `${names.join("+")}!`) }], context); + expect(old.value).toBe("a!"); + await host.reload([{ name: "@app/names", facet: provider(undefined) }], context); + expect(() => old.value).toThrow("version 1 is not served here"); + await host.close(context); + }); +}); + +describe("reload across hosts, in the app's order (slice 7b §11)", () => { + it("server, then worker, then tab: an old tab is mapped, then waits for its version, then reloads its own code", async () => { + const serveV1 = (h: FacetHost) => + h.serve(Title, () => ({ variant: "v", members: { title: constant("one") } })); + const serveV2 = (text: string, accept: boolean) => (h: FacetHost) => + h.serve( + TitleV2, + () => ({ variant: "v", members: { title: constant({ text }) } }), + accept ? { accepts: { 1: { state: { v: { title: (current) => current.text } } } } } : {}, + ); + const workerSet = (setup: (h: FacetHost) => void) => [ + { + name: "@test/worker", + facet: defineFacet((h) => { + serveNotes(h, PlacedNotes); + setup(h); + }), + }, + ]; + const { tab, server, workers } = await relayed({ worker: serveV1 }); + const old = tab.hold(Title, { session: "S1" }, context); + await transported(); + expect(old.title.value).toBe("one"); + const worker = workers.get("session|S1")!; + // The server installs first: nothing of Title changes there. + await server.reload([{ name: "@test/server", facet: defineFacet(() => {}) }], context); + // The worker installs v2 accepting v1: the old tab is mapped. + await worker.reload(workerSet(serveV2("two", true)), context); + await transported(); + expect(old.state.value).toEqual({ kind: "live" }); + expect(old.title.value).toBe("two"); + // A worker install without the adapter: the old tab waits for its version (a remote failure is its reads' error). + await worker.reload(workerSet(serveV2("three", false)), context); + await transported(); + expect(old.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + // The tab reloads its own code: its new hold (v2) binds; the old one is released. + const fresh = tab.hold(TitleV2, { session: "S1" }, context); + old.release(); + await transported(); + expect(fresh.title.value).toEqual({ text: "three" }); + await server.close(context); + }); +}); + +describe("reload: an abort while staging (fix round 1)", () => { + it("does not commit when the context is aborted by the synchronous setup", async () => { + const T = local("astra.cancel", { version: 1, params: {}, members: { n: int() } }); + const host = createHost({ onError: () => {} }); + const original = defineFacet((h) => h.serve(T, () => ({ n: constant(1) }))); + await host.load([{ name: "@astra/cancel", facet: original }], context); + const ref = await host.get(T, {}, context); + const abort = new AbortController(); + const changed = defineFacet((h) => { + abort.abort(); + h.serve(T, () => ({ n: constant(2) })); + }); + await host + .reload([{ name: "@astra/cancel", facet: changed }], withAbortSignal(abort.signal, context)) + .catch(() => {}); + await flushed(); + const actual = ref.n.value; + ref.release(); + await host.close(context); + expect(actual).toBe(1); + }); +}); diff --git a/packages/chord/test/next/host/members.test.ts b/packages/chord/test/next/host/members.test.ts new file mode 100644 index 000000000..9c5e40793 --- /dev/null +++ b/packages/chord/test/next/host/members.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { loaded } from "../support.ts"; + +describe("the member check (Q14)", () => { + it("refuses an inherited property for a member", async () => { + const Named = local("app.named", { version: 1, params: { id: int() }, members: { toString: method() } }); + const { host } = await loaded((h) => h.serve(Named, () => ({}) as never)); + await expect(host.get(Named, { id: 1 }, context)).rejects.toThrow( + "app.named(1): toString: a method is a function", + ); + }); + + it("names a derive in errors by its member when exposed, else by its position", async () => { + const Named = local("app.named", { version: 1, params: { n: int() }, members: { label: int() } }); + const bad = { value: 1 }; + const { host } = await loaded((h) => + h.serve(Named, ({ n }, self) => { + const inner = self.derive(n === 1 ? [] : [bad as never], () => 1); + return { label: self.derive(n === 1 ? [inner, bad as never] : [inner], () => 2) }; + }), + ); + await expect(host.get(Named, { n: 1 }, context)).rejects.toThrow( + "app.named(1): label: source 1 is not a Chord watchable", + ); + await expect(host.get(Named, { n: 2 }, context)).rejects.toThrow( + "app.named(2): derive #1: source 0 is not a Chord watchable", + ); + }); +}); diff --git a/packages/chord/test/next/host/native.test.ts b/packages/chord/test/next/host/native.test.ts new file mode 100644 index 000000000..d2ad9ae97 --- /dev/null +++ b/packages/chord/test/next/host/native.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import type { FacetHost, Native } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { failures, fakeTimers, flushed, loaded, rejection } from "../support.ts"; + +const Res = local("app.res", { version: 1, params: { id: int() }, members: { n: int() } }); + +describe("opening", () => { + it("opens one after another in declaration order; a later opener reads an earlier handle", async () => { + const events: string[] = []; + const first = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + const a = self.open( + async () => { + events.push("open a"); + return first.promise; + }, + () => {}, + ); + self.open( + async () => { + events.push(`open b with ${a.value}`); + return "b"; + }, + () => {}, + ); + return { n: constant(1) }; + }), + ); + const got = host.get(Res, { id: 1 }, context); + await flushed(); + expect(events).toEqual(["open a"]); + first.resolve("a"); + await got; + expect(events).toEqual(["open a", "open b with a"]); + }); + + it("opens nothing for an object ended before its first open started", async () => { + const opens = vi.fn(async () => null); + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open(opens, () => {}); + return { n: constant(1) }; + }), + ); + host.hold(Res, { id: 1 }, context).release(); + await flushed(); + expect(opens).not.toHaveBeenCalled(); + }); + + it("throws not_ready before its open resolved and released once its close started", async () => { + const reads: unknown[] = []; + const read = (native: Native): void => { + try { + reads.push(native.value); + } catch (error) { + reads.push(isChordError(error) ? error.toJSON() : error); + } + }; + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + const a: Native = self.open( + async () => { + read(b); + return "a"; + }, + () => read(a), + ); + const b: Native = self.open( + async () => "b", + () => read(a), + ); + read(a); + return { n: constant(1) }; + }), + ); + (await host.get(Res, { id: 1 }, context)).release(); + await flushed(); + expect(reads).toEqual([ + // In the factory, and b in a's opener. + { code: "not_ready", message: "app.res(1) native #1: not open yet" }, + { code: "not_ready", message: "app.res(1) native #2: not open yet" }, + // a in b's close (b closes first), then in a's own close. + "a", + { code: "released", reason: "closed", message: "app.res(1) native #1: closed" }, + ]); + }); + + it("a partly failed attempt closes what it opened, with reason end", async () => { + const closes: string[] = []; + const { host, errors } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open( + async () => "a", + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + self.open( + async () => { + throw new Error("offline"); + }, + () => void closes.push("b"), + ); + return { n: constant(1) }; + }), + ); + expect(await rejection(host.get(Res, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.res(1): offline", + }); + expect(closes).toEqual(["a end"]); + expect(failures(errors)).toEqual(["app.res(1): transient"]); + await host.close(context); + }); + + it.each(["end", "close"])( + "aborts the opener's context when the object ends while opening; a late value is closed once (%s)", + async (reason) => { + const opener = Promise.withResolvers(); + const late = Promise.withResolvers(); + const closes: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open( + async (ctx) => { + opener.resolve(ctx); + return late.promise; + }, + // The close runs under the object's context, not under the opener's aborted one. + (value, reason, ctx) => void closes.push([value, reason.kind, ctx.abortSignal?.aborted]), + ); + return { n: constant(1) }; + }), + ); + const { context: cancellable, cancel } = withCancel(context); + const got = host.get(Res, { id: 1 }, cancellable); + const openContext = await opener.promise; + if (reason === "end") cancel(new Error("left")); + else await host.close(context); + await expect(got).rejects.toThrow(reason === "end" ? "left" : "app.res(1): ended"); + expect(openContext.abortSignal?.aborted).toBe(true); + late.resolve("value"); + await flushed(); + await host.close(context); + expect(closes).toEqual([["value", reason, false]]); + }, + ); +}); + +describe("closing", () => { + it("closes in reverse, with reason end when the object ends and close when the host closes", async () => { + const closes: string[] = []; + const { host } = await loaded((h) => + h.serve(Res, ({ id }, self) => { + for (const name of ["a", "b"]) { + self.open( + async () => `${id}${name}`, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + } + return { n: constant(id) }; + }), + ); + const one = await host.get(Res, { id: 1 }, context); + await host.get(Res, { id: 2 }, context); + one.release(); + await flushed(); + await host.close(context); + expect(closes).toEqual(["1b end", "1a end", "2b close", "2a close"]); + }); + + it("collects every throwing close in host.close; an object ending in the background reports each", async () => { + const thrown = (name: string) => new Error(`close ${name} threw`); + const { host, errors } = await loaded((h) => + h.serve(Res, ({ id }, self) => { + for (const name of ["a", "b"]) { + self.open( + async () => name, + () => { + throw thrown(`${id}${name}`); + }, + ); + } + return { n: constant(id) }; + }), + ); + const one = await host.get(Res, { id: 1 }, context); + await host.get(Res, { id: 2 }, context); + one.release(); + await flushed(); + expect(errors).toEqual([ + { kind: "close_failed", object: "app.res(1)", native: { kind: "native", position: 2 }, error: thrown("1b") }, + { kind: "close_failed", object: "app.res(1)", native: { kind: "native", position: 1 }, error: thrown("1a") }, + ]); + const closed = await host.close(context).catch((error: unknown) => error); + expect(closed).toBeInstanceOf(AggregateError); + expect((closed as AggregateError).errors).toEqual([thrown("2b"), thrown("2a")]); + }); + + describe("timeouts", () => { + fakeTimers(); + + it("report a close still running after its closeTimeout, abort its context then, and close the others", async () => { + const events: string[] = []; + let slow: Context | undefined; + const { host, errors } = await loaded( + (h) => + h.serve(Res, (_, self) => { + self.open( + async () => "a", + () => void events.push("close a"), + ); + self.open( + async () => "b", + (_value, _reason, ctx) => { + slow = ctx; + return new Promise(() => {}); + }, + { closeTimeout: 500 }, + ); + return { n: constant(1) }; + }), + { closeTimeout: 10_000 }, + ); + (await host.get(Res, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(499); + expect([events, slow?.abortSignal?.aborted]).toEqual([[], false]); + await vi.advanceTimersByTimeAsync(1); + expect([events, slow?.abortSignal?.aborted]).toEqual([["close a"], true]); + expect(errors).toEqual([ + { kind: "close_timeout", object: "app.res(1)", native: { kind: "native", position: 2 } }, + ]); + }); + }); +}); + +describe("facets", () => { + it("cannot open natives", () => { + expectTypeOf().not.toHaveProperty("open"); + }); +}); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts new file mode 100644 index 000000000..d177122ea --- /dev/null +++ b/packages/chord/test/next/host/object.test.ts @@ -0,0 +1,1930 @@ +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, evolve, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { Attempt } from "../../../src/next/host/attempt.ts"; +import { + createHost, + defineFacet, + type FacetHost, + type FacetSelf, + type Host, + type HostError, + type Self, + type StoreSelf, + untilAborted, +} from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant, type Watchable } from "../../../src/next/state/index.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + failures, + fakeTimers, + flushed, + holdNextCommit, + instancesOf, + loaded, + memoryStore, + metered, + RawPeer, + rawWorkers, + rejection, + relayed, + restartable, + SERVER_ADDRESS, + Server, + Session, + StoreRestartable, + statesOf, + storeRestartable, + testCodec, + transported, + valuesOf, +} from "../support.ts"; + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const COUNTER = { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) } as const; +const Store = local("app.store", { + version: 1, + params: {}, + members: { count: int(), add: method(obj({ text: str() })), clear: method() }, +}); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); + +/** The store: a list document, its count, and methods to add and clear. */ +function serveStore(self: Self) { + const list = self.state(List); + return { + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }: { text: string }, ctx: Context) => { + await list.change((draft) => draft.items.push(text), ctx); + return null; + }, + clear: async (ctx: Context) => { + await list.retire(ctx); + return null; + }, + }; +} + +const Svc = local("app.svc", { version: 1, params: { id: int() }, members: { n: int() } }); +const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + +/** + * A factory with one native whose open throws "offline" on the attempts `fails` picks (counted from 1), and the times + * (fake, from 0) of its attempts. `gate`, when given, holds every open that does not fail until it resolves. + */ +function flaky(fails: (attempt: number) => boolean, gate?: Promise) { + const times: number[] = []; + const factory = (_: unknown, self: Self) => { + times.push(Date.now()); + const attempt = times.length; + self.open( + async () => { + if (fails(attempt)) throw new Error("offline"); + await gate; + return null; + }, + () => {}, + ); + return { n: constant(attempt) }; + }; + return { times, factory }; +} + +describe("identity and lifetime", () => { + it("one object per params key; a params type lives from its first get to its last release", async () => { + const factory = vi.fn(({ id }: { id: number }, self: Self) => { + const store = self.get(Store, {}); + return { + label: self.derive([store.count], ([count]) => `${id}: ${count}`), + slow: async () => null, + }; + }); + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, factory); + }); + const [a, b, c] = await Promise.all([ + host.get(Page, { id: 1 }, context), + host.get(Page, { id: 1 }, context), + host.get(Page, { id: 2 }, context), + ]); + expect(factory).toHaveBeenCalledTimes(2); + const watch = await a.label.watch(context); + a.release(); + expect(b.label.value).toBe("1: 0"); + b.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const again = await host.get(Page, { id: 1 }, context); + expect(factory).toHaveBeenCalledTimes(3); + expect([again.label.value, c.label.value]).toEqual(["1: 0", "2: 0"]); + }); + + it("keeps a type with empty params across releases", async () => { + const factory = vi.fn((_: object, self: Self) => serveStore(self)); + const { host } = await loaded((h) => h.serve(Store, factory)); + const first = await host.get(Store, {}, context); + await first.add({ text: "milk" }, context); + first.release(); + const second = await host.get(Store, {}, context); + expect(second.count.value).toBe(1); + expect(factory).toHaveBeenCalledTimes(1); + }); + + it("drops an ended object's documents and detaches its derives from other objects' members", async () => { + const telemetry = await metered(); + const { host } = await loaded( + (h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, ({ id }, self) => ({ + label: self.derive([self.get(Store, {}).count], ([count]) => `${id}: ${count}`), + slow: async () => null, + })); + }, + { telemetry: telemetry.telemetry }, + ); + const store = await host.get(Store, {}, context); + const page = await host.get(Page, { id: 1 }, context); + const watch = await page.label.watch(context); + page.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const before = telemetry.counts()["chord.derive.computes"]; + await store.add({ text: "x" }, context); + expect(telemetry.counts()["chord.derive.computes"]).toBe(before); + }); +}); + +describe("retries", () => { + fakeTimers(); + + it("retry a transient failure after 1 s, doubling up to 60 s, and start over after a success", async () => { + const { times, factory } = flaky((attempt) => attempt <= 8 || attempt === 10); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const got = host.get(Svc, { id: 1 }, context, { readyTimeout: 1_000_000 }); + await vi.advanceTimersByTimeAsync(183_000); + (await got).release(); + expect(times).toEqual([0, 1000, 3000, 7000, 15000, 31000, 63000, 123000, 183000]); + // The success deleted the backoff: a new object tries at once and waits 1 s after its failure. + const again = host.get(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(1000); + expect((await again).n.value).toBe(11); + expect(times.slice(9)).toEqual([183000, 184000]); + }); + + it("share one attempt among concurrent gets", async () => { + const { times, factory } = flaky(() => false); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const refs = await Promise.all([1, 2, 3].map(() => host.get(Svc, { id: 1 }, context))); + expect([times, refs.map(({ n }) => n.value)]).toEqual([[0], [1, 1, 1]]); + }); + + it("never retry a broken failure; a new object after its end tries at once", async () => { + const bug = vi.fn(() => { + throw new Error("bug"); + }); + const { host } = await loaded((h) => { + h.serve(Svc, bug); + h.serve(Core, bug); + }); + expect(await rejection(host.get(Svc, { id: 1 }, context))).toEqual({ + code: "unavailable", + message: "app.svc(1): bug", + }); + await vi.advanceTimersByTimeAsync(120_000); + expect(bug).toHaveBeenCalledTimes(2); + expect(await rejection(host.get(Core, {}, context))).toEqual({ code: "unavailable", message: "app.core: bug" }); + await expect(host.get(Svc, { id: 1 }, context)).rejects.toThrow("bug"); + expect(bug).toHaveBeenCalledTimes(3); + }); + + it("end a failed object with its last reference; its backoff entry expires on its own", async () => { + const { times, factory } = flaky(() => true); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + expect(await first).toEqual({ code: "unavailable", message: "app.svc(1): offline" }); + await vi.advanceTimersByTimeAsync(4500); + expect([times, vi.getTimerCount()]).toEqual([[0], 0]); + // The entry expired: a new object tries at once and its window starts over at 1 s. + const second = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 1500 })); + await vi.advanceTimersByTimeAsync(1500); + await second; + expect(times).toEqual([0, 5000, 6000]); + }); + + it("make a get right after a failure wait for the scheduled retry", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + await first; + const second = host.get(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(499); + expect(times).toEqual([0]); + await vi.advanceTimersByTimeAsync(1); + expect((await second).n.value).toBe(2); + expect(times).toEqual([0, 1000]); + }); + + it("start a new instance only after the previous one closed", async () => { + const closing = Promise.withResolvers(); + const factory = vi.fn((_: { id: number }, self: Self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { n: constant(factory.mock.calls.length) }; + }); + const { host } = await loaded((h) => h.serve(Svc, factory)); + (await host.get(Svc, { id: 1 }, context)).release(); + const again = host.get(Svc, { id: 1 }, context); + await flushed(); + expect(factory).toHaveBeenCalledTimes(1); + closing.resolve(); + expect((await again).n.value).toBe(2); + }); +}); + +describe("parked on dependencies (dependency model §3)", () => { + fakeTimers(); + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + + /** A view over `Type`'s `n`, and the times of its attempts. */ + function view(Type: typeof Svc | typeof Core) { + const times: number[] = []; + const factory = ({ id }: { id: number }, self: Self) => { + times.push(Date.now()); + return { n: Type === Core ? self.get(Core, {}).n : self.get(Svc, { id }).n }; + }; + return { times, factory }; + } + + it("parks a dependent of a failed dependency failed { waiting } naming it, with no timer; it goes on when the dependency is live", async () => { + const core = flaky((attempt) => attempt === 1); + const viewer = view(Core); + const { host, errors } = await loaded((h) => { + h.serve(Core, core.factory); + h.serve(View, viewer.factory); + }); + await vi.advanceTimersByTimeAsync(500); + const held = host.hold(View, { id: 1 }, context); + const got = host.get(View, { id: 1 }, context); + await flushed(); + expect(held.state.value).toEqual({ + kind: "failed", + cause: { + kind: "waiting", + error: expect.objectContaining({ message: "waiting for app.core: app.core: offline" }), + }, + retryAt: undefined, + }); + // A parked attempt is no failure of its own: nothing is reported for the view. + expect(failures(errors)).toEqual(["app.core: transient"]); + // The core's own backoff retries it at 1 s; the view goes on then, its factory not run again. + await vi.advanceTimersByTimeAsync(500); + expect((await got).n.value).toBe(2); + await vi.advanceTimersByTimeAsync(120_000); + expect(viewer.times).toEqual([500]); + held.release(); + }); + + it("keeps its references while parked; a getter rejects with the waiting cause at its deadline; releasing ends the wait", async () => { + const svc = flaky(() => true); + const viewer = view(Svc); + const { host } = await loaded((h) => { + h.serve(Svc, svc.factory); + h.serve(View, viewer.factory); + }); + const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 1500 })); + await vi.advanceTimersByTimeAsync(1500); + expect(await got).toEqual({ + code: "unavailable", + message: "waiting for app.svc(1): app.svc(1): offline", + }); + await vi.advanceTimersByTimeAsync(10_000); + // Held by the parked view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. + expect(svc.times).toEqual([0, 1000]); + expect(viewer.times).toEqual([0]); + }); + + it("shows every change of a dependency: loading, failed, a new cause, loading again, live", async () => { + let cores = 0; + const viewer = view(Core); + const { host } = await loaded((h) => { + h.serve(Core, (_, self) => { + const attempt = ++cores; + self.open( + async () => { + if (attempt < 3) throw new Error(`offline ${attempt}`); + }, + () => {}, + ); + return { n: constant(attempt) }; + }); + h.serve(View, viewer.factory); + }); + const held = host.hold(View, { id: 1 }, context); + await flushed(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.core: app.core: offline 1" } }, + }); + const states = statesOf(await held.state.watch(context)); + await vi.advanceTimersByTimeAsync(10_000); + // The core's retries at 1 s and 3 s: each attempt shows as loading, each failure as a new cause. + expect(states).toEqual(["loading", "failed: waiting for app.core: app.core: offline 2", "loading", "live"]); + expect(viewer.times).toEqual([0]); + held.release(); + }); + + it("passes a retry on to its broken dependency, once through a diamond", async () => { + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const Left = local("app.left", { version: 1, params: { id: int() }, members: { n: int() } }); + let cores = 0; + const { host } = await loaded((h) => { + h.serve(Core, () => { + if (++cores === 1) throw new Error("bug"); + return { n: constant(cores) }; + }); + h.serve(Left, (_, self) => ({ n: self.get(Core, {}).n })); + h.serve(Top, (_, self) => { + const a = self.get(Left, { id: 1 }).n; + const b = self.get(Left, { id: 2 }).n; + return { n: self.derive([a, b], ([x, y]) => x + y) }; + }); + }); + const top = host.hold(Top, {}, context); + await flushed(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.left(1): app.core: bug" } }, + }); + await vi.advanceTimersByTimeAsync(60_000); + // A broken dependency is not retried by any timer. + expect(cores).toBe(1); + top.retry(); + await flushed(); + expect(cores).toBe(2); + expect(top.state.value).toEqual({ kind: "live" }); + expect(top.n.value).toBe(4); + top.release(); + }); + + it("walks a diamond on one host once: a shared parked dependency passes a retry on once (audit A5)", async () => { + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const Left = local("app.left", { version: 1, params: { id: int() }, members: { n: int() } }); + const Mid = local("app.mid", { version: 1, params: {}, members: { n: int() } }); + let cores = 0; + const { host } = await loaded((h) => { + h.serve(Core, () => { + if (++cores === 1) throw new Error("bug"); + return { n: constant(cores) }; + }); + h.serve(Mid, (_, self) => ({ n: self.get(Core, {}).n })); + h.serve(Left, (_, self) => ({ n: self.get(Mid, {}).n })); + h.serve(Top, (_, self) => { + const a = self.get(Left, { id: 1 }).n; + const b = self.get(Left, { id: 2 }).n; + return { n: self.derive([a, b], ([x, y]) => x + y) }; + }); + }); + const top = host.hold(Top, {}, context); + await flushed(); + const passed = vi.spyOn(Attempt.prototype, "retryDependencies"); + try { + top.retry(); + // Top, both lefts, and the shared mid once (its second visit through the other left is ignored). + expect(passed).toHaveBeenCalledTimes(4); + } finally { + passed.mockRestore(); + } + await flushed(); + expect([cores, top.n.value]).toEqual([2, 4]); + top.release(); + }); + + it("fails every object of a cycle broken; none is retried", async () => { + const Ping = local("app.ping", { version: 1, params: {}, members: { n: int() } }); + const Pong = local("app.pong", { version: 1, params: {}, members: { n: int() } }); + const factories = { ping: vi.fn(), pong: vi.fn() }; + const { host, errors } = await loaded((h) => { + h.serve(Ping, (_, self) => { + factories.ping(); + return { n: self.get(Pong, {}).n }; + }); + h.serve(Pong, (_, self) => { + factories.pong(); + return { n: self.get(Ping, {}).n }; + }); + }); + expect(failures(errors).sort()).toEqual(["app.ping: broken", "app.pong: broken"]); + await vi.advanceTimersByTimeAsync(120_000); + expect([factories.ping.mock.calls.length, factories.pong.mock.calls.length]).toEqual([1, 1]); + expect(await rejection(host.get(Pong, {}, context))).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: app.ping → app.pong → app.ping", + }); + }); +}); + +describe("cycle checks", () => { + it("stay linear on shared dependencies (a layered graph, every object depending on the whole next layer)", async () => { + const Layer = local("app.layer", { + version: 1, + params: { layer: int(), index: int() }, + members: { depth: int() }, + }); + const layers = 28; + const { host } = await loaded((h) => + h.serve(Layer, ({ layer }, self) => { + if (layer === layers) return { depth: constant(0) }; + const next = [0, 1].map((index) => self.get(Layer, { layer: layer + 1, index }).depth); + return { depth: self.derive(next, ([a]) => (a as number) + 1) }; + }), + ); + const telemetry = await metered(); + const top = await host.get(Layer, { layer: 0, index: 0 }, telemetry.context); + expect(top.depth.value).toBe(layers); + // 55 objects, 2 waits each: a search per wait visits at most every object once (an exponential one 2^28 chains). + expect(telemetry.counts()["chord.cycle.visits"]).toBeLessThanOrEqual(2 * 55 * 55); + }); +}); + +describe("GC", () => { + it("collects an ended object with its documents, derives and watches", async () => { + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + const list = self.state(List); + const label = self.derive([list, self.get(Store, {}).count], ([l, c]) => `${l.items.length}/${c}`); + refs.push(new WeakRef(self), new WeakRef(list), new WeakRef(label)); + return { label, slow: async () => null }; + }); + }); + const store = await host.get(Store, {}, context); + await (async () => { + const page = await host.get(Page, { id: 1 }, context); + const watch = await page.label.watch(context); + refs.push(new WeakRef(watch), new WeakRef(page)); + page.release(); + })(); + await store.add({ text: "x" }, context); + await delivered(); + await expectCollected(...refs); + }); + + it("keeps nothing in a released reference or a stopped watch", async () => { + const { host } = await loaded((h) => h.serve(Store, (_, self) => serveStore(self))); + const store = await host.get(Store, {}, context); + const held = await host.get(Store, {}, context); + const refs = await (async () => { + const watch = await store.count.watch(context); + await watch.stop(); + const reference = await host.get(Store, {}, context); + const watched: Watchable = reference.count; + await watched.watch(context); + reference.release(); + return [new WeakRef(watch), new WeakRef(reference)]; + })(); + await held.add({ text: "x" }, context); + await expectCollected(...refs); + }); +}); + +describe("GC of released references", () => { + it("keeps neither the declaring object nor the host in a released self.get reference", async () => { + const { reference, refs } = await (async () => { + const refs: WeakRef[] = []; + let declared: unknown; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + declared = self.get(Store, {}); + refs.push(new WeakRef(self)); + return { label: constant(""), slow: async () => null }; + }); + }); + (await host.get(Page, { id: 1 }, context)).release(); + await flushed(); + refs.push(new WeakRef(host)); + return { reference: declared, refs }; + })(); + await expectCollected(...refs); + expect(reference).toBeDefined(); + }); +}); + +describe("GC of failed objects", () => { + fakeTimers(); + + it("collects one after its last reference while its backoff entry exists; the entry expires on its own", async () => { + const refs: WeakRef[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + const list = self.state(List); + const label = self.derive([list, self.get(Store, {}).count], ([l, c]) => `${l.items.length}/${c}`); + const native = self.open( + async () => ({ buffer: new Uint8Array(16) }), + () => {}, + ); + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + const slow = async () => null; + refs.push(new WeakRef(self), new WeakRef(list), new WeakRef(label), new WeakRef(native), new WeakRef(slow)); + return { label, slow }; + }); + }); + const got = rejection(host.get(Page, { id: 1 }, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + await got; + expect(vi.getTimerCount()).toBe(1); + // The reports' errors keep their stack frames' receivers until formatted (V8): an app logs and drops them. + errors.length = 0; + await expectCollected(...refs); + await vi.advanceTimersByTimeAsync(900); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("GC of a parked chain", () => { + it("collects a chain parked on a type not served, released from its top: objects, attempts and index entries", async () => { + const Top = local("app.top", { version: 1, params: { id: int() }, members: { n: int() } }); + const Mid = local("app.mid", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Mid, (_, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Missing, {}).n }; + }); + h.serve(Top, ({ id }, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Mid, { id }).n }; + }); + }); + const top = host.hold(Top, { id: 1 }, context); + await flushed(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { message: "waiting for app.mid(1): app.missing is not served here" }, + }, + }); + top.release(); + await flushed(); + expect(refs).toHaveLength(2); + await expectCollected(...refs); + expect(host).toBeDefined(); + }); +}); + +describe("ref.retry()", () => { + fakeTimers(); + + it("skips a transient failure's backoff window: an attempt now", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const svc = host.hold(Svc, { id: 1 }, context); + await flushed(); + expect(svc.state.value.kind).toBe("failed"); + vi.advanceTimersByTime(200); + svc.retry(); + await flushed(); + expect([times, svc.state.value, svc.n.value]).toEqual([[0, 200], { kind: "live" }, 2]); + }); + + it("starts an object waiting for its identity's backoff window at once", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = host.hold(Svc, { id: 1 }, context); + await flushed(); + // The failed object ends with its reference; its identity's window (1 s) still runs. + first.release(); + vi.advanceTimersByTime(100); + const second = host.hold(Svc, { id: 1 }, context); + await flushed(); + expect([times, second.state.value]).toEqual([[0], { kind: "loading" }]); + second.retry(); + await flushed(); + expect([times, second.state.value]).toEqual([[0, 100], { kind: "live" }]); + }); + + it("is ignored while an attempt runs and once live (race 11)", async () => { + const gate = Promise.withResolvers(); + const { times, factory } = flaky((attempt) => attempt === 1, gate.promise); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const svc = host.hold(Svc, { id: 1 }, context); + await flushed(); + svc.retry(); + svc.retry(); + await flushed(); + expect(times).toEqual([0, 0]); + gate.resolve(); + await flushed(); + svc.retry(); + await vi.advanceTimersByTimeAsync(5_000); + expect([times, svc.state.value]).toEqual([[0, 0], { kind: "live" }]); + }); + + it("renews a broken object for every reference (race 13)", async () => { + let fixed = false; + const factory = vi.fn((_: { id: number }, self: Self) => { + self.open( + async () => { + if (!fixed) throw broken(new Error("no configuration")); + }, + () => {}, + ); + return { n: constant(1) }; + }); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const one = host.hold(Svc, { id: 1 }, context); + const two = host.hold(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(120_000); + expect([one.state.value.kind, two.state.value.kind, factory.mock.calls.length]).toEqual(["failed", "failed", 1]); + fixed = true; + one.retry(); + expect(two.state.value).toEqual({ kind: "loading" }); + await flushed(); + expect([one.state.value, two.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); +}); + +describe("linger", () => { + fakeTimers(); + const Tally = local("app.tally", { version: 1, params: { id: int() }, members: { n: int(), bump: method() } }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + + /** Serves `Tally` with `linger`: a document, a derive that fails at 2, a native whose closes `closes` records. */ + async function tallies(linger: number, open: () => Promise = async () => {}) { + const closes: string[] = []; + const factory = vi.fn(({ id }: { id: number }, self: Self) => { + const doc = self.state(N); + self.open(open, (_, reason) => void closes.push(`${id} ${reason.kind}`)); + return { + n: self.derive([doc], ([{ n }]) => { + if (n === 2) throw new Error("two"); + return n; + }), + bump: async (ctx: Context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; + }); + const { host } = await loaded((h) => h.serve(Tally, factory, { linger })); + return { host, factory, closes }; + } + + it("keeps a live object live for linger after its last release: a get within it reuses it, with its documents", async () => { + const { host, factory, closes } = await tallies(500); + const first = await host.get(Tally, { id: 1 }, context); + await first.bump(context); + first.release(); + await vi.advanceTimersByTimeAsync(499); + const second = await host.get(Tally, { id: 1 }, context); + expect([second.n.value, factory.mock.calls.length, closes]).toEqual([1, 1, []]); + second.release(); + await vi.advanceTimersByTimeAsync(500); + expect(closes).toEqual(["1 end"]); + expect((await host.get(Tally, { id: 1 }, context)).n.value).toBe(0); + expect(factory).toHaveBeenCalledTimes(2); + }); + + it("a reuse cancels the expiry: the holder count decides, a later release starts a new window (race 8)", async () => { + const { host, closes } = await tallies(500); + (await host.get(Tally, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(499); + const held = await host.get(Tally, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + held.release(); + await vi.advanceTimersByTimeAsync(499); + expect(closes).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(closes).toEqual(["1 end"]); + }); + + it("only a live object lingers: an activating or failed one ends with its last reference (race 9)", async () => { + const gate = Promise.withResolvers(); + const activating = await tallies(500, () => gate.promise); + const opening = activating.host.hold(Tally, { id: 1 }, context); + await flushed(); + opening.release(); + gate.resolve(); + await flushed(); + // Its open's late value was closed at once. + expect(activating.closes).toEqual(["1 end"]); + const failing = await tallies(500, async () => { + throw broken(new Error("no configuration")); + }); + const failed = failing.host.hold(Tally, { id: 1 }, context); + await flushed(); + failed.release(); + failing.host.hold(Tally, { id: 1 }, context); + await flushed(); + expect(failing.factory).toHaveBeenCalledTimes(2); + }); + + it("a failed state member does not end a lingering object", async () => { + const { host, factory, closes } = await tallies(500); + const tally = await host.get(Tally, { id: 1 }, context); + await tally.bump(context); + await tally.bump(context); + expect(() => tally.n.value).toThrow("two"); + tally.release(); + await vi.advanceTimersByTimeAsync(400); + await host.get(Tally, { id: 1 }, context); + expect([factory.mock.calls.length, closes]).toEqual([1, []]); + }); + + it("host.close ends a lingering object at once", async () => { + const { host, closes } = await tallies(60_000); + (await host.get(Tally, { id: 1 }, context)).release(); + await host.close(context); + expect([closes, vi.getTimerCount()]).toEqual([["1 close"], 0]); + }); + + it("a lingering object is collected after its expiry", async () => { + const refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve( + Svc, + ({ id }, self) => { + refs.push(new WeakRef(self)); + return { n: constant(id) }; + }, + { linger: 100 }, + ), + ); + (await host.get(Svc, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(100); + vi.useRealTimers(); + await expectCollected(...refs); + }); +}); + +/** Every object attempted in this file, weakly: the objects of the cycles among them. */ +const attempted = instancesOf(ChordObject, "attempt"); + +/** On the server: its activation holds `Far` on the session's worker. */ +const Near = remote("app.near", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +/** On the session's worker: its activation holds `Near` back on the server, or, with `local`, `Middle`. */ +const Far = remote("app.far", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, +}); + +/** On the server, held by `Far`: its activation holds `Hub` on its own host (a cycle closed on one host). */ +const Middle = remote("app.middle", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +/** On the server, a local type (no principal in its identity): its activation holds `Far` as the host. */ +const Hub = local("app.hub", { version: 1, params: { session: str() }, members: { n: int() } }); + +/** `Loop({ here, there })` lives on `here`'s worker and holds `Loop({ here: there, there: here })`. */ +const Loop = remote("app.loop", { + version: 1, + params: { here: str(), there: str() }, + hostAddress: ({ here }) => Session({ id: here }), + variants: { v: { state: { n: int() } } }, +}); + +/** A tab, a server and its workers over trusted route links, serving the cycle types (`Far` holds `back`). */ +async function cyclic(back: "near" | "middle" = "near", trusted = true) { + return relayed({ + trusted, + server: (h) => { + h.serve(Near, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Far, { session }).n }, + })); + h.serve(Middle, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Hub, { session }).n }, + })); + h.serve(Hub, ({ session }, self) => ({ n: self.get(Far, { session }).n })); + }, + worker: (h) => { + h.serve(Far, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: back === "near" ? self.get(Near, { session }).n : self.get(Middle, { session }).n }, + })); + h.serve(Loop, ({ here, there }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Loop, { here: there, there: here }).n }, + })); + }, + }); +} + +describe("cycles across hosts (Q5)", () => { + it("refuse a hold whose chain names the activation it would bind: server → worker → server", async () => { + const { tab, errors } = await cyclic(); + const error = await rejection(tab.get(Near, { session: "S1" }, context)); + // The worker's hold back acts as the worker (upward boundary): it meets the cycle one turn later, at that + // principal's `Near`. + const message = "dependency cycle: app.near(S1) → … → app.near(S1)"; + expect(error).toEqual({ code: "invalid", reason: "cycle", message }); + // Proposal: every object on the cycle fails broken with the same error, none transient (nothing retries it). + const failed = new Set(failures([...errors.server, ...errors.worker])); + expect([...failed].sort()).toEqual(["app.far(S1): broken", "app.near(S1): broken"]); + }); + + it("find a cycle closed on one host through another: server → worker → server, then a local get", async () => { + const { server } = await cyclic("middle"); + const error = await rejection(server.get(Hub, { session: "S1" }, context)); + expect(error).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: app.hub(S1) → … → app.middle(S1) → app.hub(S1)", + }); + }); + + it("carry the chain through a relay (race 7): worker → server → worker → server → worker", async () => { + const { tab, errors } = await cyclic(); + const error = await rejection(tab.get(Loop, { here: "S1", there: "S2" }, context)); + expect(error).toMatchObject({ code: "invalid", reason: "cycle" }); + expect([...new Set(failures(errors.worker))].sort()).toEqual([ + "app.loop(S1|S2): broken", + "app.loop(S2|S1): broken", + ]); + }); + + it("drop the chain from holds arriving on untrusted links, and send none for holds made outside an activation", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b); + const forged = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; + tab.send({ ...farHold(0), chain: [forged] }); + await transported(); + await transported(); + const held = await workers[0]!.next("hold"); + expect(held.chain).toBeUndefined(); + relay.tab.hold(Far, { session: "S1" }, context); + expect((await workers[0]!.next("hold")).chain).toBeUndefined(); + }); + + it("never match a restarted object: its new instance has a new id (race 8)", async () => { + // The raw worker holds `Near` on the server itself (as the worker), whose activation holds `Far` back with its + // chain. The raw worker never binds `Far`. + const { worker } = await nearBehindRawWorker(); + const chainOf = async () => (await worker.next("hold")).chain!; + worker.send(nearHold(0)); + const [old] = await chainOf(); + // Released: that instance ends. The next hold makes a new instance with a new id. + worker.send({ kind: "release", hold: 0 }); + worker.send(nearHold(1)); + const [renewed] = await chainOf(); + expect(renewed).not.toBe(old); + // A hold naming the old instance binds to the new one (it is pending there); one naming the new one is a cycle. + worker.send({ ...nearHold(2), chain: [old!] }); + worker.send({ ...nearHold(3), chain: [renewed!] }); + const failed = await worker.next("failed"); + expect(failed).toMatchObject({ + hold: 3, + cause: { kind: "broken", error: { kind: "chord", error: { reason: "cycle" } } }, + }); + await transported(); + expect(worker.received.filter((message) => message.kind === "failed")).toEqual([]); + }); + + it("keep the last 32 entries of a longer chain: the oldest is dropped (a longer cycle goes undetected)", async () => { + const { worker } = await nearBehindRawWorker(); + const forged = Array.from( + { length: 32 }, + (_, index) => `0190f5a2-7c3e-7d4a-9b1c-${String(index).padStart(12, "0")}`, + ); + worker.send({ ...nearHold(0), chain: forged }); + const { chain } = await worker.next("hold"); + expect(chain).toHaveLength(32); + expect(chain!.slice(0, 31)).toEqual(forged.slice(1)); + expect(forged).not.toContain(chain![31]); + }); + + it("miss a cycle entering an activation that started already (documented limitation): both wait, transient", async () => { + const { relay, worker } = await nearBehindRawWorker(); + // The server's `Near` activates for the tab and holds `Far` with its chain. + const getting = rejection(relay.tab.get(Near, { session: "S1" }, context, { readyTimeout: 100 })); + const { chain } = await worker.next("hold"); + // The worker's own activation, started at the same moment, holds that `Near` back with a chain of its own. + worker.send({ ...nearHold(0), chain: ["0190f5a2-7c3e-7d4a-9b1c-000000000001"] }); + await transported(); + expect(chain).toHaveLength(1); + expect(worker.received.filter((message) => message.kind === "failed")).toEqual([]); + expect(await getting).toMatchObject({ code: "unavailable" }); + }); + + it("keep no object, and so no chain, once released (GC)", async () => { + const refs = await (async () => { + const made = attempted.length; + const { tab, server } = await cyclic(); + await rejection(tab.get(Near, { session: "S1" }, context)); + await transported(); + const refs = attempted.slice(made); + expect(refs.length).toBeGreaterThanOrEqual(4); + await tab.close(context); + await server.close(context); + return refs; + })(); + await expectCollected(...refs); + }); +}); + +/** + * A server serving `Near` (its activation holds `Far`) and a raw worker for S1 over a trusted route link, made by a + * first hold of `Far` and with that hold taken. + */ +async function nearBehindRawWorker() { + const workers: RawPeer[] = []; + const relay = await relayed({ + resolve: rawWorkers(workers, { trusted: true }), + server: (h) => + h.serve(Near, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Far, { session }).n }, + })), + }); + relay.tab.hold(Far, { session: "S1" }, context); + const worker = await (async () => { + for (let tries = 0; workers[0] === undefined && tries < 10; tries++) await transported(); + return workers[0]!; + })(); + await worker.next("hold"); + return { relay, worker }; +} + +/** A hold of `Far` for S1, as a tab sends it. */ +function farHold(hold: number): Extract { + const { name: type, version, fingerprint } = Far; + const address = { kind: "session", params: { id: "S1" } }; + return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }), address }; +} + +/** A hold of `Near` for S1, as a worker sends it to the server. */ +function nearHold(hold: number): Extract { + const { name: type, version, fingerprint } = Near; + return { + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; +} + +describe("a failed activation's cleanup (audit)", () => { + it("closes the natives that opened before it releases its references: a close may use them", async () => { + const Dependent = local("app.dependent", { version: 1, params: { id: int() }, members: { n: int() } }); + const seen: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Core, () => ({ n: constant(5) })); + h.serve(Dependent, (_, self) => { + const core = self.get(Core, {}); + self.open( + async () => null, + () => { + try { + seen.push(core.n.value); + } catch (error) { + seen.push(String(error)); + } + }, + ); + self.open( + async () => { + throw broken(new Error("open failed")); + }, + () => {}, + ); + return { n: core.n }; + }); + }); + await expect(host.get(Dependent, { id: 1 }, context)).rejects.toThrow("open failed"); + await flushed(); + expect(errors.find(({ kind }) => kind === "activation_failed")).toMatchObject({ failure: { kind: "broken" } }); + expect(seen).toEqual([5]); + await host.close(context); + }); +}); + +describe("a live object failing (slice 6a §1.4)", () => { + it("fails broken: calls reject with the cause, one object_failed report, natives close, then references go", async () => { + const events: string[] = []; + let list: Watchable | undefined; + const { reference, fail, errors } = await restartable( + attempted, + (_, self) => { + list = self.state(List); + const core = self.get(Core, {}); + self.open( + async () => null, + () => { + events.push(`close sees core ${core.n.value}`); + }, + ); + }, + { serve: (h) => h.serve(Core, () => ({ n: constant(1) })) }, + ); + const states: string[] = []; + (await reference.state.watch(context)).start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); + }); + fail(); + const touched = rejection(reference.touch(context)); + expect(await touched).toMatchObject({ code: "unavailable", message: "boom" }); + expect(errors).toEqual([expect.objectContaining({ kind: "object_failed", object: "app.restartable(1)" })]); + // A failure of the sealed attempt is ignored. + fail(); + await flushed(); + expect(events).toEqual(["close sees core 1"]); + expect(errors.length).toBe(1); + expect(list!.value).toEqual({ items: [] }); + await delivered(); + expect(states).toEqual(["failed"]); + }); + + it("keeps its state members' last values for same-host reads; a live dependent's derive over them stays live", async () => { + const Viewer = local("app.count-viewer", { version: 1, params: {}, members: { tens: int() } }); + let store: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => { + store = self; + return serveStore(self); + }); + h.serve(Viewer, (_, self) => ({ tens: self.derive([self.get(Store, {}).count], ([n]) => n * 10) })); + }); + const ref = await host.get(Store, {}, context); + await ref.add({ text: "a" }, context); + const viewer = await host.get(Viewer, {}, context); + const tens = changesOf(await viewer.tens.watch(context)); + // A change committed in the turn of the failure: the frozen member computes it. + await ref.add({ text: "b" }, context); + failLive(attempted, store!, new ChordError({ code: "unavailable", message: "boom" })); + expect(ref.count.value).toBe(2); + expect(await rejection(ref.add({ text: "c" }, context))).toMatchObject({ message: "boom" }); + expect(viewer.state.value).toEqual({ kind: "live" }); + expect(viewer.tens.value).toBe(20); + await delivered(); + expect(valuesOf(tens)).toEqual([20]); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + // A retry takes the new attempt's members. + ref.retry(); + (await host.get(Store, {}, context)).release(); + expect(ref.count.value).toBe(2); + await host.close(context); + }); + + it("lets a commit admitted before the failure adopt and publish once, derives and dependents included", async () => { + const Counter = defineState("counter", { ...COUNTER, persist: true, checkpointWhen: () => false }); + const Doubled = local("app.doubled", { version: 1, params: {}, members: { double: int() } }); + const Viewer = local("app.double-viewer", { version: 1, params: {}, members: { tens: int() } }); + const store = await memoryStore(); + let self: Self | undefined; + let counter: SingletonHandle<{ n: number }> | undefined; + const { host } = await loaded( + (h) => { + h.serve(Doubled, (_, s) => { + self = s; + counter = s.state(Counter); + return { double: s.derive([counter], ([c]) => c.n * 2) }; + }); + h.serve(Viewer, (_, s) => ({ tens: s.derive([s.get(Doubled, {}).double], ([n]) => n * 10) })); + }, + { store }, + ); + const ref = await host.get(Doubled, {}, context); + const viewer = await host.get(Viewer, {}, context); + const doubles = changesOf(await ref.double.watch(context)); + const tens = changesOf(await viewer.tens.watch(context)); + const held = holdNextCommit(store); + const change = counter!.change((draft) => { + draft.n = 1; + }, context); + await held.entered; + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + await delivered(); + expect(ref.state.value.kind).toBe("failed"); + expect([counter!.value, ref.double.value, viewer.tens.value]).toEqual([{ n: 1 }, 2, 20]); + // The admitted commit publishes once; then the reference's member holds it (the failure once). A derive over it + // keeps its last value. + expect(valuesOf(doubles)).toEqual([2, { failed: "boom" }]); + expect(valuesOf(tens)).toEqual([20]); + await host.close(context); + await store.close(context); + }); + + it("starts a retry's factory only once a commit admitted before the failure adopted", async () => { + const V1 = defineState("counter", { ...COUNTER, persist: true, checkpointWhen: () => false }); + const V2 = defineState("counter", { + schema: obj({ m: int() }), + version: 2, + initial: () => ({ m: 0 }), + migrate: (old) => ({ m: (old as { n: number }).n }), + persist: true, + checkpointWhen: () => false, + }); + const store = await memoryStore(); + const counters: SingletonHandle<{ n: number } | { m: number }>[] = []; + const { host, reference, attempts } = await storeRestartable( + attempted, + (attempt, self) => void counters.push(self.state(attempt === 1 ? V1 : V2)), + { store, closeTimeout: 5 }, + ); + const v1 = counters[0] as SingletonHandle<{ n: number }>; + await v1.change((draft) => { + draft.n = 1; + }, context); + const held = holdNextCommit(store); + const change = v1.change((draft) => { + draft.n = 2; + }, context); + await held.entered; + failLive(attempted, attempts[0]!, new ChordError({ code: "unavailable", message: "boom" })); + // Past the drain: only the admitted step holds the retry. + await new Promise((resolve) => setTimeout(resolve, 20)); + reference.retry(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(counters).toHaveLength(1); + held.release(); + await change; + (await host.get(StoreRestartable, {}, context)).release(); + expect(counters[1]!.value).toEqual({ m: 2 }); + await host.close(context); + await store.close(context); + }); + + it("is recovered by ref.retry(): a new attempt, the documents as they were", async () => { + const lists: SingletonHandle<{ items: string[] }>[] = []; + const { restart } = await restartable(attempted, (_, self) => { + lists.push(self.state(List)); + }); + await lists[0]!.change((draft) => void draft.items.push("a"), context); + await restart(); + expect(lists[1]!.value).toEqual({ items: ["a"] }); + }); +}); + +/** Reloads `host`'s plugin `@test/app` (as `loaded` names it) with a facet whose setup is `setup`. */ +function reloadApp(host: Host, setup: (host: FacetHost, self: FacetSelf) => void) { + return host.reload([{ name: "@test/app", facet: defineFacet(setup) }], context); +} + +/** A local object with empty params: a counter document, `bump` (which waits for `gate` when given) and `label`. */ +const Pure = local("app.pure", { + version: 1, + params: {}, + members: { n: int(), label: str(), bump: method(undefined, int()) }, +}); + +/** Serves `Pure` labelled `label`; `bump` adds `by`, first awaiting `gates.shift()` when one is queued. */ +function servePure(h: FacetHost, label: string, by: number, gates: Promise[] = []): void { + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + return { + n: self.derive([doc], ([d]) => d.n), + label: constant(label), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += by; + }, ctx); + return doc.value.n; + }, + }; + }); +} + +describe("reload: fates (slice 7b §4)", () => { + it("swaps a pure object in place: same documents and instance id, new code for new calls", async () => { + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + const objects = instancesOf(ChordObject, "reloadFate"); + const report = await reloadApp(host, (h) => servePure(h, "v2", 10)); + expect(report.objects).toEqual([{ object: "app.pure", kind: "swapped" }]); + expect(objects).toHaveLength(1); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("v2"); + expect(await pure.bump(context)).toBe(11); + await host.close(context); + }); + + it("restarts an object with natives: the drain first, its natives closed with reason reload, a new instance", async () => { + const closes: string[] = []; + const serve = (h: FacetHost, label: string, gates: Promise[] = []) => + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + self.open( + async () => label, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + return { + n: self.derive([doc], ([d]) => d.n), + label: constant(label), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += 1; + }, ctx); + closes.push(`bump done on ${label}`); + return doc.value.n; + }, + }; + }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serve(h, "v1", [gate.promise])); + const pure = await host.get(Pure, {}, context); + const running = pure.bump(context); + await flushed(); + const report = await reloadApp(host, (h) => serve(h, "v2")); + expect(report.objects).toEqual([{ object: "app.pure", kind: "restarted" }]); + expect(pure.state.value).toEqual({ kind: "loading" }); + // A new call parks until the new attempt is live. + const parked = pure.bump(context); + await flushed(); + expect(closes).toEqual([]); + gate.resolve(); + expect(await running).toBe(1); + expect(await parked).toBe(2); + expect(closes).toEqual(["bump done on v1", "v1 reload", "bump done on v2"]); + expect(pure.label.value).toBe("v2"); + await host.close(context); + }); + + it("ends the objects of a removed type: its holders wait not_served, and bind again when it is served again", async () => { + const { host, errors } = await loaded((h) => servePure(h, "v1", 1)); + const pure = host.hold(Pure, {}, context); + await flushed(); + await pure.bump(context); + const label = await pure.label.watch(context); + const frames = changesOf(label); + const report = await host.reload([], context); + expect(report).toEqual({ + plugins: { added: [], changed: [], removed: ["@test/app"] }, + failed: [], + objects: [{ object: "app.pure", kind: "ended" }], + }); + expect(pure.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "app.pure is not served here" } }, + }); + // The member keeps its last value while waiting. + expect(pure.label.value).toBe("v1"); + await host.reload([{ name: "@test/app", facet: defineFacet((h) => servePure(h, "v2", 1)) }], context); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + // A new object: in-memory documents went with the old one. + expect(pure.n.value).toBe(0); + await delivered(); + // The removal's failure once, then the new object's value, whole. + expect(valuesOf(frames)).toEqual([{ failed: "app.pure is not served here" }, "v2"]); + expect(failures(errors)).toEqual([]); + await host.close(context); + }); + + it("a live local dependent of a removed type keeps its last value; served again, its derive heals (one frame)", async () => { + const Panel = local("app.panel", { version: 1, params: {}, members: { text: str() } }); + const panel = defineFacet((h) => + h.serve(Panel, (_, self) => { + const pure = self.get(Pure, {}); + return { text: self.derive([pure.label], ([label]) => `panel ${label}`) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: defineFacet((h) => servePure(h, "v1", 1)) }, + { name: "@test/panel", facet: panel }, + ], + context, + ); + const view = await host.get(Panel, {}, context); + const text = changesOf(await view.text.watch(context)); + await host.reload([{ name: "@test/panel", facet: panel }], context); + expect(view.state.value).toEqual({ kind: "live" }); + expect(view.text.value).toBe("panel v1"); + await host.reload( + [ + { name: "@test/app", facet: defineFacet((h) => servePure(h, "v2", 1)) }, + { name: "@test/panel", facet: panel }, + ], + context, + ); + await flushed(); + expect(view.text.value).toBe("panel v2"); + await delivered(); + expect(valuesOf(text)).toEqual(["panel v2"]); + await host.close(context); + }); + + it("retries a failed object on the new code", async () => { + const { host } = await loaded((h) => + h.serve(Pure, () => { + throw new Error("bug"); + }), + ); + const pure = host.hold(Pure, {}, context); + await flushed(); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + const report = await reloadApp(host, (h) => servePure(h, "fixed", 1)); + expect(report.objects).toEqual([{ object: "app.pure", kind: "retried" }]); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("fixed"); + await host.close(context); + }); + + it("parks a swap whose new factory needs a missing dependency; it goes live when the dependency is served", async () => { + const Dep = local("app.dep", { version: 1, params: {}, members: { name: str() } }); + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + const needsDep = (h: FacetHost) => + h.serve(Pure, (_, self) => { + const dep = self.get(Dep, {}); + return { + n: constant(0), + label: self.derive([dep.name], ([name]) => `with ${name}`), + bump: async () => 0, + }; + }); + const report = await reloadApp(host, needsDep); + expect(report.objects).toEqual([ + { + object: "app.pure", + kind: "parked", + cause: { kind: "waiting", error: expect.objectContaining({ message: expect.stringContaining("app.dep") }) }, + }, + ]); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(await rejection(pure.bump(context))).toMatchObject({ code: "unavailable" }); + await host.reload( + [ + { name: "@test/app", facet: defineFacet(needsDep) }, + { name: "@test/dep", facet: defineFacet((h) => h.serve(Dep, () => ({ name: constant("dep") }))) }, + ], + context, + ); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("with dep"); + await host.close(context); + }); +}); + +describe("reload: the pure swap (slice 7b §5)", () => { + it("lets a running call finish on the old attempt's handles; a new call waits at the gate for the new code", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise])); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => servePure(h, "v2", 10)); + // The old call commits through its old handle after the swap. + gate.resolve(); + expect(await old).toBe(1); + expect(await pure.bump(context)).toBe(11); + await host.close(context); + }); + + it("an old call touching a document whose version rose throws once (R15); the document is migrated", async () => { + const V1 = defineState("doc", COUNTER); + const V2 = defineState("doc", { + schema: obj({ n: int(), unit: str() }), + version: 2, + initial: () => ({ n: 0, unit: "" }), + migrate: (old) => ({ n: old.n as number, unit: "pcs" }), + }); + const gate = Promise.withResolvers(); + const serve = (h: FacetHost, definition: typeof V1 | typeof V2, gates: Promise[]) => + h.serve(Pure, (_, self) => { + const doc = self.state(definition as typeof V1); + return { + n: self.derive([doc], ([d]) => d.n), + label: self.derive([doc], ([d]) => JSON.stringify(d)), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }; + }); + const { host } = await loaded((h) => serve(h, V1, [Promise.resolve(), gate.promise])); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => serve(h, V2, [])); + await flushed(); + expect(pure.label.value).toBe('{"n":1,"unit":"pcs"}'); + gate.resolve(); + expect(await rejection(old)).toMatchObject({ code: "released", reason: "version" }); + expect(await pure.bump(context)).toBe(2); + await host.close(context); + }); + + it("binds the new attempt's requests before releasing the old one's: a dependency held only by it never ends", async () => { + const Dep = local("app.dep", { + version: 1, + params: { id: int() }, + members: { n: int(), set: method(obj({ n: int() })) }, + }); + const dep = defineFacet((h) => + h.serve(Dep, (_, self) => { + const doc = self.state(defineState("dep", COUNTER)); + return { + n: self.derive([doc], ([d]) => d.n), + set: async ({ n }: { n: number }, ctx: Context) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const app = (label: string) => + defineFacet((h) => + h.serve(Pure, (_, self) => { + const held = self.get(Dep, { id: 1 }); + return { + n: self.derive([held.n], ([n]) => n), + label: constant(label), + bump: async (ctx: Context) => { + await held.set({ n: 7 }, ctx); + return 7; + }, + }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: app("v1") }, + { name: "@test/dep", facet: dep }, + ], + context, + ); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + await host.reload( + [ + { name: "@test/app", facet: app("v2") }, + { name: "@test/dep", facet: dep }, + ], + context, + ); + await flushed(); + // The dependency's in-memory document is still there: its object never ended in between. + expect(pure.label.value).toBe("v2"); + expect(pure.n.value).toBe(7); + await host.close(context); + }); + + it("re-binds members by name (one frame from the last value) and rebuilds internal derives", async () => { + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + const label = changesOf(await pure.label.watch(context)); + const n = changesOf(await pure.n.watch(context)); + await pure.bump(context); + await reloadApp(host, (h) => servePure(h, "v2", 1)); + await delivered(); + expect(label).toEqual([{ kind: "value", value: "v2", ops: [["r", "v2"]], retired: false }]); + // Its derive over the same document gave the same value: no frame besides the commit's. + expect(valuesOf(n)).toEqual([1]); + await host.close(context); + }); + + it("hands its routines over: the old ones aborted, the new ones start after the old calls finished", async () => { + const events: string[] = []; + const release = Promise.withResolvers(); + const serve = (h: FacetHost, label: string) => + h.serve(Pure, (_, self) => { + self.onLive(async (ctx) => { + events.push(`${label} started`); + await untilAborted(ctx); + if (label === "v1") await release.promise; + events.push(`${label} stopped`); + }); + return { n: constant(0), label: constant(label), bump: async () => 0 }; + }); + const { host } = await loaded((h) => serve(h, "v1")); + await host.get(Pure, {}, context); + await flushed(); + await reloadApp(host, (h) => serve(h, "v2")); + await flushed(); + expect(events).toEqual(["v1 started"]); + release.resolve(); + await flushed(); + expect(events).toEqual(["v1 started", "v1 stopped", "v2 started"]); + await host.close(context); + }); + + it("starts a writable at its new initial value and replaces the contributions in one batch", async () => { + const Tools = defineRegistry("app.tools", { version: 1 }); + const compositions: string[][] = []; + const registry = defineFacet((h) => + h.serve(Tools, { + compose: (table) => { + const value = table.flatMap((entry) => entry.value); + compositions.push(value); + return value; + }, + }), + ); + const app = (tool: string) => + defineFacet((h) => + h.serve(Pure, (_, own) => { + const tools = own.writable([tool]); + (own as unknown as StoreSelf).contribute(Tools, tools); + return { n: constant(0), label: constant(tool), bump: async () => tools.value.length }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: app("a") }, + { name: "@test/tools", facet: registry }, + ], + context, + ); + const tools = await host.registry(Tools, context); + expect(tools.value).toEqual(["a"]); + compositions.length = 0; + await host.reload( + [ + { name: "@test/app", facet: app("b") }, + { name: "@test/tools", facet: registry }, + ], + context, + ); + await flushed(); + expect(tools.value).toEqual(["b"]); + // One composition with the new table, never one without a contribution in between. + expect(compositions).toEqual([["b"]]); + await host.close(context); + }); +}); + +describe("reload: a stuck routine call past the handover", () => { + fakeTimers(); + + it("an old routine call stuck past the handover: drain_timeout, the old attempt sealed, its commit refused (audit A12)", async () => { + const stuck = Promise.withResolvers(); + let late: Promise | undefined; + const errors: HostError[] = []; + const serve = (h: FacetHost, label: string) => + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + self.onLive(async (ctx) => { + if (label !== "v1") return; + await stuck.promise; + late = rejection( + self.tx(async (tx) => { + (await tx.draft(doc)).n = 99; + }, ctx), + ); + }); + return { n: self.derive([doc], ([d]) => d.n), label: constant(label), bump: async () => 0 }; + }); + const host = createHost({ closeTimeout: 1_000, onError: (error) => errors.push(error) }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serve(h, "v1")) }], context); + const pure = await host.get(Pure, {}, context); + await reloadApp(host, (h) => serve(h, "v2")); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors.map(({ kind }) => kind)).toContain("drain_timeout"); + stuck.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(await late).toMatchObject({ code: "released" }); + expect(pure.n.value).toBe(0); + await host.close(context); + }); +}); + +describe("reload: GC", () => { + const attempts = instancesOf(Attempt, "activate"); + + it("collects a swapped-out attempt and its closures once its calls settled", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise])); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + const from = attempts.length - 1; + await reloadApp(host, (h) => servePure(h, "v2", 1)); + gate.resolve(); + await old; + await flushed(); + await expectCollected(attempts[from]!); + expect(pure.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("collects a removed type's objects, its provider, its registry entries and the old facet's code", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const objects = instancesOf(ChordObject, "remove"); + const host = createHost({ onError: () => {} }); + /** Loads the plugin; only the host keeps its facet. */ + const load = (): [WeakRef, Promise] => { + const made = defineFacet((h, self) => { + servePure(h, "v1", 1); + h.serve(Names, { compose: (table) => table.map(({ value }) => value) }); + self.contribute(Names, "x"); + }); + return [new WeakRef(made), host.load([{ name: "@test/app", facet: made }], context)]; + }; + const [facet, loading] = load(); + await loading; + await host.get(Pure, {}, context).then((pure) => pure.release()); + await host.reload([], context); + await flushed(); + expect(objects).toHaveLength(1); + await expectCollected(objects[0]!, facet); + await host.close(context); + }); +}); + +describe("reload: a parked attempt's handover (fix round 1)", () => { + it("keeps a dependency held only by a parked attempt alive through reload", async () => { + const A = local("astra.a", { version: 1, params: {}, members: {} }); + const Dep = local("astra.dep", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("astra.missing", { version: 1, params: {}, members: {} }); + let opens = 0; + let closes = 0; + const host = createHost({ onError: () => {} }); + const depFacet = defineFacet((h) => + h.serve(Dep, (_, self) => { + self.open( + async () => { + opens++; + }, + () => { + closes++; + }, + ); + return {}; + }), + ); + const aFacet = () => + defineFacet((h) => + h.serve(A, (_, self) => { + self.get(Dep, { id: 1 }); + self.get(Missing, {}); + return {}; + }), + ); + await host.load( + [ + { name: "@astra/dep", facet: depFacet }, + { name: "@astra/a", facet: aFacet() }, + ], + context, + ); + await flushed(); + expect(opens).toBe(1); + await host.reload( + [ + { name: "@astra/dep", facet: depFacet }, + { name: "@astra/a", facet: aFacet() }, + ], + context, + ); + await flushed(); + const counts = { opens, closes }; + await host.close(context); + expect(counts).toEqual({ opens: 1, closes: 0 }); + }); + + it("binds replacement requests before releasing the parked predecessor's", async () => { + const A = local("sol.parked", { version: 1, params: {}, members: {} }); + const B = local("sol.native", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("sol.missing", { version: 1, params: {}, members: {} }); + let opens = 0; + let closes = 0; + const b = defineFacet((h) => + h.serve(B, (_, self) => { + self.open( + async () => ++opens, + () => { + closes++; + }, + ); + return {}; + }), + ); + const a = () => + defineFacet((h) => + h.serve(A, (_, self) => { + self.get(B, { id: 1 }); + self.get(Missing, {}); + return {}; + }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { name: "@sol/b", facet: b }, + { name: "@sol/a", facet: a() }, + ], + context, + ); + expect({ opens, closes }).toEqual({ opens: 1, closes: 0 }); + await host.reload( + [ + { name: "@sol/b", facet: b }, + { name: "@sol/a", facet: a() }, + ], + context, + ); + await delivered(); + expect({ opens, closes }).toEqual({ opens: 1, closes: 0 }); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: swaps one after another (fix round 1)", () => { + it("a second swap does not wait for the first swap's old call; that call still finishes on its old code", async () => { + const gate = Promise.withResolvers(); + const { host, errors } = await loaded((h) => servePure(h, "v1", 1, [gate.promise]), { closeTimeout: 60_000 }); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => servePure(h, "v2", 10)); + let resolved = false; + const second = reloadApp(host, (h) => servePure(h, "v3", 100)).then(() => { + resolved = true; + }); + await flushed(); + // Not held up by v1's call (Fable, Opus F1): resolved within a few turns, the gate open for new calls. + expect(resolved).toBe(true); + await second; + expect(pure.label.value).toBe("v3"); + gate.resolve(); + expect(await old).toBe(1); + expect(await pure.bump(context)).toBe(101); + expect(errors).toEqual([]); + await host.close(context); + }); +}); + +describe("reload: a reference detaching from a parked swap's old instance (Opus G1)", () => { + it("follows the old instance while its call drains, then keeps its last value with the failure once", async () => { + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise]), { closeTimeout: 60_000 }); + const pure = await host.get(Pure, {}, context); + const calls = changesOf(await pure.n.watch(context)); + const bumping = pure.bump(context); + await flushed(); + const parked = defineFacet((h) => + h.serve(Pure, (_, self) => { + self.get(Missing, {}); + return { n: constant(9), label: constant("v2"), bump: async () => 0 }; + }), + ); + const report = await host.reload([{ name: "@test/app", facet: parked }], context); + expect(report.objects[0]).toMatchObject({ kind: "parked" }); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // The old instance's call still commits: the reference follows it until it retires. + gate.resolve(); + expect(await bumping).toBe(1); + await delivered(); + expect([pure.n.value, calls.map((call) => call.kind), valuesOf(calls)[0]]).toEqual([1, ["value", "failed"], 1]); + await host.close(context); + }); +}); + +describe("a live failure: an older version's mapping follows the admitted commits (final review)", () => { + it("stops an older reference after its instance's admitted final commit published (Astra, G1)", async () => { + const V1 = local("app.admitted", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ n: int() }))); + const State = defineState("state", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const store = await memoryStore(); + let own: SingletonHandle<{ n: number }> | undefined; + let self: StoreSelf | undefined; + const facet = defineFacet((h) => + h.serve( + V2, + (_, declared) => { + self = declared; + own = declared.state(State); + return { n: own }; + }, + { accepts: { 1: { state: { n: (value) => value.n } } } }, + ), + ); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@app/admitted", facet }], context); + const old = await host.get(V1, {}, context); + const current = await host.get(V2, {}, context); + const calls = changesOf(await old.n.watch(context)); + // A commit admitted before the failure: v1's mapping and v2's member both get it, then the failure. + const held = holdNextCommit(store); + const change = own!.change((draft) => { + draft.n = 5; + }, context); + await held.entered; + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + await delivered(); + expect({ old: old.n.value, current: current.n.value, calls: valuesOf(calls) }).toEqual({ + old: 5, + current: { n: 5 }, + calls: [5, { failed: "boom" }], + }); + old.release(); + current.release(); + await host.close(context); + await store.close(context); + }); +}); + +describe("a native restart: references take their last values after the admitted commits (final review)", () => { + it("keeps the value of a commit admitted before the restart's seal while the new instance opens (Sol, G1)", async () => { + const Type = local("app.restart-admitted", { version: 1, params: {}, members: { x: obj({ n: int() }) } }); + const Doc = defineState("doc", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const facet = (open: () => Promise) => + defineFacet((h) => + h.serve(Type, (_, self) => { + counter = self.state(Doc); + self.open(open, () => {}); + return { x: counter }; + }), + ); + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const opening = Promise.withResolvers(); + let counter: SingletonHandle<{ n: number }> | undefined; + await host.load([{ name: "@app/restart", facet: facet(async () => 0) }], context); + const ref = await host.get(Type, {}, context); + // A commit admitted to the store (no running call's promise), then a restart whose new native stays opening. + const held = holdNextCommit(store); + const outgoing = counter!; + const change = outgoing.change((draft) => { + draft.n = 1; + }, context); + await held.entered; + await host.reload([{ name: "@app/restart", facet: facet(() => opening.promise) }], context); + await delivered(); + held.release(); + await change; + await delivered(); + expect([ref.state.value.kind, ref.x.value]).toEqual(["loading", { n: 1 }]); + opening.resolve(1); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/parked.test.ts b/packages/chord/test/next/host/parked.test.ts new file mode 100644 index 000000000..207da3019 --- /dev/null +++ b/packages/chord/test/next/host/parked.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { createHost, type Held } from "../../../src/next/host/index.ts"; +import { + EDITOR, + fakeTimers, + flushed, + forEachCodec, + loaded, + Notes, + partValue, + rawParent, + rawProvider, + rejection, + type TestPrincipal, + transported, + wireNames, +} from "../support.ts"; + +const Gated = local("app.gated", { version: 1, params: { id: int() }, members: { log: method(str(), int()) } }); + +/** + * Serves `Gated`, whose open waits for `gate`; `log` records its argument and returns how many it recorded. In its + * synchronous part a `log` of "release" releases `self.ref`, one of "call" makes the call `log("from a handler")` on + * it, and one of "block" takes 30 ms (the clock moves while no timer can fire). + */ +async function gated() { + const gate = Promise.withResolvers(); + const log: string[] = []; + const self: { ref?: Held } = {}; + const { host } = await loaded((h) => + h.serve(Gated, (_, attempt) => { + attempt.open( + () => gate.promise, + () => {}, + ); + return { + log: async (text) => { + log.push(text); + if (text === "release") self.ref?.release(); + if (text === "call") void self.ref?.log("from a handler", context); + if (text === "block") vi.setSystemTime(Date.now() + 30); + return log.length; + }, + }; + }), + ); + return { host, gate, log, self }; +} + +describe("calls on a loading local reference", () => { + fakeTimers(); + + it("are queued and sent in the order they were made once it is live, before later calls", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const first = ref.log("a", context); + const second = ref.log("b", context); + await flushed(); + expect(log).toEqual([]); + gate.resolve(); + await flushed(); + const third = ref.log("c", context); + expect(await Promise.all([first, second, third])).toEqual([1, 2, 3]); + expect(log).toEqual(["a", "b", "c"]); + }); + + it("keep their order when a handler calls the reference while they are sent: its call queues behind them", async () => { + const { host, gate, log, self } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + self.ref = ref; + const calls = [ref.log("call", context), ref.log("b", context)]; + gate.resolve(); + await Promise.all(calls); + await flushed(); + expect(log).toEqual(["call", "b", "from a handler"]); + }); + + it("are checked against their deadline when sent: one that passed while earlier ones ran is not sent", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context, { readyTimeout: 10 }); + const first = ref.log("block", context); + const second = rejection(ref.log("b", context)); + gate.resolve(); + expect(await first).toBe(1); + expect(await second).toMatchObject({ code: "unavailable", reason: "park_timeout" }); + expect(log).toEqual(["block"]); + }); + + it("reject unavailable/park_timeout at the reference's readyTimeout, measured from each call", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context, { readyTimeout: 100 }); + const first = rejection(ref.log("a", context)); + await vi.advanceTimersByTimeAsync(60); + const second = ref.log("b", context); + await vi.advanceTimersByTimeAsync(40); + expect(await first).toEqual({ + code: "unavailable", + reason: "park_timeout", + message: "app.gated(1): a call waited 100 ms for its reference to go live", + }); + gate.resolve(); + expect(await second).toBe(1); + expect(log).toEqual(["b"]); + }); + + it("reject at once when their caller cancels; the others stay queued", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const { context: cancellable, cancel } = withCancel(context); + const cancelled = ref.log("a", cancellable); + const kept = ref.log("b", context); + cancel(new Error("user cancelled")); + await expect(cancelled).rejects.toThrow("user cancelled"); + gate.resolve(); + expect(await kept).toBe(1); + expect(log).toEqual(["b"]); + }); + + it("reject released when the reference is released; nothing runs", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const call = ref.log("a", context); + ref.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.gated: released" }); + gate.resolve(); + await flushed(); + expect(log).toEqual([]); + }); + + it("reject with the cause when the object fails", async () => { + const { host, gate } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const call = ref.log("a", context); + gate.reject(new Error("offline")); + expect(await rejection(call)).toEqual({ code: "unavailable", message: "app.gated(1): offline" }); + }); + + it("stop being sent when one's handler releases the reference: the rest reject released (race 15)", async () => { + const { host, gate, log, self } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + self.ref = ref; + const first = ref.log("release", context); + const second = ref.log("b", context); + gate.resolve(); + expect(await rejection(second)).toEqual({ code: "released", message: "app.gated: released" }); + await first.catch(() => {}); + expect(log).toEqual(["release"]); + }); +}); + +forEachCodec(() => { + describe("calls on a loading remote reference", () => { + fakeTimers(); + + it("keep one deadline across reconnects; nothing is sent for them on either link", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context, { readyTimeout: 1_000 }); + const call = rejection(notes.wait(context)); + await vi.advanceTimersByTimeAsync(600); + peer.close(); + await transported(); + const next = await rawParent(client); + await next.next("hold"); + await vi.advanceTimersByTimeAsync(400); + expect(await call).toMatchObject({ code: "unavailable", reason: "park_timeout" }); + expect([peer.kinds(), next.kinds()]).toEqual([["hold"], []]); + }); + + it("are resolved against the binding they are sent on: indexes from the new variant, missing methods invalid", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + // Typed for the editor: before its first binding the reference may become either variant. + const editor = notes as Extract, { add: unknown }>; + const add = rejection(editor.add({ text: "a" }, context)); + const wait = editor.wait(context); + const { hold } = await peer.next("hold"); + const READER = wireNames(Notes, "reader"); + const members = [ + { member: READER.state.notes!, value: "[]" }, + { member: READER.state.count!, value: "0" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: READER.variant, members, failed: [] }] }); + expect(await add).toEqual({ + code: "invalid", + reason: "member", + message: 'app.notes(S1) (variant reader) has no method "add"', + }); + const sent = await peer.next("call"); + expect(sent.method).toBe(READER.methods.wait); + expect(sent.method).not.toBe(EDITOR.methods.wait); + peer.send({ kind: "result", call: sent.call }); + expect(await wait).toBe(null); + }); + + it("on a reference released while loading without a link: nothing is sent, the next connect holds nothing (race 3)", async () => { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const notes = client.hold(Notes, { session: "S1" }, context); + const other = client.hold(Notes, { session: "S2" }, context); + const call = notes.wait(context); + notes.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.notes: released" }); + const peer = await rawParent(client); + const { params } = await peer.next("hold"); + expect(partValue(params)).toEqual({ session: "S2" }); + await transported(); + expect(peer.kinds()).toEqual([]); + expect(other.state.value).toEqual({ kind: "loading" }); + }); + }); +}); diff --git a/packages/chord/test/next/host/presence.test.ts b/packages/chord/test/next/host/presence.test.ts new file mode 100644 index 000000000..a43b523bf --- /dev/null +++ b/packages/chord/test/next/host/presence.test.ts @@ -0,0 +1,360 @@ +// Tests of presence (§19): \`host.references(Type)\`, its entries and incarnations, its batches (a link drop as a departure, +// one discovered while a commit sends lands in the next batch), derives over it, its cost and GC. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + addNote, + delivered, + expectCollected, + framesOf, + incarnationOf, + instancesOf, + linked, + NoteStore, + Notes, + PlacedNotes, + RawPeer, + relayed, + SERVER_ADDRESS, + Server, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Every hold started in this file, weakly. */ +const holds = instancesOf(Hold, "start"); + +describe("host.references", () => { + it("lists every reference with the object's params, its principal and its incarnation", async () => { + const { server, client } = await linked((h) => void serveNotes(h), { child: { kind: "guest", session: "S1" } }); + const notes = server.references(Notes); + expect(notes.value).toEqual([]); + await client.get(Notes, { session: "S1" }, context); + const own = await server.get(Notes, { session: "S2" }, context); + expect(notes.value).toEqual([ + { params: { session: "S1" }, principal: { kind: "guest", session: "S1" }, incarnation: incarnationOf(client) }, + { params: { session: "S2" }, principal: { kind: "server" }, incarnation: incarnationOf(server) }, + ]); + // A local type's entries have no principal: the store is referenced by both views. + expect(server.references(NoteStore).value).toEqual([ + { params: { session: "S1" }, incarnation: incarnationOf(server) }, + { params: { session: "S2" }, incarnation: incarnationOf(server) }, + ]); + own.release(); + expect(notes.value).toHaveLength(1); + }); + + it("is made for the first ask with the references that exist then, and kept", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + await client.get(Notes, { session: "S1" }, context); + const notes = server.references(Notes); + expect(notes.value).toHaveLength(1); + expect(server.references(Notes)).toBe(notes); + }); + + it("changes in the batch where a reference binds or ends: a watch gets one splice per change", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const watch = await server.references(Notes).watch(context); + const frames = framesOf(watch); + const one = await client.get(Notes, { session: "S1" }, context); + await client.get(Notes, { session: "S2" }, context); + one.release(); + await transported(); + await delivered(); + const incarnation = incarnationOf(client); + const entry = (session: string) => ({ params: { session }, principal: { kind: "owner" }, incarnation }); + expect(frames.map(([, ops]) => ops)).toEqual([ + [["p", [], 0, 0, [entry("S1")]]], + [["p", [], 1, 0, [entry("S2")]]], + [["p", [], 0, 1, []]], + ]); + expect(frames.at(-1)![0]).toEqual([entry("S2")]); + }); + + it("feeds derives in factories: viewers per session, a link drop a departure in its batch (race 9)", async () => { + const Viewers = local("app.viewers", { version: 1, params: { session: str() }, members: { count: int() } }); + const Shown = remote("app.shown", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { here: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((host) => { + host.serve(Shown, () => ({ variant: "v", members: { here: constant(1) } })); + host.serve(Viewers, ({ session }, self) => ({ + count: self.derive( + [host.references(Shown)], + ([refs]) => refs.filter((ref) => ref.params.session === session).length, + ), + })); + }), + }, + ], + context, + ); + const viewers = await server.get(Viewers, { session: "S1" }, context); + const counts = framesOf(await viewers.count.watch(context)); + const tabs = await Promise.all( + [0, 1].map(async () => { + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" } }, context); + tab.connect(b, { principal: { kind: "server" } }, context); + return { tab, link: a }; + }), + ); + await Promise.all(tabs.map(({ tab }) => tab.get(Shown, { session: "S1" }, context))); + await tabs[0]!.tab.get(Shown, { session: "S2" }, context); + expect(viewers.count.value).toBe(2); + tabs[1]!.link.close(); + // The departure is in the batch of the link's close: the derive has it at once. + expect(viewers.count.value).toBe(1); + await delivered(); + expect(counts.at(-1)![0]).toBe(1); + }); + + it("puts a departure discovered while a commit sends into the next batch: one state message per commit (race 9)", async () => { + const Viewed = remote("app.viewed", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { count: int(), viewers: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + serveNotes(host); + host.serve(Viewed, ({ session }, _principal, self) => { + const store = self.get(NoteStore, { session }); + const count = self.derive([store.list], ([list]) => list.length); + const viewers = self.derive([host.references(Viewed)], ([refs]) => refs.length); + return { variant: "v", members: { count, viewers } }; + }); + }); + await server.load([{ name: "@test/app", facet }], context); + const names = wireNames(Viewed, "v").state as { count: number; viewers: number }; + /** A raw tab; its server end closes while it sends, once `drop.armed` (a transport failing on send). */ + const tab = (drop?: { armed: boolean }) => { + const [a, b] = memoryLinkPair(); + const end = drop === undefined ? a : closingOnSend(a, drop); + server.accept(end, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + peer.send({ + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); + return peer; + }; + const drop = { armed: false }; + // The dropped tab binds first, so its link sends (and fails) before the kept one's in the commit's flushes. + const dropped = tab(drop); + await dropped.next("bound"); + const kept = tab(); + await kept.next("bound"); + await transported(); + kept.received.length = 0; + drop.armed = true; + await addNote(server, "S1", "x"); + await transported(); + const states = kept.received.filter((message) => message.kind === "state"); + const changed = states.map(({ holds }) => holds.flatMap(({ members }) => members.map(({ member }) => member))); + // The commit's message carries the commit (the count); the departure is the next batch's (the viewers). + expect(changed).toEqual([[names.count], [names.viewers]]); + expect(dropped.closed).toBe(true); + }); + + it("completes a hold's bookkeeping before its presence entry publishes: a link closing then leaks nothing", async () => { + const Viewed = remote("app.viewed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { viewers: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + host.serve( + Viewed, + (_params, _principal, self) => ({ + variant: "v", + members: { viewers: self.derive([host.references(Viewed)], ([refs]) => refs.length) }, + }), + { linger: 60_000 }, + ); + }); + await server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + const drop = { armed: false }; + server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); + await peer.next("bound"); + peer.send({ kind: "ack" }); + await transported(); + // The second hold's presence entry changes the first's state: sending it closes the link. + drop.armed = true; + peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); + await transported(); + expect(peer.closed).toBe(true); + expect(server.references(Viewed).value).toEqual([]); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + await server.close(context); + }); + + it("completes a release's bookkeeping before its departure publishes: a link closing then leaves no lingering", async () => { + const Viewed = remote("app.viewed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { viewers: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + host.serve( + Viewed, + (_params, _principal, self) => ({ + variant: "v", + members: { viewers: self.derive([host.references(Viewed)], ([refs]) => refs.length) }, + }), + { linger: 50 }, + ); + }); + await server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + const drop = { armed: false }; + server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + for (const hold of [0, 1]) { + peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); + await peer.next("bound"); + peer.send({ kind: "ack" }); + } + await transported(); + // The first release changes the second hold's state: sending it closes the link, which releases the second. + drop.armed = true; + peer.send({ kind: "release", hold: 0 }); + await transported(); + expect(peer.closed).toBe(true); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(server.references(Viewed).value).toEqual([]); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 0 }); + await server.close(context); + }); + + it("removes a bulk departure with one list and one splice op per entry, in one frame (a revoke of many holds)", async () => { + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy: (principal) => (principal.kind === "guest" ? [{ session: principal.session }] : []), + onError: () => {}, + }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Notes; + for (let hold = 0; hold < 4; hold++) { + peer.send({ + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); + } + await transported(); + const references = server.references(Notes); + const frames = framesOf(await references.watch(context)); + expect(references.value).toHaveLength(4); + server.revoke({ session: "S1" }, context); + await delivered(); + expect(frames.map(([, ops]) => ops)).toEqual([Array(4).fill(["p", [], 0, 1, []])]); + expect(frames[0]![0]).toEqual([]); + }); + + it("counts two tabs through a relay as two incarnations on the worker", async () => { + const relay = await relayed(); + const second = createHost({ onError: () => {} }); + await second.load([], context); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" } }, context); + second.connect(b, { principal: { kind: "server" } }, context); + await relay.tab.get(PlacedNotes, { session: "S1" }, context); + await second.get(PlacedNotes, { session: "S1" }, context); + const worker = relay.workers.get("session|S1")!; + const incarnations = worker.references(PlacedNotes).value.map(({ incarnation }) => incarnation); + expect(incarnations).toEqual([incarnationOf(relay.tab), incarnationOf(second)]); + }); + + it("has no entries for lingering objects", async () => { + const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const { server, client } = await linked((h) => + h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 60_000 }), + ); + const kept = await client.get(Kept, {}, context); + const references = server.references(Kept); + expect(references.value).toHaveLength(1); + kept.release(); + await transported(); + expect(references.value).toEqual([]); + expect(server.activity.value.lingering).toBe(1); + await server.close(context); + }); + + it("keeps no released reference (GC)", async () => { + const { server, refs } = await (async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const references = server.references(Notes); + const held = holds.length; + const notes = await client.get(Notes, { session: "S1" }, context); + expect(references.value).toHaveLength(1); + const refs = holds.slice(held); + notes.release(); + await transported(); + expect(references.value).toEqual([]); + return { server, refs }; + })(); + await expectCollected(...refs); + expect(server.references(Notes).value).toEqual([]); + }); +}); + +/** `link`, closing itself on a send once `drop.armed`, as a transport whose send fails. */ +function closingOnSend(link: Link, drop: { readonly armed: boolean }): Link { + return { + send: (message) => (drop.armed ? link.close() : link.send(message)), + onMessage: (listener) => link.onMessage(listener), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} diff --git a/packages/chord/test/next/host/ref-state.test.ts b/packages/chord/test/next/host/ref-state.test.ts new file mode 100644 index 000000000..360a7bd3e --- /dev/null +++ b/packages/chord/test/next/host/ref-state.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + bindHold, + bindNext, + fakeTimers, + flushed, + forEachCodec, + loaded, + Notes, + rawParent, + rawProvider, + rejection, + statesOf, + transported, +} from "../support.ts"; + +const Flaky = local("app.flaky", { + version: 1, + params: { id: int() }, + members: { n: int(), get: method(undefined, int()) }, +}); + +/** Serves `Flaky`, whose open throws while `failing.left` is above 0 (broken while `failing.broken`). */ +async function flaky(failing: { left: number; broken: boolean }) { + return loaded((h) => + h.serve(Flaky, ({ id }, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + if (failing.broken) throw broken(new Error("no configuration")); + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(id), get: async () => id }; + }), + ); +} + +const offline = { code: "unavailable", message: "app.flaky(1): offline" }; + +describe("ref.state of a local reference", () => { + fakeTimers(); + + it("is watched from its current state: failed with retryAt, loading on the next attempt, live; release ends it", async () => { + const { host } = await flaky({ left: 1, broken: false }); + const ref = host.hold(Flaky, { id: 1 }, context); + expect(ref.state.value).toEqual({ kind: "loading" }); + const watch = await ref.state.watch(context); + expect(watch.value).toEqual({ kind: "loading" }); + const states = statesOf(watch); + await flushed(); + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "transient", error: offline }, + retryAt: 1_000, + }); + await vi.advanceTimersByTimeAsync(1_000); + await flushed(); + expect(ref.state.value).toEqual({ kind: "live" }); + ref.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(states).toEqual(["failed: app.flaky(1): offline", "loading", "live"]); + expect(ref.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("reads and calls follow the state: not_ready before live, the cause's error while failed, released after", async () => { + const { host } = await flaky({ left: 1, broken: true }); + const ref = host.hold(Flaky, { id: 1 }, context); + expect(() => ref.n.value).toThrow("app.flaky(1): not live yet"); + await flushed(); + const cause = { code: "unavailable", message: "app.flaky(1): no configuration" }; + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: cause }, + retryAt: undefined, + }); + expect(() => ref.n.value).toThrow("no configuration"); + expect(await rejection(ref.get(context))).toEqual(cause); + ref.release(); + expect(() => ref.n.value).toThrow("app.flaky: released"); + expect(await rejection(ref.get(context))).toEqual({ code: "released", message: "app.flaky: released" }); + }); + + it("retry renews a broken object: loading, then live", async () => { + const { host } = await flaky({ left: 1, broken: true }); + const ref = host.hold(Flaky, { id: 1 }, context); + await flushed(); + expect(ref.state.value.kind).toBe("failed"); + ref.retry(); + expect(ref.state.value).toEqual({ kind: "loading" }); + await flushed(); + expect(ref.state.value).toEqual({ kind: "live" }); + expect(await ref.get(context)).toBe(1); + }); + + it("is made on first access, once; a watch taken after release has ended", async () => { + const { host } = await flaky({ left: 0, broken: false }); + const ref = await host.get(Flaky, { id: 1 }, context); + expect(ref.state).toBe(ref.state); + expect(ref.state.value).toEqual({ kind: "live" }); + ref.release(); + const late = await ref.state.watch(context); + expect(late.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + expect(await late.closed).toEqual({ kind: "ended" }); + }); + + it("a watch ends cancelled with its context", async () => { + const { host } = await flaky({ left: 0, broken: false }); + const ref = host.hold(Flaky, { id: 1 }, context); + const { context: cancellable, cancel } = withCancel(context); + const watch = await ref.state.watch(cancellable); + cancel(); + expect(await watch.closed).toEqual({ kind: "aborted" }); + }); +}); + +forEachCodec(() => { + describe("ref.state of a remote reference", () => { + fakeTimers(); + + it("loading until bound, loading while its link is down, failed with retryAt from retryIn, live again", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const states = statesOf(await notes.state.watch(context)); + await bindNext(peer); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + peer.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + next.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error } }, + retryIn: 500, + }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient", error }, retryAt: 500 }); + next.send({ kind: "loading", hold }); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + bindHold(next, hold); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(states).toEqual(["live", "loading", "failed: offline", "loading", "live"]); + }); + + it("a failed with retryIn 0 retries now: retryAt is the current time", async () => { + const { client, peer } = await rawProvider(); + vi.setSystemTime(2_000); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "transient", error: { kind: "chord", error } }, retryIn: 0 }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", retryAt: 2_000 }); + }); + + it("variant: undefined before the first binding, the last bound one while loading or failed", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + expect(notes.variant).toBeUndefined(); + await bindNext(peer, [{ text: "a" }]); + await transported(); + expect(notes.variant).toBe("editor"); + peer.close(); + await transported(); + expect([notes.variant, notes.count.value]).toEqual(["editor", 1]); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + next.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", retryAt: undefined }); + expect(notes.variant).toBe("editor"); + // Its replica's last value stays readable while the hold is failed (rule 5). + expect(notes.count.value).toBe(1); + notes.release(); + expect(() => notes.variant).toThrow("app.notes: released"); + }); + }); +}); diff --git a/packages/chord/test/next/host/reference-keys.test.ts b/packages/chord/test/next/host/reference-keys.test.ts new file mode 100644 index 000000000..cdef55686 --- /dev/null +++ b/packages/chord/test/next/host/reference-keys.test.ts @@ -0,0 +1,704 @@ +// Tests of the keys a reference holds (slice 6a §12; decision "References are the only indirection" (4)): its key +// watches and factory pins of an object's keyed members, holding their last values while it is failed (one `failed` +// call), pinned again on its next instance once that is live (one frame), released once. + +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + flushed, + holdNextCommit, + instancesOf, + loaded, + memoryStore, + valuesOf, +} from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +type N = { n: number }; +const Schema = obj({ n: int() }); +const Notes = defineState("notes", { schema: Schema, version: 1, initial: () => ({ n: 0 }), keyed: true }); +const Own = defineState("own", { schema: Schema, version: 1, initial: () => ({ n: 0 }) }); +const Provider = local("app.provider", { version: 1, params: { id: str() }, members: { notes: keyed(Schema) } }); +const Exposer = local("app.exposer", { version: 1, params: {}, members: { notes: keyed(Schema) } }); +/** A viewer with params: it ends with its last reference (an object with empty params lives as long as its host). */ +const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { value: Schema, double: int() } }); + +/** Providers `a` and `b`, an exposer re-exposing provider `which()`'s notes, and a viewer of its key `k`. */ +async function served(which: () => string) { + const providers: Record> = {}; + const exposers: Self[] = []; + const { host } = await loaded((h) => { + h.serve(Provider, ({ id }, self) => { + const notes = self.state(Notes); + providers[id] = notes; + return { notes }; + }); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + return { notes: self.get(Provider, { id: which() }).notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }); + const a = await host.get(Provider, { id: "a" }, context); + const b = await host.get(Provider, { id: "b" }, context); + const set = (id: string, n: number) => + providers[id]!.change( + "k", + (draft) => { + draft.n = n; + }, + context, + ); + await set("a", 1); + await set("b", 2); + const exposer = await host.get(Exposer, {}, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + const fail = (): void => + failLive(attempted, exposers.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + return { host, a, b, exposer, viewer, set, fail, providers }; +} + +describe("a failed object's held keys", () => { + it("keep their last values and get the failure once (at, watch, a derive over at); the provider goes on", async () => { + const { host, a, exposer, viewer, set, fail } = await served(() => "a"); + const values = changesOf(await viewer.value.watch(context)); + const keyWatch = (await exposer.notes.watch("k", context))!; + const keyed = changesOf(keyWatch); + const direct = changesOf((await a.notes.watch("k", context))!); + fail(); + await set("a", 3); + await delivered(); + expect(exposer.state.value.kind).toBe("failed"); + expect([viewer.value.value, viewer.double.value, keyWatch.value]).toEqual([{ n: 1 }, 2, { n: 1 }]); + expect([values, keyed].map((calls) => calls.map((call) => call.kind))).toEqual([["failed"], ["failed"]]); + expect(valuesOf(direct)).toEqual([{ n: 3 }]); + await host.close(context); + }); +}); + +describe("re-binding when a new attempt goes live", () => { + it("pins again on the new instance's member of the same name (one frame); the old provider's pin is released", async () => { + let which = "a"; + const { host, exposer, viewer, set, fail, providers } = await served(() => which); + const values = changesOf(await viewer.value.watch(context)); + const doubles = changesOf(await viewer.double.watch(context)); + const keyWatch = (await exposer.notes.watch("k", context))!; + const keyed = changesOf(keyWatch); + const pins = (id: string) => + (providers[id] as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source("k") + ?.pins; + expect(pins("a")).toBe(1); + fail(); + // Its keys follow the failed instance until its admitted commits published (audit A10), then show the failure. + await delivered(); + which = "b"; + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + await delivered(); + expect([viewer.value.value, viewer.double.value, keyWatch.value]).toEqual([{ n: 2 }, 4, { n: 2 }]); + expect([pins("a"), pins("b")]).toEqual([undefined, 1]); + await set("b", 4); + await set("a", 5); + await delivered(); + // The failure once, then whole values (the first after a failure is `[["r", value]]`). + expect(valuesOf(values)).toEqual([{ failed: "boom" }, { n: 2 }, { n: 4 }]); + expect(valuesOf(keyed)).toEqual([{ failed: "boom" }, { n: 2 }, { n: 4 }]); + // A derive over the key reads its last value meanwhile: it does not fail. + expect(valuesOf(doubles)).toEqual([4, 8]); + await host.close(context); + }); + + it("loads a cold persisted key of the new member once the new instance is live (the last value meanwhile)", async () => { + const Stored = defineState("notes", { + schema: Schema, + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const Durable = local("app.durable", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const store = await memoryStore(); + let memos: KeyedHandle | undefined; + const serveDurable = (h: Parameters[0]>[0]) => + h.serve(Durable, (_, self) => { + memos = self.state(Stored); + return { notes: memos }; + }); + const seed = await loaded(serveDurable, { store }); + await seed.host.get(Durable, {}, context); + await memos!.change( + "k", + (draft) => { + draft.n = 10; + }, + context, + ); + await seed.host.close(context); + // A new host: the key is in the store only. + let runs = 0; + const exposers: Self[] = []; + const { host } = await loaded( + (h) => { + h.serve(Provider, (_, self) => ({ notes: self.state(Notes) })); + serveDurable(h); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + return { notes: ++runs === 1 ? self.get(Provider, { id: "a" }).notes : self.get(Durable, {}).notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }, + { store }, + ); + const exposer = await host.get(Exposer, {}, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + failLive(attempted, exposers[0]!, new ChordError({ code: "unavailable", message: "boom" })); + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + // The key loads from the store after the new instance went live: one frame then. + await vi.waitFor(() => expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 10 }, 20])); + await host.close(context); + await store.close(context); + }); +}); + +/** A provider with a persisted keyed state (empty params: a stable identity), whose keys a new host loads cold. */ +const ColdProvider = local("app.cold-provider", { version: 1, params: {}, members: { notes: keyed(Schema) } }); +/** A persisted keyed state whose keys a new host loads from the store. */ +const Stored = defineState("notes", { + schema: Schema, + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +/** The pins on key `k` of a provider's notes (`undefined`: none). */ +function pinsOf(notes: KeyedHandle): number | undefined { + // The handle's slot keeps the key's source while it is pinned. + return (notes as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source("k")?.pins; +} + +/** + * Provider `a` (its key `k` in the store only, its loads held at `gate` once `hold()` is called) and `b` (in memory), + * an exposer re-exposing `which()`'s notes, and a viewer of key `k`. + */ +async function cold(which: () => string) { + const store = await memoryStore(); + let seeded: KeyedHandle | undefined; + const seed = await loaded( + (h) => + h.serve(ColdProvider, (_, self) => { + seeded = self.state(Stored); + return { notes: seeded }; + }), + { + store, + }, + ); + await seed.host.get(ColdProvider, {}, context); + await seeded!.change( + "k", + (draft) => { + draft.n = 1; + }, + context, + ); + await seed.host.close(context); + // A new host: the key is in the store only. + const providers: Record> = {}; + const exposers: Self[] = []; + const { host, errors } = await loaded( + (h) => { + h.serve(ColdProvider, (_, self) => { + providers.a = self.state(Stored); + return { notes: providers.a }; + }); + h.serve(Provider, ({ id }, self) => { + providers[id] = self.state(Notes); + return { notes: providers[id] }; + }); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + const notes = which() === "a" ? self.get(ColdProvider, {}).notes : self.get(Provider, { id: "b" }).notes; + return { notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }, + { store }, + ); + await host.get(ColdProvider, {}, context); + await host.get(Provider, { id: "b" }, context); + await providers.b!.change( + "k", + (draft) => { + draft.n = 2; + }, + context, + ); + const exposer = await host.get(Exposer, {}, context); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const hold = (): void => { + // The slot's persistence reads cold keys from the store; the test holds the read open. + const persistence = (providers.a as unknown as { slot: { persistence: { read(...args: unknown[]): unknown } } }) + .slot.persistence; + const read = persistence.read.bind(persistence); + persistence.read = async (...args: unknown[]) => { + const result = await read(...args); + entered.resolve(); + await gate.promise; + return result; + }; + }; + const fail = (): void => + failLive(attempted, exposers.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + const close = async (): Promise => { + await host.close(context); + await store.close(context); + }; + return { host, errors, exposer, providers, hold, gate, entered, fail, close }; +} + +describe("a key whose first pin still loads", () => { + it("the object fails and re-binds meanwhile: the consumer retries on the new member, the late pin is released", async () => { + let which = "a"; + const { host, errors, exposer, providers, hold, gate, entered, fail, close } = await cold(() => which); + hold(); + const loading = host.get(Viewer, { id: 1 }, context); + await entered.promise; + fail(); + which = "b"; + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + gate.resolve(); + const viewer = await loading; + await delivered(); + expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 2 }, 4]); + expect([pinsOf(providers.a!), pinsOf(providers.b!)]).toEqual([undefined, 1]); + // The pin's failed load parked the viewer (a dependency failure is not its own): nothing failed it. + expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(0); + const values = changesOf(await viewer.value.watch(context)); + await providers.a!.change( + "k", + (draft) => { + draft.n = 3; + }, + context, + ); + await providers.b!.change( + "k", + (draft) => { + draft.n = 4; + }, + context, + ); + await delivered(); + expect(valuesOf(values)).toEqual([{ n: 4 }]); + await close(); + }); + + it("the object fails and stays failed: the consumer parks failed { waiting } on it, never live on it", async () => { + const { host, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); + hold(); + const abort = new AbortController(); + const viewer = host.hold(Viewer, { id: 1 }, context); + const loading = host.get(Viewer, { id: 1 }, withAbortSignal(abort.signal, context)); + await entered.promise; + fail(); + gate.resolve(); + await delivered(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(exposer.state.value.kind).toBe("failed"); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + viewer.release(); + abort.abort(); + await expect(loading).rejects.toThrow(); + expect(pinsOf(providers.a!)).toBeUndefined(); + await close(); + }); + + it("the object ends meanwhile: the consumer's wait rejects, the late pin is released", async () => { + const { host, providers, hold, gate, entered, close } = await cold(() => "a"); + hold(); + const loading = host.get(Viewer, { id: 1 }, context); + await entered.promise; + const closing = host.close(context); + gate.resolve(); + await expect(loading).rejects.toThrow(); + await closing; + expect(pinsOf(providers.a!)).toBeUndefined(); + await close(); + }); +}); + +describe("a keyed watch whose object closes while it is set up", () => { + it("rejects with the object's end error, whichever microtask the close lands in", async () => { + for (let gap = 0; gap < 40; gap++) { + let notes: KeyedHandle | undefined; + const { host } = await loaded((h) => + h.serve(Exposer, (_, self) => { + notes = self.state(Notes); + return { notes }; + }), + ); + const exposer = await host.get(Exposer, {}, context); + await notes!.change( + "a", + (draft) => { + draft.n = 1; + }, + context, + ); + const watching = exposer.notes.watch("a", context).then( + () => "watch", + (error: unknown) => `${(error as ChordError).code}/${(error as ChordError).reason}`, + ); + for (let turn = 0; turn < gap; turn++) await Promise.resolve(); + await host.close(context); + expect(["watch", "released/ended"]).toContain(await watching); + } + }); +}); + +describe("chains and releases", () => { + it("a key re-exposed through two objects holds with the middle one (the failure once) and follows its next instance", async () => { + const Outer = local("app.outer", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const OuterViewer = local("app.outer-viewer", { version: 1, params: {}, members: { value: Schema } }); + let which = "a"; + const middles: Self[] = []; + const providers: Record> = {}; + const { host } = await loaded((h) => { + h.serve(Provider, ({ id }, self) => { + const notes = self.state(Notes); + providers[id] = notes; + return { notes }; + }); + h.serve(Exposer, (_, self) => { + middles.push(self); + self.state(Own); + return { notes: self.get(Provider, { id: which }).notes }; + }); + h.serve(Outer, (_, self) => ({ notes: self.get(Exposer, {}).notes })); + h.serve(OuterViewer, (_, self) => ({ value: self.get(Outer, {}).notes.at("k") })); + }); + for (const id of ["a", "b"]) { + await host.get(Provider, { id }, context); + await providers[id]!.change( + "k", + (draft) => { + draft.n = id === "a" ? 1 : 2; + }, + context, + ); + } + const middle = await host.get(Exposer, {}, context); + const viewer = await host.get(OuterViewer, {}, context); + const values = changesOf(await viewer.value.watch(context)); + failLive(attempted, middles[0]!, new ChordError({ code: "unavailable", message: "boom" })); + await providers.a!.change( + "k", + (draft) => { + draft.n = 3; + }, + context, + ); + await delivered(); + expect([viewer.value.value, values.map((call) => call.kind)]).toEqual([{ n: 1 }, ["failed"]]); + which = "b"; + middle.retry(); + (await host.get(Exposer, {}, context)).release(); + await delivered(); + expect(viewer.value.value).toEqual({ n: 2 }); + expect(valuesOf(values)).toEqual([{ failed: "boom" }, { n: 2 }]); + await host.close(context); + }); + + it("a consumer released while the object re-activates leaves no hold behind; nothing is kept for it", async () => { + let which = "a"; + const refs = await (async () => { + const { host, exposer, viewer, fail, providers } = await served(() => which); + const pins = (id: string) => + (providers[id] as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source( + "k", + )?.pins; + fail(); + which = "b"; + exposer.retry(); + viewer.release(); + (await host.get(Exposer, {}, context)).release(); + // The viewer's teardown releases its references. + await new Promise((resolve) => setTimeout(resolve, 10)); + expect([pins("a"), pins("b")]).toEqual([undefined, undefined]); + const kept = [new WeakRef(viewer), new WeakRef(providers.a!)]; + await host.close(context); + return kept; + })(); + await expectCollected(...refs); + }); +}); + +describe("a key whose first pin still loads at startup (scenario 20, Astra)", () => { + it("keeps the first pass pending during an otherwise-ready dependency's delayed keyed pin", async () => { + const Data = defineState("app.pin-data", { + schema: Schema, + version: 1, + keyed: true, + initial: () => ({ n: 4 }), + }); + const Store = local("app.pin-store", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const Reader = local("app.pin-reader", { version: 1, params: {}, members: { value: Schema } }); + const gate = Promise.withResolvers(); + const original = ChordObject.prototype.pinKey; + const spy = vi.spyOn(ChordObject.prototype, "pinKey").mockImplementation(async function ( + this: ChordObject, + member, + key, + version, + ctx, + ) { + await gate.promise; + return original.call(this, member, key, version, ctx); + }); + const host = createHost({ onError: () => {} }); + let done = false; + const loading = host + .load( + [ + { + name: "@test/pin", + facet: defineFacet((h) => { + h.serve(Store, (_, self) => ({ notes: self.state(Data) })); + h.serve(Reader, (_, self) => ({ value: self.get(Store, {}).notes.at("k") })); + }), + }, + ], + context, + ) + .then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + spy.mockRestore(); + const ref = await host.get(Reader, {}, context); + expect(ref.value.value).toEqual({ n: 4 }); + await host.close(context); + expect(premature).toBe(false); + }); +}); + +describe("a released key leaves its document (final review)", () => { + it("collects a reference whose key watch stopped while another reference pins the same key (Astra)", async () => { + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 1 }), + keyed: true, + }); + const Type = local("app.key-gc", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), set: method() }, + }); + const facet = defineFacet((h) => + h.serve(Type, (_, self) => { + const rows = self.state(Rows); + const set = async (ctx: Context) => { + await rows.change( + "a", + (draft) => { + draft.n = 2; + }, + ctx, + ); + return null; + }; + return { rows, set }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/key-gc", facet }], context); + const keeper = await host.get(Type, {}, context); + await keeper.set(context); + const kept = (await keeper.rows.watch("a", context))!; + kept.start(async () => {}); + const released = await (async () => { + const ref = await host.get(Type, {}, context); + const watch = (await ref.rows.watch("a", context))!; + watch.start(async () => {}); + await watch.stop(); + ref.release(); + return new WeakRef(ref); + })(); + await flushed(); + await expectCollected(released); + await kept.stop(); + keeper.release(); + await host.close(context); + }); +}); + +describe("keys across a restart and a failed re-pin (Astra's coverage note)", () => { + const Keys = local("app.keys", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + + /** Serves `Keys` over the persisted `Stored` (its handle kept in `handles`), with a native opened by `open`. */ + const serveKeys = (handles: KeyedHandle[], open: () => Promise) => + defineFacet((h) => + h.serve(Keys, (_, self) => { + self.open(open, () => {}); + const notes = self.state(Stored); + handles.push(notes); + return { notes }; + }), + ); + + it("a commit admitted to a watched key, landing after a restart's drain, is in the key's last value", async () => { + const store = await memoryStore(); + const handles: KeyedHandle[] = []; + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveKeys(handles, async () => {}) }], context); + const keys = await host.get(Keys, {}, context); + const change = (n: number) => + handles.at(-1)!.change( + "k", + (draft) => { + draft.n = n; + }, + context, + ); + await change(1); + const watch = (await keys.notes.watch("k", context))!; + const calls = changesOf(watch); + const held = holdNextCommit(store); + const admitted = change(5); + await held.entered; + // The new code's native stays opening: the new instance is not live while the key holds its last value. + const opening = Promise.withResolvers(); + const reloading = host.reload([{ name: "@test/app", facet: serveKeys(handles, () => opening.promise) }], context); + await delivered(); + held.release(); + await admitted; + await reloading; + await delivered(); + expect([watch.value, valuesOf(calls)]).toEqual([{ n: 5 }, [{ n: 5 }]]); + opening.resolve(); + await vi.waitFor(() => expect(keys.state.value).toEqual({ kind: "live" })); + expect(watch.value).toEqual({ n: 5 }); + await host.close(context); + await store.close(context); + }); + + it("a key's re-pin on a new instance that fails to load keeps its last value, one failed call; the next pin heals it", async () => { + const store = await memoryStore(); + const seeded: KeyedHandle[] = []; + const seed = createHost({ store, onError: () => {} }); + await seed.load([{ name: "@test/app", facet: serveKeys(seeded, async () => {}) }], context); + await seed.get(Keys, {}, context); + await seeded[0]!.change( + "k", + (draft) => { + draft.n = 1; + }, + context, + ); + await seed.close(context); + // A new host: `Keys`' key is in the store only; the exposer re-exposes provider `b` (in memory) first. + const handles: KeyedHandle[] = []; + let b: KeyedHandle | undefined; + const providers = defineFacet((h) => { + h.serve(Keys, (_, self) => { + const notes = self.state(Stored); + handles.push(notes); + return { notes }; + }); + h.serve(Provider, (_, self) => { + b = self.state(Notes); + return { notes: b }; + }); + }); + const exposing = (which: "keys" | "b") => + defineFacet((h) => + h.serve(Exposer, (_, self) => { + self.state(Own); + const notes = which === "keys" ? self.get(Keys, {}).notes : self.get(Provider, { id: "b" }).notes; + return { notes }; + }), + ); + const host = createHost({ store, onError: () => {} }); + await host.load( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("b") }, + ], + context, + ); + await host.get(Provider, { id: "b" }, context); + await b!.change( + "k", + (draft) => { + draft.n = 2; + }, + context, + ); + const exposer = await host.get(Exposer, {}, context); + const watch = (await exposer.notes.watch("k", context))!; + const calls = changesOf(watch); + // The cold provider's next read of the key fails once. + const persistence = (handles[0] as unknown as { slot: { persistence: { read(...args: unknown[]): unknown } } }) + .slot.persistence; + const read = persistence.read.bind(persistence); + persistence.read = async () => { + persistence.read = read; + throw new Error("disk"); + }; + // A swap in place: the new instance re-exposes the cold provider's notes; the key's pin there fails to load. + await host.reload( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("keys") }, + ], + context, + ); + await flushed(); + expect(exposer.state.value).toEqual({ kind: "live" }); + expect([watch.value, valuesOf(calls)]).toEqual([{ n: 2 }, [{ failed: expect.stringContaining("disk") }]]); + // The next instance pins it again, loaded this time: one frame heals it. + await host.reload( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("keys") }, + ], + context, + ); + await vi.waitFor(() => expect(watch.value).toEqual({ n: 1 })); + expect(valuesOf(calls)).toEqual([{ failed: expect.stringContaining("disk") }, { n: 1 }]); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/reference-source.test.ts b/packages/chord/test/next/host/reference-source.test.ts new file mode 100644 index 000000000..e4e235836 --- /dev/null +++ b/packages/chord/test/next/host/reference-source.test.ts @@ -0,0 +1,647 @@ +// Tests of a reference's members (`ReferenceSource`; slice 6a §1.2, §1.4; decision "References are the only +// indirection"): consumers' watches and derives over `ref.member` pass straight through to the instance's member while +// live, keep the last value while it points at nothing (one `failed` call for a failure), move by name to the next +// instance (one frame, whole after a failure). + +import { describe, expect, it } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference, type Self, type StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Source, Subscriber } from "../../../src/next/state/internal.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + flushed, + holdNextCommit, + instancesOf, + linked, + loaded, + memoryStore, + metered, + Server, + transported, + valuesOf, +} from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Store = local("app.store", { + version: 1, + params: {}, + members: { list: obj({ items: arr(str()) }), count: int(), add: method(obj({ text: str() })) }, +}); +const Viewer = local("app.count-viewer", { version: 1, params: {}, members: { tens: int() } }); + +/** Serves `Store` (a list, its count, `add`) and `Viewer` (a derive over the store's count); `attempts` are the store's. */ +async function served(options: Parameters[1] = {}) { + const attempts: Self[] = []; + const lists: SingletonHandle<{ items: string[] }>[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => { + attempts.push(self); + const list = self.state(List); + lists.push(list); + return { + list, + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }: { text: string }, ctx: Context) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }); + h.serve(Viewer, (_, self) => ({ tens: self.derive([self.get(Store, {}).count], ([n]) => n * 10) })); + }, options); + const store = await host.get(Store, {}, context); + const fail = (): void => + failLive(attempted, attempts.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + return { host, errors, store, attempts, lists, fail }; +} + +describe("following the live attempt", () => { + it("passes the member's frames on as they are, and keeps a derive member lazy while nothing watches it", async () => { + const { telemetry, counts } = await metered(); + const { host, store, lists: handles } = await served({ telemetry }); + expect(store.count.value).toBe(0); + const computes = (): number => counts()["chord.derive.computes"] ?? 0; + const before = computes(); + expect(before).toBeGreaterThan(0); + await store.add({ text: "a" }, context); + await store.add({ text: "b" }, context); + // Unwatched: marked only, computed by the next read. + expect(computes()).toBe(before); + expect(store.count.value).toBe(2); + const watch = await store.count.watch(context); + const counted = changesOf(watch); + const lists = changesOf(await store.list.watch(context)); + const own = changesOf(await handles[0]!.watch(context)); + await store.add({ text: "c" }, context); + await delivered(); + expect(counted).toEqual([{ kind: "value", value: 3, ops: [["r", 3]], retired: false }]); + // The slot's own change, not a copy. + expect(lists).toHaveLength(1); + expect(lists[0]).toBe(own[0]); + await watch.stop(); + await delivered(); + const stopped = computes(); + await store.add({ text: "d" }, context); + expect(computes()).toBe(stopped); + await host.close(context); + }); +}); + +describe("a failed object's members", () => { + it("keep their last values and get the failure once; a new watch opens at the last value (slice 6a §1.4)", async () => { + const { host, store, lists, fail } = await served(); + await store.add({ text: "a" }, context); + const counted = changesOf(await store.count.watch(context)); + fail(); + await delivered(); + expect(store.state.value.kind).toBe("failed"); + const late = await store.count.watch(context); + expect(late.value).toBe(1); + const lately = changesOf(late); + const listed = changesOf(await store.list.watch(context)); + // The failed attempt's handle is sealed; the slot changes only through a new attempt. + expect(lists[0]!.value).toEqual({ items: ["a"] }); + store.retry(); + await (await host.get(Store, {}, context)).add({ text: "b" }, context); + await delivered(); + // The failure once; after it, whole values: the next instance's (unchanged) 1, then 2. + expect(valuesOf(counted)).toEqual([{ failed: "boom" }, 1, 2]); + // Opened while failed: at the last value, the failure once, then whole values. + expect(valuesOf(listed)).toEqual([{ failed: "boom" }, { items: ["a"] }, { items: ["a", "b"] }]); + expect(valuesOf(lately)).toEqual([{ failed: "boom" }, 1, 2]); + await host.close(context); + }); + + it("re-exposed from another object hold with the re-exposing object; the provider's source goes on", async () => { + const Note = obj({ title: str() }); + const Memo = defineState("memo", { schema: Note, version: 1, initial: () => ({ title: "" }), keyed: true }); + const Own = defineState("own", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Provider = local("app.provider", { + version: 1, + params: {}, + members: { n: int(), notes: keyed(Note), set: method(int()) }, + }); + const Exposer = local("app.exposer", { version: 1, params: {}, members: { n: int(), note: Note } }); + let exposer: Self | undefined; + const { host } = await loaded((h) => { + h.serve(Provider, (_, self) => { + const memos = self.state(Memo); + const n = self.derive([memos.at("a")], ([note]) => note.title.length); + return { + n, + notes: memos, + set: async (length, ctx) => { + await memos.change( + "a", + (draft) => { + draft.title = "x".repeat(length); + }, + ctx, + ); + return null; + }, + }; + }); + h.serve(Exposer, (_, self) => { + exposer = self; + self.state(Own); + const provider = self.get(Provider, {}); + return { n: provider.n, note: provider.notes.at("a") }; + }); + }); + const provider = await host.get(Provider, {}, context); + const exposed = await host.get(Exposer, {}, context); + await provider.set(1, context); + const exposedN = changesOf(await exposed.n.watch(context)); + const exposedNote = changesOf(await exposed.note.watch(context)); + const providerN = changesOf(await provider.n.watch(context)); + failLive(attempted, exposer!, new ChordError({ code: "unavailable", message: "boom" })); + await delivered(); + await provider.set(7, context); + await delivered(); + expect(exposed.state.value.kind).toBe("failed"); + expect([exposed.n.value, exposed.note.value]).toEqual([1, { title: "x" }]); + expect([exposedN, exposedNote].map((calls) => calls.map((call) => call.kind))).toEqual([["failed"], ["failed"]]); + expect(provider.n.value).toBe(7); + expect(valuesOf(providerN)).toEqual([7]); + await host.close(context); + }); +}); + +describe("the window of a live failure", () => { + it("every member follows on while an admitted commit of the object runs, then all hold (audit A10)", async () => { + const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Persisted = defineState("own", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const Provider = local("app.provider", { version: 1, params: {}, members: { n: int(), set: method(int()) } }); + const Exposer = local("app.exposer", { version: 1, params: {}, members: { n: int(), own: obj({ n: int() }) } }); + const store = await memoryStore(); + let exposer: StoreSelf | undefined; + let own: SingletonHandle<{ n: number }> | undefined; + const { host } = await loaded( + (h) => { + h.serve(Provider, (_, self) => { + const doc = self.state(Counter); + return { + n: self.derive([doc], ([d]) => d.n), + set: async (n, ctx) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Exposer, (_, self) => { + exposer = self; + own = self.state(Persisted); + return { n: self.get(Provider, {}).n, own }; + }); + }, + { store }, + ); + const provider = await host.get(Provider, {}, context); + const exposed = await host.get(Exposer, {}, context); + await provider.set(1, context); + const frames = changesOf(await exposed.n.watch(context)); + const owned = changesOf(await exposed.own.watch(context)); + const held = holdNextCommit(store); + const change = own!.change((draft) => { + draft.n = 5; + }, context); + await held.entered; + failLive(attempted, exposer!, new ChordError({ code: "unavailable", message: "boom" })); + await provider.set(9, context); + await delivered(); + // A re-exposed member shows one more change of its provider in the window. + expect([exposed.n.value, frames.map((call) => call.kind)]).toEqual([9, ["value"]]); + held.release(); + await change; + await delivered(); + // The object's own admitted commit still publishes once; then every member holds (the failure once). + expect(valuesOf(owned)).toEqual([{ n: 5 }, { failed: "boom" }]); + expect([exposed.n.value, frames.map((call) => call.kind)]).toEqual([9, ["value", "failed"]]); + await host.close(context); + await store.close(context); + }); +}); + +describe("re-binding when a new attempt goes live", () => { + it("recovers a consumer's derive that failed on the provider's earlier error, and the derives downstream", async () => { + const Counted = local("app.counted", { version: 1, params: {}, members: { count: int(), add: method(str()) } }); + const Tens = local("app.tens", { version: 1, params: {}, members: { tens: int(), hundreds: int() } }); + const selves: Self[] = []; + const { host } = await loaded((h) => { + h.serve(Counted, (_, self) => { + selves.push(self); + const first = selves.length === 1; + const list = self.state(List); + return { + count: self.derive([list], ([l]) => { + if (first && l.items.length === 1) throw new Error("count broke"); + return l.items.length; + }), + add: async (text, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }); + h.serve(Tens, (_, self) => { + const tens = self.derive([self.get(Counted, {}).count], ([n]) => n * 10); + return { tens, hundreds: self.derive([tens], ([n]) => n * 10) }; + }); + }); + const counted = await host.get(Counted, {}, context); + const tens = await host.get(Tens, {}, context); + await counted.add("a", context); + expect(() => tens.tens.value).toThrow("count broke"); + expect(() => tens.hundreds.value).toThrow("count broke"); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + counted.retry(); + (await host.get(Counted, {}, context)).release(); + expect([tens.tens.value, tens.hundreds.value]).toEqual([10, 100]); + const frames = changesOf(await tens.hundreds.watch(context)); + await counted.add("b", context); + await delivered(); + expect(valuesOf(frames)).toEqual([200]); + await host.close(context); + }); + + it("re-binds a consumer's derive over a retried provider's derive member, and its watches get frames again", async () => { + const { host, store, fail } = await served(); + const viewer = await host.get(Viewer, {}, context); + const tens = changesOf(await viewer.tens.watch(context)); + const counted = changesOf(await store.count.watch(context)); + await store.add({ text: "a" }, context); + await delivered(); + fail(); + await delivered(); + expect([store.count.value, viewer.tens.value]).toEqual([1, 10]); + store.retry(); + await (await host.get(Store, {}, context)).add({ text: "b" }, context); + await delivered(); + expect(viewer.state.value.kind).toBe("live"); + expect([store.count.value, viewer.tens.value]).toEqual([2, 20]); + // A derive over the held member keeps its last value (no failure); the member's watch gets it once. + expect(valuesOf(tens)).toEqual([10, 20]); + expect(valuesOf(counted)).toEqual([1, { failed: "boom" }, 1, 2]); + await host.close(context); + }); + + it("sends one frame from the last value when the new attempt's member differs", async () => { + const Shown = local("app.shown", { version: 1, params: {}, members: { label: str() } }); + const labels = ["old", "new"]; + let attempt = 0; + const selves: Self[] = []; + const { host } = await loaded((h) => + h.serve(Shown, (_, self) => { + selves.push(self); + self.state(List); + const label = labels[attempt++]!; + return { label: self.derive([], () => label) }; + }), + ); + const shown = await host.get(Shown, {}, context); + const frames = changesOf(await shown.label.watch(context)); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + shown.retry(); + (await host.get(Shown, {}, context)).release(); + await delivered(); + expect(shown.label.value).toBe("new"); + expect(frames).toEqual([{ kind: "value", value: "new", ops: [["r", "new"]], retired: false }]); + await host.close(context); + }); +}); + +describe("subscribers", () => { + it("a hold over a link on a remote object re-exposing a local member gets frames again after the local retry", async () => { + const Count = local("app.count", { version: 1, params: {}, members: { n: obj({ n: int() }), add: method() } }); + const View = remote("app.view", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }) } } }, + }); + const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const selves: Self[] = []; + const { server, client } = await linked((h) => { + h.serve(Count, (_, self) => { + selves.push(self); + const counter = self.state(Counter); + return { + n: self.derive([counter], ([c]) => ({ n: c.n * 10 })), + add: async (ctx: Context) => { + await counter.change((draft) => void draft.n++, ctx); + return null; + }, + }; + }); + h.serve(View, (_, _principal, self) => ({ variant: "v", members: { n: self.get(Count, {}).n } })); + }); + const count = await server.get(Count, {}, context); + const view = await client.get(View, {}, context); + if (view.variant !== "v") throw new Error("expected variant v"); + const frames = changesOf(await view.n.watch(context)); + await count.add(context); + await transported(); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + count.retry(); + await (await server.get(Count, {}, context)).add(context); + await transported(); + expect(view.n.value).toEqual({ n: 20 }); + expect(valuesOf(frames)).toEqual([{ n: 10 }, { n: 20 }]); + await client.close(context); + await server.close(context); + }); +}); + +describe("subscribers, directly", () => { + it("are fed the member's calls: a failure, then its recovery at the re-bind as a root replacement", async () => { + const Counted = local("app.counted", { version: 1, params: {}, members: { count: int(), add: method(str()) } }); + const selves: Self[] = []; + const { host } = await loaded((h) => + h.serve(Counted, (_, self) => { + selves.push(self); + const first = selves.length === 1; + const list = self.state(List); + return { + count: self.derive([list], ([l]) => { + if (first && l.items.length === 2) throw new Error("count broke"); + return l.items.length; + }), + add: async (text, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }), + ); + const counted = await host.get(Counted, {}, context); + const source = resolveWatchable(counted.count) as Source; + const events: unknown[] = []; + const subscriber: Subscriber = { + advance: (call) => + void events.push(call.kind === "value" ? [call.value, call.ops] : (call.error as Error).message), + end: (end) => void events.push(end.kind), + endAfterDelivery: () => void events.push("ended"), + }; + expect(source.value).toBe(0); + source.subscribe(subscriber); + await counted.add("a", context); + await counted.add("b", context); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + counted.retry(); + await (await host.get(Counted, {}, context)).add("c", context); + expect(counted.count.value).toBe(3); + expect(events).toEqual([[1, [["r", 1]]], "count broke", [2, [["r", 2]]], [3, [["r", 3]]]]); + await host.close(context); + }); +}); + +describe("a member the new attempt's variant lacks", () => { + it("fails: its watch gets one failure call and stays open, reads throw invalid/member; a later attempt brings it back", async () => { + const Two = remote("app.two", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() } }, view: { state: { m: int() } } }, + }); + const owner = { kind: "owner" } as const; + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); + const selves: Self[] = []; + const facet = defineFacet((h) => + h.serve(Two, (_, _principal, self) => { + selves.push(self); + self.state(List); + return selves.length === 2 + ? { variant: "view", members: { m: constant(2) } } + : { variant: "full", members: { n: constant(selves.length) } }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const two = await host.get(Two, {}, context); + if (two.variant !== "full") throw new Error("expected the full variant"); + const n = two.n; + const calls = changesOf(await n.watch(context)); + const boom = new ChordError({ code: "unavailable", message: "boom" }); + failLive(attempted, selves[0]!, boom); + two.retry(); + (await host.get(Two, {}, context)).release(); + await delivered(); + // The failed instance is followed until its admitted commits published (audit A10), when the retry's attempt + // loads already (no failure call for it); then the new variant's lack of it. + expect(calls).toEqual([ + { kind: "failed", error: expect.objectContaining({ code: "invalid", reason: "member" }) }, + ]); + expect(() => n.value).toThrow('app.two (variant view) has no state member "n"'); + // The same as over a link: the member comes back with the next attempt that has it. + failLive(attempted, selves[1]!, boom); + two.retry(); + (await host.get(Two, {}, context)).release(); + await delivered(); + expect(n.value).toBe(3); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + await host.close(context); + }); +}); + +describe("an ended object's members", () => { + it("mark the derives over them: a consumer's derive keeps no cached value, its read throws released", async () => { + const Paged = local("app.paged", { version: 1, params: { id: int() }, members: { n: int() } }); + const Reader = local("app.reader", { version: 1, params: {}, members: { twice: int() } }); + const { host } = await loaded((h) => { + h.serve(Paged, () => ({ n: constant(4) })); + h.serve(Reader, (_, self) => ({ twice: self.derive([self.get(Paged, { id: 1 }).n], ([n]) => n * 2) })); + }); + const reader = await host.get(Reader, {}, context); + expect(reader.twice.value).toBe(8); + for (const object of attempted) { + const paged = object.deref(); + if (paged?.label === "app.paged(1)") paged.end(); + } + expect(() => reader.twice.value).toThrow(expect.objectContaining({ code: "released" })); + await host.close(context); + }); +}); + +describe("member sources and memory", () => { + it("a failed attempt is collected after a retry while the object, its member source and a retained watch live", async () => { + const attempts: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Store, (_, self) => { + attempts.push(new WeakRef(self)); + const list = self.state(List); + return { + list, + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }), + ); + const store = await host.get(Store, {}, context); + const frames = changesOf(await store.count.watch(context)); + failLive(attempted, attempts[0]!.deref()!, new ChordError({ code: "unavailable", message: "boom" })); + store.retry(); + (await host.get(Store, {}, context)).release(); + await store.add({ text: "a" }, context); + await delivered(); + expect(valuesOf(frames)).toEqual([1]); + await expectCollected(attempts[0]!); + expect(attempts[1]!.deref()).toBeDefined(); + await host.close(context); + }); + + it("an ended object's member sources are collected with it", async () => { + const refs = await (async () => { + const { host, store } = await served(); + const source = resolveWatchable(store.count); + expect(source?.value).toBe(0); + const ref = new WeakRef(source!); + await host.close(context); + return [ref]; + })(); + await expectCollected(...refs); + }); +}); + +describe('what a reference keeps (decision "References are the only indirection")', () => { + it("keeps nothing of an ended instance while it waits with an open watch, its failure's error included", async () => { + const Count = local("app.kept", { version: 1, params: {}, members: { n: int() } }); + const Doc = defineState("kept", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 5 }) }); + const selves: WeakRef[] = []; + const facet = defineFacet((h) => + h.serve(Count, (_, self) => { + selves.push(new WeakRef(self)); + const doc = self.state(Doc); + return { n: self.derive([doc], ([value]) => value.n) }; + }), + ); + const host = createHost({ onError: () => {} }); + const from = attempted.length; + await host.load([{ name: "@app/kept", facet }], context); + let ref: Reference | undefined = await host.get(Count, {}, context); + const watch = await ref.n.watch(context); + const calls = changesOf(watch); + const objects = attempted.slice(from); + const documents = objects.map((object) => new WeakRef(object.deref()!.documents)); + // Removed: the reference waits with its last value and its failure; the instance ends. + await host.reload([], context); + await delivered(); + const state = ref.state.value; + if (state.kind !== "failed") throw new Error("expected failed"); + const kept = state.cause.error; + expect([ref.n.value, valuesOf(calls)]).toEqual([5, [{ failed: kept.message }]]); + await expectCollected(...selves, ...objects, ...documents); + // Released: its last values and bindings go with it. + const released = new WeakRef(ref); + await watch.stop(); + ref.release(); + ref = undefined; + await expectCollected(released); + expect(kept.code).toBe("unavailable"); + await host.close(context); + }); + + it("never takes values from an instance it never attached to: a reference made while its object is failed has none", async () => { + const { host, store, fail } = await served(); + await store.add({ text: "a" }, context); + fail(); + await delivered(); + const late = host.hold(Store, {}, context); + expect(store.count.value).toBe(1); + expect(() => late.count.value).toThrow("boom"); + late.release(); + await host.close(context); + }); + + it("computes a consumer's derive over a watched reference member once per commit (cost guard)", async () => { + const { telemetry, counts } = await metered(); + const { host, store } = await served({ telemetry }); + const viewer = await host.get(Viewer, {}, context); + const counted = changesOf(await store.count.watch(context)); + const tens = changesOf(await viewer.tens.watch(context)); + await store.add({ text: "a" }, context); + await delivered(); + const before = counts()["chord.derive.computes"] ?? 0; + for (const text of ["b", "c", "d"]) await store.add({ text }, context); + await delivered(); + // The store's count and the viewer's tens: one compute each per commit, nothing for passing through. + expect((counts()["chord.derive.computes"] ?? 0) - before).toBe(6); + expect([valuesOf(counted), valuesOf(tens)]).toEqual([ + [1, 2, 3, 4], + [10, 20, 30, 40], + ]); + await host.close(context); + }); +}); + +describe("a reference keeps the last values of its attached instance's variant (final review)", () => { + it("keeps an unread member of a remote contract served on its host when the type is removed (Astra)", async () => { + const Type = remote("app.remote-unread", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() } } }, + }); + const owner = { kind: "owner" } as const; + const facet = defineFacet((h) => + h.serve(Type, () => ({ variant: "full", members: { n: constant(7) } })), + ); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); + await host.load([{ name: "@app/remote-unread", facet }], context); + const ref = await host.get(Type, {}, context); + // `n` never read: the removal takes its last value from the instance's variant (the object is live no more). + await host.reload([], context); + await delivered(); + expect(ref.n.value).toBe(7); + ref.release(); + await host.close(context); + }); +}); + +describe("a failed object lets its ended instance go (final review)", () => { + it("collects the instance while a reference keeps its last value with an open watch (Astra, GC)", async () => { + const Type = local("app.failed-gc", { version: 1, params: {}, members: { n: int() } }); + let self: Self | undefined; + const facet = defineFacet((h) => + h.serve(Type, (_, declared) => { + self = declared; + return { n: constant(7) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/failed-gc", facet }], context); + const ref = await host.get(Type, {}, context); + changesOf(await ref.n.watch(context)); + const instance = new WeakRef(self!); + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + self = undefined; + await flushed(); + await delivered(); + // The object stays failed (host lifetime) and its reference keeps 7: neither keeps the ended instance. + expect(ref.n.value).toBe(7); + await expectCollected(instance); + ref.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/reference.test.ts b/packages/chord/test/next/host/reference.test.ts new file mode 100644 index 000000000..50b8b5372 --- /dev/null +++ b/packages/chord/test/next/host/reference.test.ts @@ -0,0 +1,702 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, type ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type Held, + type Reference, + type RefIn, + type RefState, + type Self, +} from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { Binding } from "../../../src/next/host/reference.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Store } from "../../../src/next/store/index.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import type { NativeWatchable, Watchable } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + flushed, + framesOf, + instancesOf, + loaded, + memoryStore, + Notes, + rejection, + Server, + settledHeap, +} from "../support.ts"; + +/** Every remote object bound while this file runs, weakly. */ +const remoteObjects = instancesOf(RemoteObject, "hold"); +const chordObjects = instancesOf(ChordObject, "attempt"); + +const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("app.counter", { version: 1, params: {}, members: { doc: obj({ n: int() }), bump: method() } }); +const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + +/** A counter: one document, exposed, and a method that increments it. */ +function serveCounter(self: Self) { + const doc = self.state(Doc); + return { + doc, + bump: async (ctx: typeof context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; +} + +describe("references", () => { + it("are hydrated when got; their state members are watchables", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + expect(counter.doc.value).toEqual({ n: 0 }); + const frames = framesOf(await counter.doc.watch(context)); + await counter.bump(context); + await delivered(); + expect(frames).toEqual([[{ n: 1 }, [["s", ["n"], 1]]]]); + expect(counter.doc.value).toEqual({ n: 1 }); + }); + + it("throw released on every use after release, which is idempotent", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + counter.release(); + counter.release(); + const released = { code: "released", message: "app.counter: released" }; + expect(() => counter.doc.value).toThrow("app.counter: released"); + expect(await rejection(counter.doc.watch(context))).toEqual(released); + expect(await rejection(counter.bump(context))).toEqual(released); + }); + + it("end their watches when released, while other references keep the object", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const first = await host.get(Counter, {}, context); + const second = await host.get(Counter, {}, context); + const one = await first.doc.watch(context); + const two = await second.doc.watch(context); + first.release(); + expect(await one.closed).toEqual({ kind: "ended" }); + const frames = framesOf(two); + await second.bump(context); + await delivered(); + expect(frames.map(([value]) => value)).toEqual([{ n: 1 }]); + }); + + it("declared with self.get keep their object alive and are released when the declaring object ends", async () => { + const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { n: int() } }); + const inner = vi.fn(() => ({ n: constant(1) })); + const { host } = await loaded((h) => { + h.serve(Inner, inner); + h.serve(Viewer, ({ id }, self) => ({ n: self.get(Inner, { id }).n })); + }); + const viewer = await host.get(Viewer, { id: 1 }, context); + const direct = await host.get(Inner, { id: 1 }, context); + direct.release(); + expect(viewer.n.value).toBe(1); + expect(inner).toHaveBeenCalledTimes(1); + viewer.release(); + // The viewer's teardown releases its reference in the background. + await flushed(); + await host.get(Inner, { id: 1 }, context); + expect(inner).toHaveBeenCalledTimes(2); + }); +}); + +describe("cycles (Q5)", () => { + const A = local("a", { version: 1, params: { n: int() }, members: { v: str() } }); + const B = local("b", { version: 1, params: { n: int() }, members: { v: str() } }); + const C = local("c", { version: 1, params: { n: int() }, members: { v: str() } }); + + it("fail a two-object cycle with invalid/cycle and keep nothing", async () => { + const factories = { a: vi.fn(), b: vi.fn() }; + const { host } = await loaded((h) => { + h.serve(A, ({ n }, self) => { + factories.a(); + return { v: self.get(B, { n }).v }; + }); + h.serve(B, ({ n }, self) => { + factories.b(); + return { v: self.get(A, { n }).v }; + }); + }); + expect(await rejection(host.get(A, { n: 1 }, context))).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: a(1) → b(1) → a(1)", + }); + await expect(host.get(A, { n: 1 }, context)).rejects.toThrow("dependency cycle"); + expect([factories.a.mock.calls.length, factories.b.mock.calls.length]).toEqual([2, 2]); + }); + + it("fail a three-object cycle", async () => { + const { host } = await loaded((h) => { + h.serve(A, ({ n }, self) => ({ v: self.get(B, { n }).v })); + h.serve(B, ({ n }, self) => ({ v: self.get(C, { n }).v })); + h.serve(C, ({ n }, self) => ({ v: self.get(A, { n }).v })); + }); + expect(await rejection(host.get(A, { n: 1 }, context))).toMatchObject({ + message: "dependency cycle: a(1) → b(1) → c(1) → a(1)", + }); + }); + + it("fail an object that gets itself", async () => { + const { host } = await loaded((h) => h.serve(A, ({ n }, self) => ({ v: self.get(A, { n }).v }))); + expect(await rejection(host.get(A, { n: 1 }, context))).toMatchObject({ + message: "dependency cycle: a(1) → a(1)", + }); + }); +}); + +describe("GC", () => { + // AbortSignal.any keeps cleanup entries per call while its sources live before Node 24.16.0 (nodejs/node#62367): a + // call's signal is linked to its reference and its attempt only until it settles. + it.each([ + ["without a caller's signal", () => context], + ["with a long-lived caller's signal", () => withAbortSignal(new AbortController().signal, context)], + ])("10,000 calls on one reference retain flat memory, %s", async (_case, caller) => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + const calls = caller(); + const run = async () => { + for (let i = 0; i < 10_000; i++) await counter.bump(calls); + }; + await run(); + const before = await settledHeap(); + await run(); + // The leak retained about 350 B per call; a document commit itself retains nothing. + expect((await settledHeap()) - before).toBeLessThan(500_000); + }); + + it("a released reference keeps neither its contract nor its object", async () => { + const objects: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Viewer, ({ id }, self) => { + objects.push(new WeakRef(self)); + return { n: constant(id) }; + }), + ); + const { reference, contract } = await (async () => { + // The caller's own declaration, equal to the served one. + const Same = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + const reference = await host.get(Same, { id: 1 }, context); + await reference.n.watch(context); + reference.release(); + return { reference, contract: new WeakRef(Same) }; + })(); + await expectCollected(contract, ...objects); + expect(() => reference.n.value).toThrow("app.viewer: released"); + }); + + it("a released reference whose state was read keeps no host", async () => { + const { reference, host } = await (async () => { + const { host } = await loaded((h) => h.serve(Viewer, ({ id }) => ({ n: constant(id) }))); + const reference = await host.get(Viewer, { id: 1 }, context); + expect(reference.state.value).toEqual({ kind: "live" }); + await (await reference.state.watch(context)).stop(); + reference.release(); + return { reference, host: new WeakRef(host) }; + })(); + await expectCollected(host); + expect(reference.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("a released reference whose state was read after a failure keeps neither the failure's error nor the host", async () => { + const { reference, refs } = await (async () => { + const { host } = await loaded((h) => + h.serve(Viewer, (_, self) => { + self.open( + async () => { + throw broken(new Error("no configuration")); + }, + () => {}, + ); + return { n: constant(1) }; + }), + ); + const reference = host.hold(Viewer, { id: 1 }, context); + await flushed(); + const state = reference.state.value; + if (state.kind !== "failed") throw new Error("expected a failure"); + const refs = [new WeakRef(state.cause.error), new WeakRef(host)]; + reference.release(); + return { reference, refs }; + })(); + await expectCollected(...refs); + expect(reference.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("a hold released before load keeps nothing: the reference and its contract go while the host waits", async () => { + const host = createHost({ onError: () => {} }); + const refs = (() => { + const Same = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + const reference = host.hold(Same, { id: 1 }, context); + reference.release(); + return [new WeakRef(reference), new WeakRef(Same)]; + })(); + await expectCollected(...refs); + expect(host).toBeDefined(); + }); + + it("a released hold keeps neither its object, nor its state's watches, nor a remote object without a link", async () => { + const objects: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Viewer, ({ id }, self) => { + objects.push(new WeakRef(self)); + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: constant(id) }; + }), + ); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const { refs, held } = await (async () => { + const made = remoteObjects.length; + const local = host.hold(Viewer, { id: 1 }, context); + const remote = client.hold(Notes, { session: "S1" }, context); + const watches = [await local.state.watch(context), await remote.state.watch(context)]; + // A member watch waiting for the first value, and a parked call: both rejected by the release. + const waits = [local.n.watch(context).catch(() => {}), remote.wait(context).catch(() => {})]; + local.release(); + remote.release(); + await Promise.all(waits); + const refs = [...watches.map((watch) => new WeakRef(watch)), ...remoteObjects.slice(made)]; + return { refs, held: [local, remote] }; + })(); + await expectCollected(...objects, ...refs); + expect(held).toHaveLength(2); + }); +}); + +describe("release", () => { + it("rejects the calls made through the reference with released and aborts their handlers' contexts", async () => { + const Slow = local("app.slow", { version: 1, params: {}, members: { wait: method() } }); + const aborted: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Slow, () => ({ + wait: (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => { + aborted.push(ctx.abortSignal?.reason); + resolve(null); + }), + ), + })), + ); + const first = await host.get(Slow, {}, context); + const second = await host.get(Slow, {}, context); + const pending = first.wait(context); + const other = second.wait(context); + first.release(); + expect(await rejection(pending)).toEqual({ code: "released", message: "app.slow: released" }); + expect(aborted).toHaveLength(1); + second.release(); + await other.catch(() => {}); + expect(aborted).toHaveLength(2); + }); + + it("reads, watches and calls after the object ended carry the end's reason; the holder's own release has none", async () => { + const Ended = local("app.ended", { version: 1, params: {}, members: { n: int(), m: method() } }); + const { host } = await loaded((h) => h.serve(Ended, () => ({ n: constant(1), m: async () => null }))); + const ended = await host.get(Ended, {}, context); + const released = await host.get(Ended, {}, context); + released.release(); + await host.close(context); + const reasonOf = (error: unknown) => { + const { code, reason } = error as ChordError; + return `${code}/${reason}`; + }; + const thrown = (read: () => unknown) => { + try { + read(); + return "no throw"; + } catch (error) { + return reasonOf(error); + } + }; + expect([ + thrown(() => ended.n.value), + reasonOf(await ended.n.watch(context).catch((error: unknown) => error)), + reasonOf(await ended.m(context).catch((error: unknown) => error)), + ]).toEqual(["released/ended", "released/ended", "released/ended"]); + expect(thrown(() => released.n.value)).toBe("released/undefined"); + }); +}); + +describe("remote references", () => { + it("have every member of every variant; variant is undefined before live and released after release", async () => { + const Two = remote("app.two", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + const Holder = local("app.holder", { version: 1, params: {}, members: { n: int() } }); + const seen: unknown[] = []; + const owner = { kind: "owner" } as const; + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); + const facet = defineFacet((h) => { + h.serve(Two, () => ({ variant: "view", members: { m: constant(1) } })); + h.serve(Holder, (_, self) => { + const two = self.get(Two, {}); + seen.push(Object.keys(two).sort()); + seen.push(two.variant); + return { n: constant(1) }; + }); + }); + await host.load([{ name: "@test/app", facet }], context); + expect(seen).toEqual([["bump", "m", "n", "release", "retry", "state", "variant"], undefined]); + const two = await host.get(Two, {}, context); + expect(() => (two as unknown as { n: { value: number } }).n.value).toThrow( + 'app.two (variant view) has no state member "n"', + ); + two.release(); + expect(() => two.variant).toThrow("app.two: released"); + }); +}); + +describe("remote reference types", () => { + it("narrow on variant; RefIn picks variants", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + type Ref = Reference; + expectTypeOf().toEqualTypeOf<"full" | "view">(); + expectTypeOf["m"]["value"]>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + "variant" | "n" | "bump" | "release" | "retry" | "state" + >(); + expectTypeOf>().not.toHaveProperty("variant"); + }); + + it("a held reference's variant may be undefined (before its first binding) and still narrows", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + expectTypeOf["variant"]>().toEqualTypeOf<"full" | "view" | undefined>(); + const narrow = (held: Held) => (held.variant === "full" ? held.bump : undefined); + expectTypeOf(narrow).returns.toEqualTypeOf<((ctx: Context) => Promise) | undefined>(); + expectTypeOf>().toEqualTypeOf>(); + expectTypeOf["state"]>().toEqualTypeOf>(); + expectTypeOf["retry"]>().toEqualTypeOf<() => void>(); + }); +}); + +describe("keyed state members (slice 6a §12)", () => { + const NoteSchema = obj({ title: str() }); + const Memo = defineState("memo", { + schema: NoteSchema, + version: 1, + initial: () => ({ title: "" }), + keyed: true, + }); + const NoteStore = local("app.note-store", { + version: 1, + params: {}, + members: { notes: keyed(NoteSchema), write: method(obj({ id: str(), title: str() })), remove: method(str()) }, + }); + const Relay = local("app.relay", { version: 1, params: {}, members: { notes: keyed(NoteSchema) } }); + /** A view of note `id`; views of one note with another `copy` are other objects. */ + const NoteView = local("app.note-view", { + version: 1, + params: { id: str(), copy: int() }, + members: { note: NoteSchema }, + }); + + /** Serves the store, a relay re-exposing its keyed member, and views pinning one key each; `memos()` is the store's. */ + async function notes(options: { readonly viewOf?: "store" | "relay" } = {}) { + let memos: KeyedHandle<{ title: string }> | undefined; + const pinned: Watchable<{ title: string }>[] = []; + const refs: object[] = []; + const { host, errors } = await loaded((h) => { + h.serve(NoteStore, (_, self) => { + memos = self.state(Memo); + return { + notes: memos, + write: async ({ id, title }, ctx) => { + await memos!.change( + id, + (draft) => { + draft.title = title; + }, + ctx, + ); + return null; + }, + remove: async (id, ctx) => { + await memos!.retire(id, ctx); + return null; + }, + }; + }); + h.serve(Relay, (_, self) => ({ notes: self.get(NoteStore, {}).notes })); + h.serve(NoteView, ({ id }, self) => { + const source = options.viewOf === "relay" ? self.get(Relay, {}) : self.get(NoteStore, {}); + refs.push(source); + const note = source.notes.at(id); + pinned.push(note); + return { note }; + }); + }); + const store = await host.get(NoteStore, {}, context); + return { host, errors, store, memos: () => memos!, pinned, refs }; + } + + it("a consumer's at(key) is the provider's per-key source, loaded before it goes live; retirement carries the flag", async () => { + const { host, store } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + const changes = changesOf(await view.note.watch(context)); + await store.write({ id: "a", title: "y" }, context); + await delivered(); + await store.remove("a", context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { title: "y" }, ops: [["s", ["title"], "y"]], retired: false }, + { kind: "value", value: { title: "" }, ops: [["r", { title: "" }]], retired: true }, + ]); + view.release(); + await host.close(context); + }); + + it("two consumers of one key each hold their own pin (references own their keys); each released once, with its reference", async () => { + const { host, store, memos, pinned } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const first = await host.get(NoteView, { id: "a", copy: 1 }, context); + const second = await host.get(NoteView, { id: "a", copy: 2 }, context); + const slot = (memos() as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot; + const [one, two] = pinned.map((at) => (at as unknown as { source: object }).source); + // Each reference's own source of the key, on the store's instance (decision "References are the only indirection"). + expect(one).not.toBe(two); + const shared = slot.source("a")!; + expect(shared.pins).toBe(2); + first.release(); + await flushed(); + expect([slot.source("a"), shared.pins]).toEqual([shared, 1]); + second.release(); + await flushed(); + expect(slot.source("a")).toBeUndefined(); + await host.close(context); + }); + + it("a re-exposed keyed member resolves to the ultimate provider's source (no second tracker)", async () => { + const { host, store, memos } = await notes({ viewOf: "relay" }); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + const slot = (memos() as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot; + // One pin on the ultimate slot, through the relay's key and the store's. + expect(slot.source("a")!.pins).toBe(1); + await store.write({ id: "a", title: "y" }, context); + expect(view.note.value).toEqual({ title: "y" }); + await host.close(context); + }); + + it("ref.member.get and watch work anywhere; at(key) only in a factory", async () => { + const { host, store } = await notes(); + expect(await store.notes.get("a", context)).toBeUndefined(); + expect(await store.notes.watch("a", context)).toBeUndefined(); + await store.write({ id: "a", title: "x" }, context); + expect(await store.notes.get("a", context)).toEqual({ title: "x" }); + const watch = (await store.notes.watch("a", context))!; + expect(watch.value).toEqual({ title: "x" }); + expect(() => store.notes.at("a")).toThrow("valid only in the factory"); + store.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + await host.close(context); + }); + + it("a watch(key) settling after its reference was released is ended at once with its pin; it rejects released", async () => { + const { host, store, memos } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const slot = (memos() as unknown as { slot: { source(key: string): object | undefined } }).slot; + const watching = store.notes.watch("a", context); + // The slot's watch is underway: the release lands before the reference follows it. + await Promise.resolve(); + await Promise.resolve(); + store.release(); + expect(await rejection(watching)).toMatchObject({ code: "released" }); + await flushed(); + expect(slot.source("a")).toBeUndefined(); + await host.close(context); + }); + + it("a retained released factory reference keeps no provider: its at(key) drops the source", async () => { + const from = chordObjects.length; + const { host, store, refs, pinned } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + view.release(); + store.release(); + await host.close(context); + // The factory's reference and its at(key) are still held here (and by the host's factory). + expect(refs).toHaveLength(1); + await expectCollected(...chordObjects.slice(from)); + expect(() => (pinned[0] as unknown as { value: unknown }).value).toThrow("released"); + }); + + /** Serves `NoteStore` over `definition` (persisted) on `store`, and views pinning one key each. */ + async function persistedNotes(store: Store, definition: ReturnType) { + return loaded( + (h) => { + h.serve(NoteStore, (_, self) => { + const memos = self.state(definition); + return { + notes: memos, + write: async ({ id, title }, ctx) => { + await memos.change( + id, + (draft) => { + draft.title = title; + }, + ctx, + ); + return null; + }, + remove: async () => null, + }; + }); + h.serve(NoteView, ({ id }, self) => ({ note: self.get(NoteStore, {}).notes.at(id) })); + }, + { store }, + ); + } + + /** A persisted keyed `memo` at `version`, whose `migrate` (from older versions) throws. */ + function persistedMemo(version: number) { + return defineState("memo", { + schema: NoteSchema, + version, + initial: () => ({ title: "" }), + migrate: () => { + throw new Error("cannot migrate"); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + } + + /** A store holding memo `a` = "x" at version 1, written by a host of its own. */ + async function seeded(): Promise { + const store = await memoryStore(); + const { host } = await persistedNotes(store, persistedMemo(1)); + await (await host.get(NoteStore, {}, context)).write({ id: "a", title: "x" }, context); + await host.close(context); + return store; + } + + it("a persisted key a consumer pins is loaded from the store before the consumer goes live", async () => { + const store = await seeded(); + const { host } = await persistedNotes(store, persistedMemo(1)); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + await host.close(context); + await store.close(context); + }); + + it("a pin load that fails its provider parks the consumer failed { waiting } on it (a dependency failure)", async () => { + const store = await seeded(); + const { host, errors } = await persistedNotes(store, persistedMemo(2)); + const provider = await host.get(NoteStore, {}, context); + const view = host.hold(NoteView, { id: "a", copy: 1 }, context); + await vi.waitFor(() => expect(view.state.value.kind).toBe("failed")); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: expect.stringContaining("cannot migrate") } }, + }); + expect(provider.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + expect(errors.map(({ kind }) => kind)).toContain("object_failed"); + view.release(); + await host.close(context); + await store.close(context); + }); + + it("a failed provider's ref.member.get rejects with the failure's cause", async () => { + const store = await seeded(); + const { host } = await persistedNotes(store, persistedMemo(2)); + const provider = await host.get(NoteStore, {}, context); + expect(await rejection(provider.notes.get("a", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + expect(await rejection(provider.notes.get("b", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + await host.close(context); + await store.close(context); + }); + + it("a provider returning something else for a keyed member fails broken", async () => { + const { host } = await loaded((h) => { + // Plugin code returning a wrong value for the keyed member. + h.serve( + NoteStore, + () => ({ notes: constant({}), write: async () => null, remove: async () => null }) as never, + ); + }); + expect(await rejection(host.get(NoteStore, {}, context))).toMatchObject({ + message: expect.stringContaining("notes: a keyed member is a keyed document handle"), + }); + await host.close(context); + }); +}); + +describe("a binding released while it binds (fix round 5)", () => { + it("ends cleanly when its object's hold releases it at once, keeping nothing (Sol)", async () => { + const Type = local("app.reentrant", { version: 1, params: {}, members: { x: int() } }); + const facet = defineFacet((h) => h.serve(Type, () => ({ x: constant(1) }))); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/reentrant", facet }], context); + let ref = await host.get(Type, {}, context); + const released = new WeakRef(ref); + // The type is removed: the reference waits, with its last values. + await host.reload([], context); + const bind = Binding.prototype.bind; + const failures: unknown[] = []; + const binding = vi.spyOn(Binding.prototype, "bind").mockImplementation(function (this: Binding, target) { + try { + bind.call(this, target); + } catch (error) { + failures.push(error); + throw error; + } + }); + // `ChordObject.hold` lets a presence derive release a holder synchronously: the binding ends inside its bind. + const hold = ChordObject.prototype.hold; + const holding = vi.spyOn(ChordObject.prototype, "hold").mockImplementation(function (this: ChordObject, holder) { + hold.call(this, holder); + ref.release(); + }); + try { + await host.reload([{ name: "@app/reentrant", facet }], context); + expect(holding).toHaveBeenCalledOnce(); + } finally { + holding.mockRestore(); + binding.mockRestore(); + } + expect(failures).toEqual([]); + expect(ref.state.value.kind).toBe("ended"); + ref = undefined as never; + await expectCollected(released); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/retry-on.test.ts b/packages/chord/test/next/host/retry-on.test.ts new file mode 100644 index 000000000..31710c226 --- /dev/null +++ b/packages/chord/test/next/host/retry-on.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import type { FacetHost, Self, StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, failLive, flushed, instancesOf, loaded } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The models registry: the number of models contributed; it throws when told to. */ +const Models = defineRegistry("app.models", { version: 1 }); + +const Runtime = local("app.runtime", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Models` (throwing for a negative sum) and `Runtime`, contributing a writable count of models. */ +function serveModels(h: FacetHost): void { + h.serve(Models, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error("negative"); + return sum; + }, + }); + h.serve(Runtime, (_, self) => { + const count = self.writable(0); + self.contribute(Models, count); + return { + set: async (n: number) => { + count.set(n); + return null; + }, + }; + }); +} + +const User = local("app.user", { version: 1, params: { id: int() }, members: {} }); + +/** + * Serves `User`, whose open reads the models first and fails broken when there were none (after `wait`, if given); + * `opens` counts its attempts. + */ +function serveUser(h: FacetHost, opens: { n: number }, wait?: () => Promise): void { + h.serve(User, (_, self) => { + const models = self.registry(Models); + self.retryOn(models); + self.open( + async () => { + const seen = models.value; + opens.n++; + await wait?.(); + if (seen === 0) throw broken(new ChordError({ code: "unavailable", message: "no model" })); + }, + () => {}, + ); + return {}; + }); +} + +describe("retryOn", () => { + it("retries a failed object (a broken open) when a source gets a new value: the watches were installed before the open", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect(user.state.value).toEqual({ kind: "live" }); + expect(opens.n).toBe(2); + await host.close(context); + }); + + it("does nothing while the object is parked failed { waiting }: its dependencies decide (dependency model §3.8)", async () => { + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + let factories = 0; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + factories++; + self.retryOn(self.registry(Models)); + self.get(Missing, {}); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect([user.state.value.kind, factories]).toEqual(["failed", 1]); + await host.close(context); + }); + + it("wakes on value calls only: a throwing compose wakes nothing, its next good composition does", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + const runtime = await host.get(Runtime, {}, context); + await runtime.set(-1, context); + await flushed(); + expect(opens.n).toBe(1); + await runtime.set(2, context); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("remembers a wake while it activates: that attempt failing is retried at once", async () => { + const opens = { n: 0 }; + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens, async () => { + if (opens.n === 1) await gate.promise; + }); + }); + const runtime = await host.get(Runtime, {}, context); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + // The first attempt read no model and waits; a model arrives meanwhile; then that attempt fails broken. + await runtime.set(1, context); + expect(user.state.value).toEqual({ kind: "loading" }); + gate.resolve(); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("ignores wakes while live", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + self.retryOn(self.registry(Models)); + self.open( + async () => void opens.n++, + () => {}, + ); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + await (await host.get(Runtime, {}, context)).set(3, context); + await flushed(); + expect(opens.n).toBe(1); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("refuses sources that end with an attempt: a reference's member, its own writable, `ref.state`", async () => { + const Other = local("app.other", { version: 1, params: {}, members: { n: int() } }); + const Picky = local("app.picky", { version: 1, params: { which: int() }, members: {} }); + const { host } = await loaded((h) => { + h.serve(Other, () => ({ n: constant(1) })); + h.serve(Picky, ({ which }, self) => { + const other = self.get(Other, {}); + const sources = [other.n, self.writable(1), other.state, constant(1)]; + self.retryOn(sources[which]!); + return {}; + }); + }); + for (const which of [0, 1, 2]) { + await expect(host.get(Picky, { which }, context)).rejects.toThrow( + `app.picky(${which}): self.retryOn takes registry values and constants (sources that outlive the attempt)`, + ); + } + (await host.get(Picky, { which: 3 }, context)).release(); + await host.close(context); + }); + + it("are the object's: a failed object that ends releases them, and is collected while the registry lives", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const refs = await (async () => { + const user = host.hold(User, { id: 1 }, context); + await flushed(); + user.release(); + await flushed(); + return objects.filter((ref) => ref.deref()?.label === "app.user(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await expectCollected(...refs); + await host.close(context); + }); + + it("does not wake an object by its own contributions leaving (it failed live: no retry without backoff)", async () => { + const Contributor = local("app.contributor", { version: 1, params: {}, members: {} }); + let contributor: StoreSelf | undefined; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(Contributor, (_, self) => { + contributor = self; + self.contribute(Models, 1); + self.retryOn(self.registry(Models)); + return {}; + }); + }); + const models = await host.registry(Models, context); + const watched: number[] = []; + (await models.watch(context)).start(async (call) => void (call.kind === "value" && watched.push(call.value))); + failLive(objects, contributor as Self, new ChordError({ code: "unavailable", message: "boom" })); + await flushed(); + const reference = host.hold(Contributor, {}, context); + await flushed(); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + expect(watched).toEqual([0]); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts new file mode 100644 index 000000000..a73b222f8 --- /dev/null +++ b/packages/chord/test/next/host/revocation.test.ts @@ -0,0 +1,586 @@ +// Tests of revocation (§4, R10, R11): revocation keys, \`host.revoke\` on one host (objects, running calls, activations, +// holds bound or not), child links and \`revoke\` messages, GC, and revocation across a relay. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { hostKind, int, local, remote } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + expectCollected, + forEachCodec, + instancesOf, + Notes, + PlacedNotes, + RawPeer, + rawParent, + rawWorkers, + rejection, + relayed, + SERVER_ADDRESS, + Server, + Session, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Every object attempted in this file, weakly. */ +const objects = instancesOf(ChordObject, "attempt"); + +/** The test app's revocations: a guest's session, or every owner. */ +type Revocation = { session: string } | { owners: true }; + +const revokedBy = (principal: TestPrincipal): Revocation[] => + principal.kind === "guest" + ? [{ session: principal.session }] + : principal.kind === "owner" + ? [{ owners: true }] + : principal.kind === "worker" + ? [{ session: principal.session }] + : []; + +const S1: TestPrincipal = { kind: "guest", session: "S1" }; + +/** A host-lifetime type whose activation waits for an open: `load` waits for it (the start barrier). */ +const Barrier = local("app.barrier", { version: 1, params: {}, members: { n: int() } }); + +/** A guest's variant: its `wait` method's index. */ +const { wait } = wireNames(Notes, "reader").methods as { wait: number }; + +/** A worker of session S1 serving `Notes` (as `serveNotes`, at the address `Notes` names), its parent a raw peer whose holds name their principal. */ +async function worker(setup: (host: FacetHost) => void = (h) => void serveNotes(h)) { + const host = createHost({ + address: Server(), + principal: { kind: "worker", session: "S1" }, + revokedBy, + onError: () => {}, + }); + await host.load([{ name: "@test/worker", facet: defineFacet(setup) }], context); + const parent = await rawParent(host); + return { host, parent }; +} + +/** Sends a hold of `Notes` for S1 with id `hold`, acting as `principal` (honoured: the peer is the parent). */ +function holdAs(peer: RawPeer, hold: number, principal: TestPrincipal, Type: typeof Notes = Notes): void { + peer.send({ + kind: "hold", + hold, + type: Type.name, + version: Type.version, + fingerprint: Type.fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + principal, + }); +} + +describe("revocation keys", () => { + it("are computed once per principal object, compared as canonical JSON", async () => { + let calls = 0; + const host = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy: () => { + calls++; + return [{ b: 2, a: 1 }]; + }, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveNotes(h)) }], context); + const peer = await rawParent(host); + holdAs(peer, 0, S1); + holdAs(peer, 1, { kind: "guest", session: "S2" }); + await peer.next("bound"); + await peer.next("bound"); + host.revoke({ a: 1, b: 2 }, context); + host.revoke({ a: 1, b: 3 }, context); + await transported(); + expect(calls).toBe(2); + expect((await peer.next("gone")).hold).toBe(0); + expect((await peer.next("gone")).hold).toBe(1); + }); + + it("keep no principal alive (a WeakMap keyed by the principal the host cached them for)", async () => { + const cached: WeakRef[] = []; + const host = createHost({ + address: Server(), + principal: { kind: "worker", session: "S1" }, + revokedBy: (principal) => { + // The principal decoded on this host, the cache's key: not the sender's object. + cached.push(new WeakRef(principal)); + return revokedBy(principal); + }, + onError: () => {}, + }); + await host.load([{ name: "@test/worker", facet: defineFacet((h) => serveNotes(h)) }], context); + const parent = await rawParent(host); + holdAs(parent, 0, { kind: "guest", session: "S9" }); + await parent.next("bound"); + host.revoke({ session: "S1" }, context); + parent.send({ kind: "release", hold: 0 }); + await transported(); + expect(cached.length).toBeGreaterThan(0); + await expectCollected(...cached); + // The cache's owner is still alive and working. + holdAs(parent, 1, S1); + await parent.next("bound"); + host.revoke({ session: "S1" }, context); + expect((await parent.next("gone")).hold).toBe(1); + await host.close(context); + }); +}); + +forEachCodec(() => { + describe("revoke on one host", () => { + it("ends the principal's objects at once, whatever references remain: each hold gets gone forbidden", async () => { + const { host, parent } = await worker(); + holdAs(parent, 0, S1); + holdAs(parent, 1, S1); + holdAs(parent, 2, { kind: "owner" }); + for (let i = 0; i < 3; i++) await parent.next("bound"); + host.revoke({ session: "S1" }, context); + await transported(); + const gone = [await parent.next("gone"), await parent.next("gone")]; + expect(gone.map(({ hold }) => hold).sort()).toEqual([0, 1]); + expect(gone[0]!.error).toEqual({ + kind: "chord", + error: { code: "forbidden", reason: "revoked", message: "app.notes(S1): revoked" }, + }); + // The owner's hold, the parent link and the local store stay. + expect(parent.closed).toBe(false); + expect(parent.kinds()).toEqual([]); + holdAs(parent, 3, S1); + expect((await parent.next("bound")).holds[0]!.hold).toBe(3); + }); + + it("aborts running calls (race 2): the handler's context, no result; a call crossing the gone is ignored", async () => { + let probe: ReturnType | undefined; + const { host, parent } = await worker((h) => { + probe = serveNotes(h); + }); + holdAs(parent, 0, S1); + await parent.next("bound"); + parent.send({ kind: "call", call: 0, hold: 0, method: wait }); + await transported(); + host.revoke({ session: "S1" }, context); + expect(probe!.waits[0]!.abortSignal?.reason).toMatchObject({ code: "forbidden" }); + expect((await parent.next("gone")).hold).toBe(0); + await transported(); + expect(parent.kinds()).toEqual([]); + parent.send({ kind: "call", call: 1, hold: 0, method: wait }); + await transported(); + expect(parent.kinds()).toEqual([]); + }); + + it("abandons an activation in progress (race 1): its open's context is aborted, its holders get gone", async () => { + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const aborted: unknown[] = []; + const { host, parent } = await worker((h) => + h.serve(Slow, (_, _principal, self) => { + self.open( + (opening) => + new Promise((_, reject) => + opening.abortSignal?.addEventListener("abort", () => { + aborted.push(opening.abortSignal?.reason); + reject(opening.abortSignal?.reason); + }), + ), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + parent.send({ + kind: "hold", + hold: 0, + type: Slow.name, + version: Slow.version, + fingerprint: Slow.fingerprint, + params: "{}", + address: SERVER_ADDRESS, + principal: S1, + }); + await transported(); + host.revoke({ session: "S1" }, context); + expect((await parent.next("gone")).error).toMatchObject({ kind: "chord", error: { code: "forbidden" } }); + expect(aborted).toMatchObject([{ code: "forbidden" }]); + }); + + it("never revokes the host's own principal", async () => { + const { host } = await worker(); + const own = await host.get(Notes, { session: "S1" }, context); + host.revoke({ session: "S1" }, context); + expect(own.state.value.kind).toBe("live"); + await host.close(context); + }); + + it("ends a hold the parent sent before its revoke while it still waits for load (race 3): gone, never bound", async () => { + for (const delayed of [false, true]) { + // A worker connects before it loads (W6); a load can wait for an activation (the start barrier). + const opened = Promise.withResolvers(); + const host = createHost({ + address: Server(), + principal: { kind: "worker", session: "S1" }, + revokedBy, + onError: () => {}, + }); + const parent = await rawParent(host); + const facet = defineFacet((h) => { + serveNotes(h); + if (delayed) { + h.serve(Barrier, (_params, self) => { + self.open( + () => opened.promise, + () => {}, + ); + return { n: constant(0) }; + }); + } + }); + const loading = host.load([{ name: "@test/worker", facet }], context); + holdAs(parent, 0, S1); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + await transported(); + opened.resolve(); + await loading; + await transported(); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + expect(parent.kinds()).toEqual([]); + expect(host.references(Notes).value).toEqual([]); + await host.close(context); + } + }); + + it("ends a relayed hold of the principal still waiting for its route's link: never held there", async () => { + const host = createHost({ + principal: { kind: "worker", session: "S0" }, + revokedBy, + onError: () => {}, + }); + const resolving = Promise.withResolvers(); + const routed: RawPeer[] = []; + host.route( + Session, + { + authorize: () => true, + resolve: async () => { + await resolving.promise; + const [a, b] = memoryLinkPair(); + routed.push(new RawPeer(b)); + return { link: a, principal: { kind: "worker", session: "S1" }, codec: testCodec() }; + }, + }, + context, + ); + await host.load([], context); + const parent = await rawParent(host); + const { name: type, version, fingerprint } = PlacedNotes; + const address = { kind: "session", params: { id: "S1" } }; + parent.send({ + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address, + principal: S1, + }); + await transported(); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + resolving.resolve(); + await transported(); + await transported(); + expect(routed.flatMap((peer) => peer.received.filter(({ kind }) => kind === "hold"))).toEqual([]); + await host.close(context); + }); + + it("ends an unbound hold of the principal (refused, waiting for a provider): gone, nothing left of it", async () => { + const { host, parent } = await worker(); + const Absent = { ...Notes, name: "app.absent" } as unknown as typeof Notes; + holdAs(parent, 0, S1, Absent); + expect(await parent.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + await transported(); + expect(host.activity.value.references).toBe(0); + await host.close(context); + }); + + it("binds a hold made for a revoked principal, then the parent's revoke after it ends it (race 3, R10's window)", async () => { + const { host, parent } = await worker(); + holdAs(parent, 0, S1); + await parent.next("bound"); + // The parent's revoke follows the hold on the same link, so it reaches the worker after it. + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect((await parent.next("gone")).error).toMatchObject({ kind: "chord", error: { code: "forbidden" } }); + await host.close(context); + }); + }); + + describe("links and revoke messages", () => { + it("closes the child links whose principal it revokes, never the parent link", async () => { + const host = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy, + onError: () => {}, + }); + await host.load([], context); + const peers = [S1, { kind: "guest", session: "S2" } as const, S1].map((principal) => { + const [a, b] = memoryLinkPair(); + host.accept(a, { principal, codec: testCodec() }, context); + return new RawPeer(b); + }); + const parent = await rawParent(host); + host.revoke({ session: "S1" }, context); + await transported(); + // Two tabs of one grant: both links close (race 4). + expect(peers.map(({ closed }) => closed)).toEqual([true, false, true]); + expect(parent.closed).toBe(false); + // A revocation is no message for a child, nor for the parent. + expect(peers[1]!.kinds()).toEqual(["hello"]); + expect(parent.kinds()).toEqual([]); + }); + + it("ignores a revoke from a child link (race 6)", async () => { + const host = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveNotes(h)) }], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: S1, codec: testCodec() }, context); + const child = new RawPeer(b); + await child.next("hello"); + holdAs(child, 0, S1); + await child.next("bound"); + child.send({ kind: "revoke", revocation: { session: "S1" } }); + await transported(); + expect(child.closed).toBe(false); + expect(child.kinds()).toEqual([]); + }); + + it("sends revoke on every route link, and on none made later (race 5: nothing is replayed)", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const server = relay.server as unknown as Host; + relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + server.revoke({ session: "S7" }, context); + await transported(); + expect(workers[0]!.received.filter(({ kind }) => kind === "revoke")).toEqual([ + { kind: "revoke", revocation: { session: "S7" } }, + ]); + relay.tab.hold(PlacedNotes, { session: "S2" }, context); + await transported(); + expect(workers[1]!.received.some(({ kind }) => kind === "revoke")).toBe(false); + }); + }); +}); + +describe("GC of revoked objects", () => { + it("keeps no revoked object, however many references its holders still have", async () => { + const { host, refs, parent } = await (async () => { + const { host, parent } = await worker(); + const made = objects.length; + holdAs(parent, 0, S1); + await parent.next("bound"); + const refs = objects.slice(made); + expect(refs.length).toBeGreaterThan(0); + host.revoke({ session: "S1" }, context); + await parent.next("gone"); + return { host, refs, parent }; + })(); + await expectCollected(...refs.filter((ref: WeakRef) => ref.deref()?.served.kind === "remote")); + expect(parent.closed).toBe(false); + await host.close(context); + }); +}); + +describe("revocation across a relay", () => { + it("closes a revoked tab's link at the server and ends its objects on the worker", async () => { + const relay = await relayed({ tab: S1, revokedBy }); + const notes = relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("live"); + const worker = relay.workers.get("session|S1")!; + const present = worker.references(PlacedNotes); + expect(present.value).toHaveLength(1); + const server = relay.server as unknown as Host; + server.revoke({ session: "S1" }, context); + await transported(); + await transported(); + // The tab's link closed: its reference waits for a new link, its holds on the worker are released. + expect(notes.state.value.kind).toBe("loading"); + expect(present.value).toEqual([]); + }); + + it("ends the worker's objects of the principal through the revoke message alone; the relay forwards their gone", async () => { + // The server's own revokedBy names no guest: its tab link stays, only the worker applies the revocation. + const relay = await relayed({ tab: S1, revokedBy, serverRevokedBy: () => [] }); + const notes = relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("live"); + const server = relay.server as unknown as Host; + server.revoke({ session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("ended"); + expect(relay.workers.get("session|S1")!.references(PlacedNotes).value).toEqual([]); + expect(relay.errors.tab).toEqual([]); + // The tab's link stayed open: a new hold binds. + const again = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + expect(again.state.value.kind).toBe("live"); + }); +}); + +describe("revocation of this host's own requests over its paths (fix round 1)", () => { + type Guest = { kind: "owner" } | { kind: "guest"; grant: string }; + const There = hostKind("app-there"); + const Remote = remote("app.remote", { + version: 1, + params: {}, + hostAddress: () => There(), + variants: { view: { state: {}, methods: {} } }, + }); + const guestRevokedBy = (p: Guest) => (p.kind === "guest" ? [p.grant] : []); + + it("ends a guest's ctx.get waiting for its route before any link exists, at once (Astra 1, Sol 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route(There, { authorize: () => true, resolve: () => new Promise(() => {}) }, context); + await host.load([], context); + let outcome: unknown; + void rejection( + host.root((ctx) => ctx.get(Remote, {}, { readyTimeout: 60_000 }), context, { + principal: { kind: "guest", grant: "g" }, + }), + ).then((error) => { + outcome = error; + }); + await transported(); + host.revoke("g", context); + await transported(); + expect(outcome).toMatchObject({ code: "forbidden", reason: "revoked" }); + await host.close(context); + }); + + it("never lets the revoked request bind on the route's later link (Astra 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const gate = Promise.withResolvers<{ link: Link; principal: Guest; trusted: boolean }>(); + host.route(There, { authorize: () => true, resolve: () => gate.promise }, context); + await host.load([], context); + const pending = rejection( + host.root( + async (ctx) => { + await ctx.get(Remote, {}, { readyTimeout: 500 }); + throw new Error("revoked request became live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ), + ); + await transported(); + host.revoke("g", context); + const worker = createHost({ + address: There(), + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const [up, down] = memoryLinkPair(); + worker.connect(up, { principal: { kind: "owner" }, trusted: true }, context); + const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "view", members: {} }))); + await worker.load([{ name: "@test/late", facet }], context); + gate.resolve({ link: down, principal: { kind: "owner" }, trusted: true }); + expect(await pending).toMatchObject({ code: "forbidden", reason: "revoked" }); + await transported(); + expect(worker.activity.value.references).toBe(0); + await host.close(context); + await worker.close(context); + }); + + it("ends a guest's reference over a route link that dropped, and nothing holds it on the next one", async () => { + const routed: RawPeer[] = []; + // The drop resolves again at once (slice 10 R2); the next link comes only after the revocation. + let next: () => void = () => {}; + const revoked = new Promise((resolve) => { + next = resolve; + }); + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route( + There, + { + authorize: () => true, + resolve: async () => { + if (routed.length > 0) await revoked; + const [a, b] = memoryLinkPair(); + routed.push(new RawPeer(b)); + return { link: a, principal: { kind: "owner" }, codec: testCodec() }; + }, + }, + context, + ); + await host.load([], context); + const states: string[] = []; + void host.root( + async (ctx) => { + const held = await ctx.get(Remote, {}, { readyTimeout: 60_000 }).catch((error: Error) => error); + states.push(held instanceof Error ? held.message : "live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ); + await transported(); + const first = routed[0]!; + await first.next("hold"); + first.close(); + await transported(); + host.revoke("g", context); + next(); + await transported(); + expect(states).toEqual(["app.remote: revoked"]); + expect(routed.flatMap((peer, i) => (i === 0 ? [] : peer.received.filter(({ kind }) => kind === "hold")))).toEqual( + [], + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/root.test.ts b/packages/chord/test/next/host/root.test.ts new file mode 100644 index 000000000..0847d0477 --- /dev/null +++ b/packages/chord/test/next/host/root.test.ts @@ -0,0 +1,208 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, flushed, loaded, rejection, type TestPrincipal } from "../support.ts"; + +const Info = local("app.info", { version: 1, params: { id: int() }, members: { id: int(), wait: method() } }); + +/** Serves `Info`; its `wait` waits until the caller's context aborts. */ +function serveInfo(h: Parameters[0]>[0]): void { + h.serve(Info, ({ id }) => ({ + id: constant(id), + wait: async (ctx) => { + await sleep(60_000, ctx); + return null; + }, + })); +} + +describe("host.root", () => { + it("gives fn a call context: the host's principal (or the given one), this host's incarnation, `ctx.get`", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + const seen = await host.root(async (ctx) => { + const info = await ctx.get(Info, { id: 7 }); + return { principal: ctx.principal, incarnation: ctx.incarnation, id: info.id.value }; + }, context); + expect(seen).toMatchObject({ principal: { kind: "server" }, id: 7 }); + const owner = { kind: "owner", access: "local" } as const; + expect(await host.root(async (ctx) => ctx.principal, context, { principal: owner })).toEqual(owner); + await host.close(context); + }); + + it("runs anonymous with an explicit undefined principal; only an omitted one takes the host's (slice 10 §6)", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([], context); + expect(await host.root(async (ctx) => ctx.principal, context, { principal: undefined })).toBeUndefined(); + expect(await host.root(async (ctx) => ctx.principal, context, {})).toEqual({ kind: "server" }); + await host.close(context); + }); + + // Opus's build review F5: an anonymous scope's `ctx.get` of a remote type served here failed on a WeakMap key. + it("refuses an anonymous scope a remote type served here (forbidden); a local one it gets", async () => { + const Server = hostKind("server"); + const Secrets = remote("app.secrets", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { read: method(undefined, str()) } } }, + }); + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => { + serveInfo(h); + h.serve(Secrets, () => ({ variant: "main", members: { read: async () => "secret" } })); + }); + await host.load([{ name: "@app/secrets", facet }], context); + const anonymous = { principal: undefined }; + const refused = await rejection(host.root(async (ctx) => ctx.get(Secrets, {}), context, anonymous)); + expect(refused).toMatchObject({ + code: "forbidden", + message: "app.secrets: an anonymous caller gets no remote object", + }); + expect(await host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context, anonymous)).toBe(3); + await host.close(context); + }); + + it("seals the context when fn settles: its references released, its signal aborted, `ctx.get` refused `released`", async () => { + const { host } = await loaded(serveInfo); + let kept: Ctx | undefined; + const info = await host.root(async (ctx) => { + kept = ctx; + return ctx.get(Info, { id: 1 }); + }, context); + expect(info.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(await rejection(kept!.get(Info, { id: 1 }))).toMatchObject({ code: "released", reason: "settled" }); + // A throwing fn seals too, and rejects with its error. + await expect( + host.root(async () => { + throw new Error("fn failed"); + }, context), + ).rejects.toThrow("fn failed"); + await host.close(context); + }); + + it("an abort rejects at once and seals: a running call through its reference rejects, a later get is `released`", async () => { + const { host } = await loaded(serveInfo); + const { context: cancellable, cancel } = withCancel(context); + let kept: Ctx | undefined; + let call: Promise | undefined; + const running = host.root(async (ctx) => { + kept = ctx; + const info = await ctx.get(Info, { id: 1 }); + call = info.wait(context); + // Uncooperative: it waits on nothing of Chord's. + await new Promise(() => {}); + }, cancellable); + await flushed(); + cancel(new Error("stop")); + expect(await rejection(running)).toEqual(new Error("stop")); + expect(await rejection(call!)).toMatchObject({ code: "released" }); + expect(await rejection(kept!.get(Info, { id: 2 }))).toMatchObject({ code: "released", reason: "settled" }); + await host.close(context); + }); + + it("an aborted context rejects before fn runs; a synchronous throw is a rejection", async () => { + const { host } = await loaded(serveInfo); + const { context: aborted, cancel } = withCancel(context); + cancel(new Error("gone")); + let ran = false; + const fn = async (): Promise => { + ran = true; + }; + await expect(host.root(fn, aborted)).rejects.toThrow("gone"); + expect(ran).toBe(false); + await expect( + host.root( + (() => { + throw new Error("sync"); + }) as never, + context, + ), + ).rejects.toThrow("sync"); + await host.close(context); + }); + + it("a `ctx.get` before `load` resolved parks until it did", async () => { + const host = createHost({ onError: () => {} }); + const result = host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context); + await flushed(); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + expect(await result).toBe(3); + await host.close(context); + }); + + it("`host.close` seals every open scope (an fn sleeping on its ctx ends); after it, `root` rejects `released`", async () => { + const { host } = await loaded(serveInfo); + const sleeping = rejection(host.root((ctx) => sleep(60_000, ctx), context)); + await flushed(); + await host.close(context); + expect(await sleeping).toMatchObject({ code: "released", reason: "closed" }); + expect(await rejection(host.root(async () => 1, context))).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("aborts the scope's own signal when fn settles and at `host.close`, the caller's context having one (Fable, Opus, Sol)", async () => { + const { host } = await loaded(serveInfo); + const settled = withCancel(context); + let kept: Ctx | undefined; + await host.root(async (ctx) => { + kept = ctx; + }, settled.context); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(settled.context.abortSignal?.aborted).toBe(false); + const closing = withCancel(context); + let slept: "running" | "ended" = "running"; + const sleeping = host.root(async (ctx) => { + try { + await sleep(60_000, ctx); + } finally { + slept = "ended"; + } + }, closing.context); + const outcome = rejection(sleeping); + await flushed(); + await host.close(context); + await flushed(); + expect(slept).toBe("ended"); + expect(closing.context.abortSignal?.aborted).toBe(false); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("`host.close` rejects the caller at once, whatever an uncooperative fn still does (Sol)", async () => { + const { host } = await loaded(serveInfo); + const running = host.root(() => new Promise(() => {}), context); + const outcome = running.then( + () => "resolved", + (error: unknown) => error, + ); + await host.close(context); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("leaves no listener on the caller's signal once `host.close` rejected an fn that never settles (Astra)", async () => { + const controller = new AbortController(); + const host = createHost({ onError: () => {} }); + const running = host.root(async () => new Promise(() => {}), withAbortSignal(controller.signal, context)); + const rejected = running.catch((error: unknown) => error); + await host.close(context); + expect(await rejected).toMatchObject({ code: "released", reason: "closed" }); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("keeps nothing once sealed: its scope and references are collected while the caller's signal lives on", async () => { + const { host } = await loaded(serveInfo); + const { context: long } = withCancel(context); + const refs = await (async () => { + const made: WeakRef[] = []; + await host.root(async (ctx) => { + made.push(new WeakRef(ctx), new WeakRef(await ctx.get(Info, { id: 1 }))); + }, long); + return made; + })(); + await expectCollected(...refs); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts new file mode 100644 index 000000000..3d9fbf1f8 --- /dev/null +++ b/packages/chord/test/next/host/route.test.ts @@ -0,0 +1,491 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, createContextKey, withContextValue } from "../../../src/context/index.ts"; +import { hostKind, int, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { + addNote, + expectCollected, + fakeTimers, + flushed, + forEachCodec, + PlacedNotes, + rejection, + relayed, + Server, + Session, + serveNotes, + type TestPrincipal, + transported, +} from "../support.ts"; + +/** Advances fake time by `ms`, then lets links and promises settle. */ +async function elapse(ms: number): Promise { + await vi.advanceTimersByTimeAsync(ms); + await transported(); +} + +forEachCodec(() => { + describe("route admission", () => { + it("runs one resolve per address, shared by the holds that wait for it (race 1)", async () => { + const { tab, resolves, workers } = await relayed(); + const [one, two] = await Promise.all([ + tab.get(PlacedNotes, { session: "S1" }, context), + tab.get(PlacedNotes, { session: "S1" }, context), + ]); + expect(resolves).toEqual(["session|S1"]); + await addNote(workers.get("session|S1")!, "S1", "a"); + await transported(); + expect(one.notes.value).toEqual([{ text: "a" }]); + expect(two.notes.value).toEqual([{ text: "a" }]); + await tab.get(PlacedNotes, { session: "S2" }, context); + expect(resolves).toEqual(["session|S1", "session|S2"]); + }); + + it("authorizes every hold with its principal: a refusal is broken forbidden, other principals still resolve (race 4)", async () => { + const authorize = (principal: TestPrincipal, address: { readonly key: string }) => + principal.kind === "owner" || + (principal.kind === "guest" && address.key === `session|${principal.session}`); + const guest = await relayed({ tab: { kind: "guest", session: "S1" }, authorize }); + expect(await rejection(guest.tab.get(PlacedNotes, { session: "S2" }, context))).toEqual({ + code: "forbidden", + message: "session|S2: not authorized", + }); + expect(guest.resolves).toEqual([]); + expect((await guest.tab.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("reader"); + const owner = await relayed({ authorize }); + expect((await owner.tab.get(PlacedNotes, { session: "S2" }, context)).variant).toBe("editor"); + }); + + it("ref.retry() holds a refused hold again: authorize admitting it later, over a link and on this host (audit)", async () => { + let admit = false; + const { tab, server } = await relayed({ authorize: () => admit }); + const overLink = tab.hold(PlacedNotes, { session: "S1" }, context); + const here = server.hold(PlacedNotes, { session: "S2" }, context); + await transported(); + for (const ref of [overLink, here]) { + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "forbidden" } }, + }); + } + admit = true; + overLink.retry(); + here.retry(); + await transported(); + expect(overLink.state.value).toEqual({ kind: "live" }); + expect(here.state.value).toEqual({ kind: "live" }); + }); + + it("ref.retry() of a reference refused on this host after host.close ends it released, nothing resolved (audit)", async () => { + let admit = false; + const { server, resolves } = await relayed({ authorize: () => admit }); + const ref = server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(ref.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "forbidden" } } }); + await server.close(context); + admit = true; + ref.retry(); + await transported(); + expect(ref.state.value).toMatchObject({ kind: "ended", error: { code: "released", reason: "closed" } }); + expect(resolves).toEqual([]); + }); + + it("fails a hold broken with what authorize throws", async () => { + const { tab } = await relayed({ + authorize: () => { + throw new Error("authorize threw"); + }, + }); + expect(await rejection(tab.get(PlacedNotes, { session: "S1" }, context))).toMatchObject({ + message: "app.placed-notes: authorize threw", + }); + }); + + it("fails a hold broken invalid/host_address for an address whose params do not fit the kind", async () => { + const Misplaced = remote("app.misplaced", { + version: 1, + params: { n: int() }, + // An address of the right kind with params of the wrong type, as a buggy `hostAddress` could compute. + hostAddress: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, + variants: { v: { state: { n: int() } } }, + }); + const { tab, resolves } = await relayed(); + expect(await rejection(tab.get(Misplaced, { n: 1 }, context))).toMatchObject({ + code: "invalid", + reason: "host_address", + }); + expect(resolves).toEqual([]); + }); + }); + + describe("resolve failures and re-resolves (no timer, slice 10 R2)", () => { + fakeTimers(); + + it("a rejected resolve fails its holders waiting with its error, is reported, and runs no timer (race 2)", async () => { + let attempts = 0; + const { tab, server, errors } = await relayed({ + resolve: async () => { + attempts++; + throw new Error("no worker"); + }, + }); + // One relayed from the tab, one the server's own. + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + const own = server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + for (const ref of [notes, own]) { + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: no worker" } }, + }); + } + expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a new hold resolves a failed address again; the earlier holders bind on its link without a re-hold (Q1)", async () => { + let fail = true; + const { tab, resolves } = await relayed({ + resolve: async (_address, worker) => { + if (fail) throw new Error("no worker"); + return worker(); + }, + }); + const first = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(first.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + fail = false; + const second = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(resolves).toEqual(["session|S1"]); + expect([first.state.value, second.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("RouteHandle.retry resolves a failed address now; a retry of an address nobody holds keeps nothing", async () => { + let fail = true; + const { tab, route, resolves } = await relayed({ + resolve: async (_address, worker) => { + if (fail) throw new Error("no worker"); + return worker(); + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + fail = false; + route.retry(Session({ id: "S1" })); + route.retry(Session({ id: "S9" })); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(resolves).toEqual(["session|S1"]); + }); + + it("a returned link closed already fails the holders", async () => { + const { tab } = await relayed({ + resolve: async (address) => { + const [a, b] = memoryLinkPair(); + b.close(); + await flushed(); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "session|S1: the link closed before its handshake" } }, + }); + }); + + it("a link closing before its handshake fails its holders: no re-resolve loop over a worker dying at start", async () => { + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; + const [a, b] = memoryLinkPair(); + setTimeout(() => b.close(), 10); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await elapse(10); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a dropped link that completed its handshake resolves again at once, once (race 6)", async () => { + const { tab, resolves, routeLinks, workers } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const first = workers.get("session|S1")!; + routeLinks[0]!.close(); + await transported(); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect(workers.get("session|S1")).not.toBe(first); + expect(notes.state.value).toEqual({ kind: "live" }); + await addNote(workers.get("session|S1")!, "S1", "fresh"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "fresh" }]); + }); + + it("a link whose peer never greets closes after 30 s of silence and fails its holders", async () => { + // The heartbeat's timer too, before the hosts start it. + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; + // Its other end is never read: no hello, no close. + const [a] = memoryLinkPair(); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await elapse(29_000); + expect(notes.state.value).toEqual({ kind: "loading" }); + // The heartbeat checks every 1.5 s: the silent link closes within it. + await elapse(2_500); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("an address without holds is not resolved again after its link drops", async () => { + const { tab, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + notes.release(); + await transported(); + routeLinks[0]!.close(); + await elapse(60_000); + expect(resolves).toEqual(["session|S1"]); + }); + + it("a resolve whose holds were all released meanwhile finishes and its link is kept (race 3)", async () => { + let finish: () => void = () => {}; + const { tab, resolves, routeLinks, server } = await relayed({ + resolve: async (_address, worker) => { + await new Promise((resolve) => { + finish = resolve; + }); + return worker(); + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + notes.release(); + await transported(); + finish(); + await transported(); + expect(resolves).toEqual(["session|S1"]); + let closed = false; + routeLinks[0]!.onClose(() => { + closed = true; + }); + await elapse(60_000); + expect(closed).toBe(false); + // The kept link serves the next hold without a resolve. + await tab.get(PlacedNotes, { session: "S1" }, context); + expect(resolves).toEqual(["session|S1"]); + await server.close(context); + expect(closed).toBe(true); + }); + }); + + describe("RouteHandle.suspend (R1, Q3)", () => { + it("resolves nothing while held: the holders fail waiting suspended once the link is gone; retry lifts it", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + expect(notes.state.value).toEqual({ kind: "live" }); + routeLinks[0]!.close(); + await transported(); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: suspended" } }, + }); + const another = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(another.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(resolves).toEqual(["session|S1"]); + route.retry(Session({ id: "S1" })); + await transported(); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect([notes.state.value, another.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("goes with the last holder: a later hold resolves on demand; an unheld address is never suspended", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + routeLinks[0]!.close(); + await transported(); + notes.release(); + await transported(); + route.suspend(Session({ id: "S1" })); + await tab.get(PlacedNotes, { session: "S1" }, context); + expect(resolves).toEqual(["session|S1", "session|S1"]); + }); + }); + + describe("host.close during a resolve (race 7)", () => { + it("aborts the resolve's context and closes a link returned afterwards", async () => { + let routing: AbortSignal | undefined; + let finish: () => void = () => {}; + const { tab, server, routeLinks } = await relayed({ + resolve: async (_address, worker, context) => { + routing = context.abortSignal; + await new Promise((resolve) => { + finish = resolve; + }); + return worker(); + }, + }); + tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(routing?.aborted).toBe(false); + await server.close(context); + expect(routing?.aborted).toBe(true); + finish(); + await transported(); + let closed = false; + routeLinks[0]!.onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + }); + }); + + describe("declaring routes", () => { + it("is done before load, once per kind", async () => { + const host = createHost({ onError: () => {} }); + const options = { authorize: () => true, resolve: () => new Promise(() => {}) }; + host.route(Session, options, context); + expect(() => host.route(Session, options, context)).toThrow("route: session is routed already"); + await host.load([], context); + expect(() => host.route(hostKind("other"), options, context)).toThrow( + "route: routes are declared before load", + ); + }); + }); + + describe("a route's context", () => { + it("is the parent of its resolves and of the work arriving on its links", async () => { + const App = createContextKey("app"); + const Who = remote("app.who", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { methods: { app: method(undefined, str()) } } }, + }); + const resolved: (string | undefined)[] = []; + let worker: Host | undefined; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + server.route( + Session, + { + authorize: () => true, + resolve: async (address, routing) => { + resolved.push(routing.value(App)); + const [a, b] = memoryLinkPair(); + worker = createHost({ + address, + principal: { kind: "worker", session: "S1" }, + onError: () => {}, + }); + worker.connect(b, { principal: { kind: "server" } }, context); + await worker.load( + [ + { + name: "@test/worker", + facet: defineFacet((h) => void serveNotes(h, PlacedNotes)), + }, + ], + context, + ); + return { link: a, principal: { kind: "worker", session: "S1" } }; + }, + }, + withContextValue(App, "the app's", context), + ); + const facet = defineFacet((h) => + h.serve(Who, () => ({ variant: "v", members: { app: async (ctx) => ctx.value(App) ?? "none" } })), + ); + await server.load([{ name: "@test/server", facet }], context); + server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + // The worker's call reaches the server over the route link: its handler runs under the route's context. + const who = await worker!.get(Who, {}, context); + expect(await who.app(context)).toBe("the app's"); + expect(resolved).toEqual(["the app's"]); + }); + }); + + describe("GC", () => { + it("a failed or suspended address keeps no route state once its holders are released", async () => { + const { refs, server } = await (async () => { + const refs: WeakRef[] = []; + const { tab, server, route, routeLinks, workers, errors } = await relayed({ + authorize: (_principal, address) => { + refs.push(new WeakRef(address)); + return true; + }, + resolve: async (address, worker) => { + if (address.params.id === "failed") throw new Error("no worker"); + return worker(); + }, + }); + const failed = tab.hold(PlacedNotes, { session: "failed" }, context); + const held = await tab.get(PlacedNotes, { session: "held" }, context); + route.suspend(Session({ id: "held" })); + routeLinks[0]!.close(); + await transported(); + for (const ref of [failed, held]) ref.release(); + await transported(); + // The reported error's stack keeps the test's `resolve` frame (and its address) alive, the worker host its + // own address: not the route's. + errors.server.length = 0; + routeLinks.length = 0; + workers.clear(); + return { refs, server }; + })(); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + + it("a closed route link and its released references keep no route state", async () => { + const { refs, tab } = await (async () => { + const { tab, routeLinks, workers } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const refs = [new WeakRef(notes), new WeakRef(workers.get("session|S1")!), new WeakRef(routeLinks[0]!)]; + notes.release(); + await transported(); + routeLinks[0]!.close(); + workers.clear(); + routeLinks.length = 0; + await transported(); + return { refs, tab }; + })(); + await expectCollected(...refs); + expect(tab).toBeDefined(); + }); + + it("holds released while their resolve never settles are not kept", async () => { + const { refs, server } = await (async () => { + const { tab, server } = await relayed({ resolve: () => new Promise(() => {}) }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + notes.release(); + await transported(); + return { refs: [new WeakRef(notes)], server }; + })(); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); diff --git a/packages/chord/test/next/host/routines.test.ts b/packages/chord/test/next/host/routines.test.ts new file mode 100644 index 000000000..7dfa40565 --- /dev/null +++ b/packages/chord/test/next/host/routines.test.ts @@ -0,0 +1,1156 @@ +// Tests of routines (§14): `on` (the first call with the current value, folding, no overlap, retries after a throw +// with the whole value, retirements, the source ending or failing, native sources, facets), `onLive` (returning, +// retries, the backoff reset after a long run, aborts), the drain and `host.close`, and what routines leave behind. + +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { + type Ctx, + createHost, + defineFacet, + type HostError, + type RefState, + type Self, + sleep, + untilAborted, +} from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { + NativeWatch, + type NativeWatchable, + type NativeWatchCall, + Watch, + type Watchable, + type WatchCall, +} from "../../../src/watch.ts"; +import { + EDITOR, + expectCollected, + fakeTimers, + flushed, + linked, + loaded, + Notes, + rawParent, + Server, + serveNotes, + type TestPrincipal, + transported, +} from "../support.ts"; + +/** What an `on` handler gets: a watch's value call. */ +type WatchValueCall = Extract, { kind: "value" }>; +type NativeWatchValueCall = Extract, { kind: "value" }>; + +const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +/** A host-lifetime object with a counter document and `bump`; its factory declares routines with `routines`. */ +const Store = local("app.store", { version: 1, params: {}, members: { n: int(), bump: method() } }); + +/** The same with params: it lives while referenced. */ +const Item = local("app.item", { version: 1, params: { id: int() }, members: { n: int(), bump: method() } }); + +/** Loads a host serving `Store`; `declare` declares its routines over its document. */ +async function store( + declare: (doc: SingletonHandle<{ n: number }>, self: Self) => void, + options: { readonly closeTimeout?: number } = {}, +) { + const loadedHost = await loaded( + (h) => + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + declare(doc, self); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + options, + ); + const ref = await loadedHost.host.get(Store, {}, context); + return { ...loadedHost, ref }; +} + +/** Counts one up in one transaction. */ +async function bump(doc: SingletonHandle<{ n: number }>, ctx: Context): Promise { + await doc.change((d) => { + d.n++; + }, ctx); + return null; +} + +/** Waits one macrotask and a little: a call after a busy one waits for one (`settling`). */ +function settled(): Promise { + return new Promise((resolve) => setTimeout(resolve, 5)); +} + +/** The routine reports among `errors`, as `kind #position: message`. */ +function reports(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => + error.kind === "routine_failed" + ? [`${error.routine.kind} #${error.routine.position}: ${(error.error as Error).message}`] + : [], + ); +} + +/** A JSON watchable (Chord's shared `Watch`) the test advances by hand, as a Durable document would. */ +function jsonSource(initial: { n: number }) { + let value = initial; + const watches = new Set>(); + const source: Watchable<{ n: number }> = { + get value() { + return value; + }, + async watch(ctx) { + const watch: Watch<{ n: number }> = new Watch( + value, + () => watches.delete(watch), + () => {}, + ); + watches.add(watch); + if (ctx.abortSignal !== undefined) watch.observeAbort(ctx.abortSignal); + return watch; + }, + }; + const advance = (call: WatchCall<{ n: number }>): void => { + if (call.kind === "value") value = call.value; + for (const watch of watches) watch.advance(call, context); + }; + return { source, advance }; +} + +/** A native watchable the test sets and ends by hand. */ +function writable(initial: T) { + let value = initial; + const watches = new Set>(); + const source: NativeWatchable & { set(next: T): void; end(): void } = { + get value() { + return value; + }, + async watch(ctx) { + const watch: NativeWatch = new NativeWatch( + value, + () => watches.delete(watch), + () => {}, + ); + watches.add(watch); + if (ctx.abortSignal !== undefined) watch.observeAbort(ctx.abortSignal); + return watch; + }, + set(next) { + value = next; + for (const watch of watches) watch.advance(next, context); + }, + end() { + for (const watch of watches) watch.end({ kind: "ended" }); + }, + }; + return { source, watches }; +} + +describe("on", () => { + it("calls first with the current value as [['r', value]], then once per change with its ops", async () => { + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + }); + }); + await flushed(); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 1 }, [["s", ["n"], 1]]], + ]); + }); + + it("never overlaps calls: changes during a call fold into the next, which is not aborted by them", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + let running = 0; + let overlapped = false; + let aborted = false; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }, ctx) => { + overlapped ||= running > 0; + running++; + calls.push([value, ops]); + if (calls.length === 2) await gate.promise; + aborted ||= ctx.abortSignal?.aborted === true; + running--; + }); + }); + await flushed(); + await ref.bump(context); + await flushed(); + await ref.bump(context); + await ref.bump(context); + gate.resolve(); + await settled(); + expect(calls.map(([value]) => value)).toEqual([{ n: 0 }, { n: 1 }, { n: 3 }]); + expect(calls[2]![1]).toEqual([["s", ["n"], 3]]); + expect({ overlapped, aborted }).toEqual({ overlapped: false, aborted: false }); + }); + + describe("after a throw", () => { + fakeTimers(); + + it("reports, keeps the watch, and calls again with [['r', value]] after 1 s doubling; success resets", async () => { + const calls: [unknown, readonly Op[], number][] = []; + let failing = 3; + const { ref, errors } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (failing-- > 0) throw new Error(`fail ${calls.length}`); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000); + await vi.advanceTimersByTimeAsync(4_000); + expect(calls.map(([, , at]) => at)).toEqual([0, 1_000, 3_000, 7_000]); + expect(calls.every(([, ops]) => ops[0]![0] === "r")).toBe(true); + expect(reports(errors)).toEqual(["on #1: fail 1", "on #1: fail 2", "on #1: fail 3"]); + // Succeeded: the next change gets its ops, and a later throw backs off from 1 s again. + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + expect(calls.at(-1)!.slice(0, 2)).toEqual([{ n: 1 }, [["s", ["n"], 1]]]); + failing = 1; + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + expect(calls.map(([, , at]) => at).slice(-2)).toEqual([7_000, 8_000]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("retries a change folded behind the throwing call after the backoff, once: no third call, no timer", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[], number][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (calls.length === 1) { + await gate.promise; + throw new Error("first"); + } + }); + }); + await vi.advanceTimersByTimeAsync(0); + await ref.bump(context); + gate.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5_000); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 1_000], + ]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("makes one retry per throw; a change during a failing retry waits for its backoff; no timer left", async () => { + const calls: [unknown, number][] = []; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => { + calls.push([value, Date.now()]); + // The retry at 1 s changes the source, then throws too. + if (calls.length === 2) await bump(doc, context); + if (calls.length <= 2) throw new Error(`fail ${calls.length}`); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toEqual([ + [{ n: 0 }, 0], + [{ n: 0 }, 1_000], + [{ n: 1 }, 3_000], + ]); + expect(reports(errors)).toEqual(["on #1: fail 1", "on #1: fail 2"]); + expect(vi.getTimerCount()).toBe(0); + item.release(); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("backs off after a throw even when the failed call changed its own source: no burst", async () => { + const calls: number[] = []; + const { errors } = await store((doc, self) => { + self.on(doc, async (_change, ctx) => { + calls.push(Date.now()); + if (calls.length > 100) return; + // A reconciler that records its attempt in its own source, then fails. + await bump(doc, ctx); + throw new Error("downstream unavailable"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([0]); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000); + expect(calls).toEqual([0, 1_000, 3_000]); + expect(reports(errors)).toHaveLength(3); + }); + + it("retries with the value that arrived during the failed call, as a whole value", async () => { + const calls: [unknown, readonly Op[], number][] = []; + await store((doc, self) => { + self.on(doc, async ({ value, ops }, ctx) => { + calls.push([value, ops, Date.now()]); + if (calls.length > 1) return; + await bump(doc, ctx); + throw new Error("once"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 1_000], + ]); + }); + + it("folds changes made during a slow retry into one call with the newest value", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + if (calls.length === 1) throw new Error("first"); + if (calls.length === 2) await gate.promise; + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + for (let i = 0; i < 3; i++) await ref.bump(context); + gate.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("waits for the next good value when its source fails during the backoff: the retry is dropped", async () => { + const calls: unknown[] = []; + let set: ((n: number) => Promise) | undefined; + const { errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("compute"); + return d.n; + }); + set = (n) => + doc.change((d) => { + d.n = n; + }, context); + self.on(failing, async ({ value }) => { + calls.push(value); + if (value === 1) throw new Error("handler threw"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await set!(1); + await vi.advanceTimersByTimeAsync(0); + await set!(-1); + await vi.advanceTimersByTimeAsync(2_000); + expect(calls).toEqual([0, 1]); + expect(vi.getTimerCount()).toBe(0); + await set!(3); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([0, 1, 3]); + expect(reports(errors)).toEqual(["on #1: handler threw"]); + }); + + it("clears its retry timer when its object ends during the backoff", async () => { + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async () => { + throw new Error("always"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(1); + item.release(); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("reports a retirement again on the retry after the call that got it threw", async () => { + const { source, advance } = jsonSource({ n: 0 }); + const calls: WatchValueCall<{ n: number }>[] = []; + await store((_, self) => { + self.on(source, async (call) => { + calls.push(call); + if (calls.length === 2) throw new Error("retirement not handled"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + advance({ kind: "value", value: { n: 0 }, ops: [], retired: true }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls.slice(1)).toEqual([ + { kind: "value", value: { n: 0 }, ops: [], retired: true }, + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: true }, + ]); + }); + + it("calls at once with the newest whole value when the source changes during the backoff", async () => { + const calls: [unknown, readonly Op[], number][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (calls.length === 1) throw new Error("first"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(500); + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 500], + [{ n: 2 }, [["s", ["n"], 2]], 500], + ]); + // The backoff timer is gone: nothing calls again. + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toHaveLength(3); + }); + }); + + it("folds every change made during the first call into one next call: [0, 3], not [0, 1, 3]", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + for (let i = 0; i < 3; i++) await ref.bump(context); + await flushed(); + gate.resolve(); + await settled(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("follows an app's JSON watchable whose watch is Chord's shared Watch with ops, as its type says", async () => { + const { source, advance } = jsonSource({ n: 0 }); + const calls: [unknown, readonly Op[]][] = []; + await store((_, self) => { + self.on(source, async ({ value, ops }) => void calls.push([value, ops])); + }); + await flushed(); + advance({ kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: false }); + await flushed(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 1 }, [["s", ["n"], 1]]], + ]); + }); + + it("reports a retirement as retired, folded with the changes around it into one call, and goes on", async () => { + const gate = Promise.withResolvers(); + const { source, advance } = jsonSource({ n: 0 }); + const calls: WatchValueCall<{ n: number }>[] = []; + await store((_, self) => { + self.on(source, async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + // A Durable document retired (the last value, no ops) and created again, while the first call runs. + advance({ kind: "value", value: { n: 0 }, ops: [], retired: true }); + advance({ kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: false }); + gate.resolve(); + await settled(); + advance({ kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }); + await flushed(); + expect(calls).toEqual([ + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: false }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + { kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }, + ]); + }); + + it("folds a native source to its newest value after a busy call", async () => { + const gate = Promise.withResolvers(); + const { source } = writable("a"); + const calls: unknown[] = []; + await store((_, self) => { + self.on(source, async ({ value }: { value: string }) => { + calls.push(value); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + source.set("b"); + source.set("c"); + await flushed(); + gate.resolve(); + await settled(); + expect(calls).toEqual(["a", "c"]); + }); + + it("takes every value a busy call left buffered before calling again: [0, 20], also when it set them itself", async () => { + const { source } = writable(0); + const calls: unknown[] = []; + await store((_, self) => { + self.on(source, async ({ value }: { value: number }) => { + calls.push(value); + if (value === 0) for (let n = 1; n <= 20; n++) source.set(n); + }); + }); + await settled(); + expect(calls).toEqual([0, 20]); + const gate = Promise.withResolvers(); + const later = writable(0); + const seen: unknown[] = []; + await store((_, self) => { + self.on(later.source, async ({ value }: { value: number }) => { + seen.push(value); + if (value === 0) await gate.promise; + }); + }); + await flushed(); + for (let n = 1; n <= 20; n++) later.source.set(n); + // No flush in between: the watch still holds the values when the call ends. + gate.resolve(); + await settled(); + expect(seen).toEqual([0, 20]); + }); + + it("reports a source whose watch cannot be taken; a source that ended meanwhile ends it quietly", async () => { + const failing: NativeWatchable = { + value: 0, + watch: async () => { + throw new Error("no watch"); + }, + }; + const ended: NativeWatchable = { + value: 0, + watch: async () => { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + watch.end({ kind: "ended" }); + return watch; + }, + }; + const calls: unknown[] = []; + const { errors } = await store((_, self) => { + self.on(failing, async ({ value }) => void calls.push(value)); + self.on(ended, async ({ value }) => void calls.push(value)); + }); + await flushed(); + expect(calls).toEqual([]); + expect(reports(errors)).toEqual(["on #1: no watch"]); + }); + + it("ends without a report when its source ends", async () => { + const Other = local("app.other", { version: 1, params: { id: int() }, members: { n: int() } }); + let otherDoc: SingletonHandle<{ n: number }> | undefined; + const calls: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Other, (_, self) => { + otherDoc = self.state(Doc); + return { n: self.derive([otherDoc], ([d]) => d.n) }; + }); + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + // Another object's document: it ends with that object. + self.on(otherDoc!, async ({ value }) => void calls.push(value)); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + const other = await host.get(Other, { id: 1 }, context); + await host.get(Item, { id: 1 }, context); + await flushed(); + other.release(); + await flushed(); + expect(calls).toEqual([{ n: 0 }]); + expect(errors).toEqual([]); + }); + + it("is not called for its source's failures: it waits for the next value (a derive healing)", async () => { + const calls: unknown[] = []; + const { ref, errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n > 0 && d.n < 3) throw new Error("compute"); + return d.n; + }); + self.on(failing, async ({ value, ops }) => void calls.push([value, ops])); + }); + await flushed(); + await ref.bump(context); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([[0, [["r", 0]]]]); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([ + [0, [["r", 0]]], + [3, [["r", 3]]], + ]); + // The derive's owner reports the failure (once for the episode); the routine reports nothing. + expect(errors).toEqual([expect.objectContaining({ kind: "derive_failed", error: new Error("compute") })]); + }); + + it("drops a value queued behind a busy call when a failure follows it; the next call is the healed value", async () => { + const calls: unknown[] = []; + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + let set: ((n: number) => Promise) | undefined; + const { errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("compute"); + return d.n; + }); + set = (n) => + doc.change((d) => { + d.n = n; + }, context); + self.on(failing, async ({ value }) => { + calls.push(value); + if (value === 1) { + entered.resolve(); + await gate.promise; + } + }); + }); + await flushed(); + await set!(1); + await entered.promise; + // Queued behind the busy call, then replaced by the failure. + await set!(2); + await set!(-1); + gate.resolve(); + await settled(); + expect(calls).toEqual([0, 1]); + await set!(3); + await settled(); + expect(calls).toEqual([0, 1, 3]); + expect(reports(errors)).toEqual([]); + }); + + it("starts on a source failed now: no call and no report until its first good value (a reference member)", async () => { + const Failing = local("app.failing", { version: 1, params: {}, members: { t: int(), set: method(int()) } }); + const Viewer = local("app.on-viewer", { version: 1, params: { id: int() }, members: { m: method() } }); + const calls: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Failing, (_, self) => { + const doc = self.state(Doc); + return { + t: self.derive([doc], ([d]) => { + if (d.n === 1) throw new Error("t threw"); + return d.n; + }), + set: async (n, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Viewer, (_, self) => { + self.on(self.get(Failing, {}).t, async ({ value, ops }) => void calls.push([value, ops])); + return { m: async () => null }; + }); + }); + const failing = await host.get(Failing, {}, context); + await failing.set(1, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + await settled(); + expect(calls).toEqual([]); + await failing.set(2, context); + await settled(); + expect(calls).toEqual([[2, [["r", 2]]]]); + expect(errors.map(({ kind }) => kind)).toEqual(["derive_failed"]); + viewer.release(); + await host.close(context); + }); + + it("follows a reference member: the provider's value first, then its changes", async () => { + const calls: [unknown, readonly Op[]][] = []; + let bumper: (() => Promise) | undefined; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + bumper = () => bump(doc, context); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + const store = self.get(Store, {}); + self.on(store.n, async ({ value, ops }) => void calls.push([value, ops])); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + await host.get(Item, { id: 1 }, context); + await flushed(); + await bumper!(); + await flushed(); + expect(calls).toEqual([ + [0, [["r", 0]]], + [1, [["r", 1]]], + ]); + }); + + it("converges when it writes its own source: an unchanged draft commits nothing", async () => { + let calls = 0; + await store((doc, self) => { + self.on(doc, async ({ value }, ctx) => { + calls++; + await doc.change((d) => { + d.n = Math.min(value.n + 1, 3); + }, ctx); + }); + }); + // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again (waited for, not timed: Opus F3). + await vi.waitFor(() => expect(calls).toBe(4), { timeout: 2_000 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(calls).toBe(4); + }); + + it("follows a native source: one call per new value, without ops; ends with it", async () => { + const { source, watches } = writable("a"); + const calls: unknown[] = []; + const { errors } = await store((_, self) => { + self.on(source, async (call) => void calls.push(call)); + }); + await flushed(); + source.set("b"); + source.set("c"); + await settled(); + expect(calls).toEqual([ + { kind: "value", value: "a" }, + { kind: "value", value: "b" }, + { kind: "value", value: "c" }, + ]); + source.end(); + expect(watches.size).toBe(0); + expect(errors).toEqual([]); + }); + + it("on a remote member with no value yet, a first value followed at once by a failure calls nothing", async () => { + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const calls: unknown[] = []; + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + self.on(self.get(Notes, { session: "S1" }).count, async ({ value }) => void calls.push(value)); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const peer = await rawParent(client); + const getting = client.get(Viewer, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const member = EDITOR.state.count!; + const failed = [{ member, error: { kind: "plugin", code: "internal", message: "count threw" } } as const]; + const notes = { member: EDITOR.state.notes!, value: jsonPart([]) }; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members: [notes], failed }] }); + await getting; + await transported(); + // Two state messages in one receive turn: the first value, then a failure again. + const value = (n: number) => [{ member, ops: jsonPart([["r", n]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: value(5), failed: [], retired: [] }] }); + peer.send({ kind: "state", holds: [{ hold, members: [], failed, retired: [] }] }); + await transported(); + await flushed(); + expect(calls).toEqual([]); + peer.send({ kind: "state", holds: [{ hold, members: value(6), failed: [], retired: [] }] }); + await transported(); + await flushed(); + expect(calls).toEqual([6]); + await client.close(context); + }); + + it("on a remote member failed since its binding (no value yet), waits for its first value", async () => { + const Halves = remote("app.halves", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const serve = defineFacet((h) => + h.serve(Halves, (_, _principal, self) => { + const doc = self.state(Doc); + return { + variant: "v", + members: { + half: self.derive([doc], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (n, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }, + }; + }), + ); + await server.load([{ name: "@test/app", facet: serve }], context); + const halves = await server.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + await halves.set(2, context); + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const calls: unknown[] = []; + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + self.on(self.get(Halves, {}).half, async ({ value, ops }) => void calls.push([value, ops])); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const [a, b] = memoryLinkPair(); + // The server's own principal: the client holds the object `halves` refers to. + server.accept(a, { principal: { kind: "server" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Viewer, { id: 1 }, context); + await transported(); + await flushed(); + expect([calls, errors]).toEqual([[], []]); + await halves.set(4, context); + await transported(); + await flushed(); + expect(calls).toEqual([[2, [["r", 2]]]]); + await client.close(context); + await server.close(context); + }); + + it("follows ref.state: a reference over a link going loading and live again", async () => { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const states: string[] = []; + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + const notes = self.get(Notes, { session: "S1" }); + self.on(notes.state, async ({ value: state }: { value: RefState }) => void states.push(state.kind)); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const connect = () => { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + return a; + }; + const link = connect(); + await client.get(Viewer, { id: 1 }, context); + await flushed(); + link.close(); + await transported(); + connect(); + await transported(); + await flushed(); + expect(states).toEqual(["live", "loading", "live"]); + await client.close(context); + await server.close(context); + }); + + it("releases its watch when its object ends; a call that throws as it ends is not reported", async () => { + const { source, watches } = writable(0); + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(source, async (_change, ctx) => { + await untilAborted(ctx); + throw new Error("stopping"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + expect(watches.size).toBe(1); + item.release(); + await flushed(); + expect(watches.size).toBe(0); + expect(errors).toEqual([]); + }); +}); + +describe("onLive", () => { + fakeTimers(); + + it("is called once when its object is live; returning ends it", async () => { + let calls = 0; + await store((_, self) => { + self.onLive(async () => { + calls++; + }); + }); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toBe(1); + }); + + it("restarts after a throw with the backoff, reset after a run of at least a minute", async () => { + const starts: number[] = []; + const { errors } = await store((_, self) => { + self.onLive(async (ctx) => { + starts.push(Date.now()); + if (starts.length === 3) await sleep(60_000, ctx); + throw new Error(`run ${starts.length}`); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000 + 60_000); + await vi.advanceTimersByTimeAsync(1_000); + expect(starts).toEqual([0, 1_000, 3_000, 64_000]); + expect(reports(errors)).toEqual(["onLive #1: run 1", "onLive #1: run 2", "onLive #1: run 3", "onLive #1: run 4"]); + }); + + it("is aborted by its object's end: no report, no restart", async () => { + let calls = 0; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + calls++; + await sleep(1_000_000, ctx); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + item.release(); + await vi.advanceTimersByTimeAsync(10_000); + expect({ calls, errors, timers: vi.getTimerCount() }).toEqual({ calls: 1, errors: [], timers: 0 }); + }); +}); + +describe("routines and the end", () => { + it("start in the turn their object went live, next to a method call admitted in that turn", async () => { + const order: string[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => void order.push(`on ${value.n}`)); + self.onLive(async () => void order.push("onLive")); + return { + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + order.push("bump"); + return bump(doc, ctx); + }, + }; + }), + ); + const item = host.hold(Item, { id: 1 }, context); + const bumped = item.bump(context); + // The parked call runs as the object goes live, then `onLive` in that turn, the first `on` call after its watch; + // the call's commit may land while `on 0` runs, and a call after a busy one waits for the settling turn. + await item.state.watch(context).then((watch) => watch.stop()); + await bumped; + await settled(); + expect(order).toEqual(["bump", "onLive", "on 0", "on 1"]); + }); + + it("never start a routine call once the object ended in its first turn: no call, no report, no rejection", async () => { + const calls: string[] = []; + let held: { release(): void } | undefined; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async () => void calls.push("on")); + self.onLive(async () => { + calls.push("onLive #1"); + held!.release(); + }); + self.onLive(async () => void calls.push("onLive #2")); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + held = host.hold(Item, { id: 1 }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect({ calls, errors }).toEqual({ calls: ["onLive #1"], errors: [] }); + }); + + it("are in the drain before their handler runs: a handler releasing the last reference keeps natives open", async () => { + const gate = Promise.withResolvers(); + const events: string[] = []; + let held: { release(): void } | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.open( + async () => null, + () => void events.push("native closed"), + ); + self.on(doc, async ({ value }) => { + if (value.n === 0) return; + // Ends its own object before its first await. + held!.release(); + await gate.promise; + events.push("handler done"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + held = item; + await flushed(); + // The call may see its object end: its own routine ends it. + await item.bump(context).catch(() => {}); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await flushed(); + expect(events).toEqual(["handler done", "native closed"]); + }); + + it("make no call queued behind a busy one once the object ended", async () => { + const gate = Promise.withResolvers(); + const calls: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => { + calls.push(value); + await gate.promise; + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + await item.bump(context); + await flushed(); + item.release(); + gate.resolve(); + await flushed(); + expect(calls).toEqual([{ n: 0 }]); + }); + + it("release their calls' ctx.get references when each call settles", async () => { + const Other = local("app.other", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host } = await loaded((h) => { + h.serve(Other, (_, self) => ({ n: self.derive([self.state(Doc)], ([d]) => d.n) })); + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + await ctx.get(Other, { id: 1 }); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + await flushed(); + expect(host.references(Other).value).toEqual([]); + }); + + it("act as a remote object's principal", async () => { + const Watcher = remote("app.watcher", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); + const seen: unknown[] = []; + const { client } = await linked( + (h) => + h.serve(Watcher, (_, _principal, self) => { + self.onLive(async (ctx) => void seen.push(ctx.principal)); + return { variant: "v", members: {} }; + }), + { child: { kind: "guest", session: "S1" } }, + ); + await client.get(Watcher, {}, context); + await flushed(); + expect(seen).toEqual([{ kind: "guest", session: "S1" }]); + }); + + it("type their handler by the source's kind: ops and retired for a JSON watchable, the value alone for a native one", () => { + const declare = (self: Self, doc: Watchable<{ n: number }>, state: NativeWatchable) => { + self.on(doc, async (call, ctx) => { + expectTypeOf(call).toEqualTypeOf>(); + expectTypeOf(call.ops).toEqualTypeOf(); + expectTypeOf(call.retired).toEqualTypeOf(); + expectTypeOf(ctx).toEqualTypeOf>(); + }); + self.on(state, async (call, ctx) => { + expectTypeOf(call).toEqualTypeOf>(); + expectTypeOf(ctx).toEqualTypeOf>(); + }); + }; + void declare; + }); + + it("are aborted by host.close, which drains their running calls (bounded by closeTimeout)", async () => { + const ended: string[] = []; + const { host, errors } = await loaded( + (h) => { + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + await new Promise((resolve) => setTimeout(resolve, 10)); + ended.push("object"); + }); + self.onLive(async () => { + // Never stops: the drain gives up at closeTimeout. + await new Promise(() => {}); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }, + { closeTimeout: 50 }, + ); + await flushed(); + await host.close(context); + expect(ended).toEqual(["object"]); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.store" }]); + }); + + it("leave nothing of an ended object: its routines' watches and contexts are collected", async () => { + const { source, watches } = writable(0); + let refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(source, async (_change, ctx) => void refs.push(new WeakRef(ctx))); + self.on(doc, async (_change, ctx) => void refs.push(new WeakRef(ctx))); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + item.release(); + await flushed(); + expect({ watches: watches.size, calls: refs.length }).toEqual({ watches: 0, calls: 2 }); + await expectCollected(...refs); + refs = []; + }); +}); diff --git a/packages/chord/test/next/host/sleep.test.ts b/packages/chord/test/next/host/sleep.test.ts new file mode 100644 index 000000000..93c5fb9ab --- /dev/null +++ b/packages/chord/test/next/host/sleep.test.ts @@ -0,0 +1,74 @@ +// Tests of the waiting helpers (§14): `sleep` resolving, rejecting with the abort reason, chaining delays above the +// timer limit and leaving no timer behind; `untilAborted`. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { sleep, untilAborted } from "../../../src/next/host/index.ts"; +import { fakeTimers } from "../support.ts"; + +describe("sleep", () => { + fakeTimers(); + + it("resolves after its delay", async () => { + let done = false; + void sleep(1_000, BACKGROUND_CONTEXT).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(999); + expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(done).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects with the abort reason and clears its timer", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + const waiting = sleep(60_000, context); + const reason = new Error("stop"); + cancel(reason); + await expect(waiting).rejects.toBe(reason); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects at once when its context is aborted already", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + cancel(new Error("before")); + await expect(sleep(10, context)).rejects.toThrow("before"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("chains timers for delays above 2^31-1 ms; an abort clears the chained timer", async () => { + const limit = 2 ** 31 - 1; + let done = false; + void sleep(limit + 5_000, BACKGROUND_CONTEXT).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(limit); + expect(done).toBe(false); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(5_000); + expect(done).toBe(true); + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + const long = sleep(3 * limit, context); + await vi.advanceTimersByTimeAsync(limit + 1); + cancel(new Error("late")); + await expect(long).rejects.toThrow("late"); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("untilAborted", () => { + it("resolves once its context is aborted, at once when it is already", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + let done = false; + const waiting = untilAborted(context).then(() => { + done = true; + }); + await Promise.resolve(); + expect(done).toBe(false); + cancel(); + await waiting; + expect(done).toBe(true); + await untilAborted(context); + }); +}); diff --git a/packages/chord/test/next/host/staging.test.ts b/packages/chord/test/next/host/staging.test.ts new file mode 100644 index 000000000..b0b01b563 --- /dev/null +++ b/packages/chord/test/next/host/staging.test.ts @@ -0,0 +1,516 @@ +// Tests of staging (slice 7b §2, per plugin since slice 7c): a reload's setups and recorded factories; each kind of +// throw keeps the failing plugin on its old code, reported; recorded declarations decide an object's fate. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type FacetHost, + type Host, + type PluginFacet, + type PluginFailure, + type StoreSelf, +} from "../../../src/next/host/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { flushed, memoryStore, Server } from "../support.ts"; + +const Store = local("app.store", { version: 1, params: {}, members: { label: str(), bump: method(undefined, int()) } }); + +/** Serves `Store` labelled `label`, its factory running `declare` first. */ +function serveStore(label: string, declare: (self: StoreSelf) => void = () => {}) { + return defineFacet((h: FacetHost) => + h.serve(Store, (_, self) => { + declare(self); + return { label: constant(label), bump: async () => 0 }; + }), + ); +} + +/** The error of the one plugin `facets`' reload reports failed (it kept its old code). */ +async function failedReload(host: Host, facets: readonly PluginFacet[]): Promise { + const { failed } = await host.reload(facets, context); + expect(failed).toHaveLength(1); + return failed[0]!.error as Error; +} + +describe("staging: a throw keeps the plugin on its old code, reported", () => { + const throwing: [string, ReturnType, string][] = [ + [ + "a setup", + defineFacet(() => { + throw new Error("boom"); + }), + 'plugin "@test/app": boom', + ], + [ + "a factory", + serveStore("v2", () => { + throw new Error("boom"); + }), + 'plugin "@test/app", app.store: boom', + ], + [ + "a declaration error", + serveStore("v2", (self) => { + self.state(defineState("d", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + self.state(defineState("d", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + }), + 'state "d" is declared twice', + ], + [ + "a document at an older version than the object's", + serveStore("v2", (self) => + self.state(defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })), + ), + "version 1 is older than its documents' (2)", + ], + [ + "a broken accepts table", + defineFacet((h) => + h.serve(Store, () => ({ label: constant("x"), bump: async () => 0 }), { + accepts: { 7: {} } as never, + }), + ), + "accepts[7]: not an older version", + ], + ]; + for (const [what, facet, message] of throwing) { + it(`${what}: the reload reports it, the plugin unchanged`, async () => { + const host = createHost({ onError: () => {} }); + const v2 = defineState("doc", { + schema: obj({ n: int() }), + version: 2, + initial: () => ({ n: 0 }), + migrate: () => ({ n: 0 }), + }); + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => self.state(v2)) }], context); + const store = await host.get(Store, {}, context); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain(message); + expect(store.label.value).toBe("v1"); + expect(store.state.value).toEqual({ kind: "live" }); + // The plugin is still the old one: reloading it with the same facet set again is a change of nothing. + await host.close(context); + }); + } + + it("a remote type on a host without an address is refused, as at load", async () => { + const Remote = remote("app.remote", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([], context); + const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "v", members: { n: constant(1) } }))); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "a host without an address serves only local types", + ); + await host.close(context); + }); +}); + +describe("staging: recorded declarations decide the fate", () => { + it("records without activating: no native opens, no reference binds, no document is made", async () => { + const Dep = local("app.dep", { version: 1, params: {}, members: { n: int() } }); + const opened: string[] = []; + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveStore("v1") }], context); + const store = await host.get(Store, {}, context); + const facet = serveStore("v2", (self) => { + self.open( + async () => void opened.push("opened"), + () => {}, + ); + self.get(Dep, {}); + self.state(defineState("fresh", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + }); + const report = await host.reload([{ name: "@test/app", facet }], context); + // It opens natives now: a restart, whose activation opened its native once. + expect(report.objects).toEqual([{ object: "app.store", kind: "restarted" }]); + await flushed(); + // Neither staging nor the restart opened it: the new attempt parks on the missing `Dep` first. + expect(opened).toEqual([]); + expect(store.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + }); + + it("a table of the new factory at a newer version than the store's restarts the object (its eager migration)", async () => { + const store = await memoryStore(); + const V1 = defineTable("items", { version: 1, columns: { id: str(), name: str() }, key: ["id"] }); + const V2 = evolveTable(V1, (t) => t.rename("name", "title")); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => self.table(V1)) }], context); + await host.get(Store, {}, context); + const same = await host.reload( + [{ name: "@test/app", facet: serveStore("v2", (self) => self.table(V1)) }], + context, + ); + expect(same.objects).toEqual([{ object: "app.store", kind: "swapped" }]); + const newer = await host.reload( + [{ name: "@test/app", facet: serveStore("v3", (self) => self.table(V2)) }], + context, + ); + expect(newer.objects).toEqual([{ object: "app.store", kind: "restarted" }]); + await host.close(context); + await store.close(context); + }); + + it("a job declared at its version with another schema is refused at staging", async () => { + const Job1 = defineJob("app.job", { version: 1, args: obj({}), result: int() }); + const Job1b = defineJob("app.job", { version: 1, args: obj({ x: int() }), result: int() }); + const host = createHost({ onError: () => {} }); + const run = { run: async () => 1 }; + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => void self.job(Job1, run)) }], context); + await host.get(Store, {}, context); + const facet = serveStore("v2", (self) => void self.job(Job1b, run)); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "schema changed without a version bump", + ); + await host.close(context); + }); + + it("a contribution outside a host-lifetime object is a declaration error at staging", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const Page = local("app.page", { version: 1, params: { id: int() }, members: { id: int() } }); + const host = createHost({ onError: () => {} }); + const page = (contribute: boolean) => + defineFacet((h) => + h.serve(Page, ({ id }, self) => { + if (contribute) (self as unknown as StoreSelf).contribute(Names, "x"); + return { id: constant(id) }; + }), + ); + await host.load([{ name: "@test/app", facet: page(false) }], context); + await host.get(Page, { id: 1 }, context); + expect((await failedReload(host, [{ name: "@test/app", facet: page(true) }])).message).toContain( + "self.contribute is valid only in a local type with empty params", + ); + await host.close(context); + }); +}); + +describe("staging: declaration parity with an attempt (fix round 1)", () => { + it("refuses retryOn of an attempt-owned writable, the plugin kept on its old code", async () => { + const T = local("astra.staging", { version: 1, params: {}, members: { n: int() } }); + const oldFacet = defineFacet((h) => h.serve(T, () => ({ n: constant(1) }))); + const newFacet = defineFacet((h) => + h.serve(T, (_, self) => { + self.retryOn(self.writable(0)); + return { n: constant(2) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/staging", facet: oldFacet }], context); + const old = await host.get(T, {}, context); + const { failed } = await host.reload([{ name: "@astra/staging", facet: newFacet }], context); + const state = old.state.value; + old.release(); + await host.close(context); + expect({ failed: failed.map(({ plugin }) => plugin), state: state.kind }).toEqual({ + failed: ["@astra/staging"], + state: "live", + }); + }); +}); + +describe("staging per plugin (slice 7c §4)", () => { + const NotesV1 = local("app.notes", { version: 1, params: {}, members: { count: int() } }); + const NotesV2 = local("app.notes", { version: 2, params: {}, members: { count: int(), title: str() } }); + const View = local("app.view", { version: 1, params: {}, members: { label: str() } }); + const Extra = local("app.extra", { version: 1, params: {}, members: { n: int() } }); + + /** The notes provider at `version`, labelled through its count. */ + const notes = (version: 1 | 2, count: number) => + defineFacet((h) => + version === 1 + ? h.serve(NotesV1, () => ({ count: constant(count) })) + : h.serve(NotesV2, () => ({ count: constant(count), title: constant("t") })), + ); + /** A consumer of the notes at `version`, labelled `label` and the count it reads. */ + const view = (version: 1 | 2, label: string) => + defineFacet((h) => + h.serve(View, (_, self) => { + const count = (version === 1 ? self.get(NotesV1, {}) : self.get(NotesV2, {})).count; + return { label: self.derive([count], ([n]) => `${label} ${n}`) }; + }), + ); + const throwing = defineFacet(() => { + throw new Error("setup threw"); + }); + const failedOf = (report: { readonly failed: readonly PluginFailure[] }) => + report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + + it("a failing plugin keeps its old code; the others of the reload apply", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + const store = await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/store"], removed: [] }); + expect(failedOf(report)).toEqual([["@test/notes", 'plugin "@test/notes": setup threw']]); + expect(report.objects).toEqual([{ object: "app.store", kind: "swapped" }]); + await flushed(); + expect([store.label.value, counted.count.value, counted.state.value]).toEqual(["v2", 1, { kind: "live" }]); + // Fixed later, it applies as any change does. + const fixed = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 2) }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect([fixed.plugins.changed, fixed.failed]).toEqual([["@test/notes", "@test/store"], []]); + await host.close(context); + }); + + it("a factory throwing in staging fails its plugin alone; an added plugin that fails is left out", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/notes", facet: notes(1, 2) }, + { + name: "@test/store", + facet: serveStore("v2", () => { + throw new Error("factory threw"); + }), + }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/notes"], removed: [] }); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/store", 'plugin "@test/store", app.store: factory threw'], + ]); + await flushed(); + expect(counted.count.value).toBe(2); + await host.close(context); + }); + + it("the closure rule: a consumer edited together with a failing provider keeps its old code", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect(report.plugins.changed).toEqual([]); + expect(failedOf(report)).toEqual([ + ["@test/notes", 'plugin "@test/notes": setup threw'], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect(report.objects).toEqual([]); + await flushed(); + expect([shown.label.value, shown.state.value]).toEqual(["old 1", { kind: "live" }]); + // Both fixed together: both apply. + const both = await host.reload( + [ + { name: "@test/notes", facet: notes(2, 5) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([both.plugins.changed, both.failed]).toEqual([["@test/notes", "@test/view"], []]); + await flushed(); + expect(shown.label.value).toBe("new 5"); + await host.close(context); + }); + + it("takes the plugins whose new code failed before staging (`failed`): the closure rule and the report see them", async () => { + const host = createHost({ onError: () => {} }); + const provider = notes(1, 1); + await host.load( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + // Its new build failed: the caller passes its loaded facet, and says so. + const before = [{ plugin: "@test/notes", error: new Error("build failed") }]; + const report = await host.reload( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + { failed: before }, + ); + expect(failedOf(report)).toEqual([ + ["@test/notes", "build failed"], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect([report.plugins.changed, shown.label.value]).toEqual([[], "old 1"]); + await host.close(context); + }); + + it("the closure rule checks a request as a real bind does, the kept provider's host address included (Astra)", async () => { + const Here = hostKind("app.here"); + const Elsewhere = hostKind("app.elsewhere"); + const placed = (only: boolean) => + remote("app.provider", { + version: 1, + params: { id: int() }, + hostAddress: ({ id }) => (only && id !== 1 ? Elsewhere() : Here()), + variants: { main: { state: { n: int() } } }, + }); + const Consumer = local("app.consumer", { version: 1, params: {}, members: { n: int() } }); + const consumer = (id: number) => + defineFacet((h) => + h.serve(Consumer, (_, self) => ({ n: self.derive([self.get(placed(false), { id }).n], ([n]) => n) })), + ); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + await host.load( + [ + // The provider's own code places every id but 1 on another host. + { + name: "@test/provider", + facet: defineFacet((h) => + h.serve(placed(true), ({ id }) => ({ variant: "main", members: { n: constant(id) } })), + ), + }, + { name: "@test/consumer", facet: consumer(1) }, + ], + context, + ); + const shown = await host.get(Consumer, {}, context); + const report = await host.reload( + [ + { name: "@test/provider", facet: throwing }, + { name: "@test/consumer", facet: consumer(2) }, + ], + context, + ); + expect(report.failed.map(({ plugin }) => plugin)).toEqual(["@test/provider", "@test/consumer"]); + expect([shown.state.value, shown.n.value]).toEqual([{ kind: "live" }, 1]); + await host.close(context); + }); + + it("the closure rule covers a name nobody serves while a plugin failed, and runs to a fixpoint", async () => { + const Middle = local("app.middle", { version: 2, params: {}, members: { n: int() } }); + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const host = createHost({ onError: () => {} }); + const middle = (needs: boolean) => + defineFacet((h) => h.serve(Middle, (_, self) => ({ n: needs ? self.get(Extra, {}).n : constant(0) }))); + const top = (needs: boolean) => + defineFacet((h) => h.serve(Top, (_, self) => ({ n: needs ? self.get(Middle, {}).n : constant(0) }))); + await host.load( + [ + { name: "@test/middle", facet: middle(false) }, + { name: "@test/top", facet: top(false) }, + ], + context, + ); + await host.get(Top, {}, context); + await host.get(Middle, {}, context); + // The new extra plugin fails: the middle's new code needs Extra, served by nobody now; the top's new code needs + // the middle, which keeps its old code (another version is not involved: its request binds to the old one). + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/middle", facet: middle(true) }, + { name: "@test/top", facet: top(true) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/middle", 'plugin "@test/middle": needs app.extra v1, which no plugin serves now (a plugin failed)'], + ]); + expect(report.plugins.changed).toEqual(["@test/top"]); + await host.close(context); + }); + + it("without a failing plugin, a request that does not bind is an ordinary wait: the consumer applies and parks", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([report.plugins.changed, report.failed]).toEqual([["@test/notes", "@test/view"], []]); + expect(report.objects).toEqual([ + { object: "app.notes", kind: "swapped" }, + expect.objectContaining({ object: "app.view", kind: "parked" }), + ]); + await flushed(); + expect(shown.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + }); + + it("a type moving between plugins while its old owner fails: the new owner collides and keeps its old code too", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/a", facet: notes(1, 1) }, + { name: "@test/b", facet: defineFacet(() => {}) }, + ], + context, + ); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/a", facet: throwing }, + { name: "@test/b", facet: notes(1, 2) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/a", 'plugin "@test/a": setup threw'], + ["@test/b", 'plugin "@test/b": app.notes is served twice (plugin "@test/a")'], + ]); + expect(counted.count.value).toBe(1); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/unbound.test.ts b/packages/chord/test/next/host/unbound.test.ts new file mode 100644 index 000000000..c5430a447 --- /dev/null +++ b/packages/chord/test/next/host/unbound.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, type Principal } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { Binding } from "../../../src/next/host/reference.ts"; +import { UnboundIndex, type UnboundRequest } from "../../../src/next/host/unbound.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + delivered, + expectCollected, + instancesOf, + RawPeer, + rejection, + SERVER_ADDRESS, + Server, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +const bindings = instancesOf(Binding, "release"); +const holds = instancesOf(Hold, "release"); + +const Info = local("app.info", { version: 1, params: {}, members: { title: str() } }); +const Missing = local("app.missing", { version: 1, params: { id: int() }, members: { n: int() } }); + +/** A request recording its wakes and ends. */ +function request(principal?: Principal): UnboundRequest & { readonly log: string[] } { + const log: string[] = []; + return { principal, log, wake: () => log.push("wake"), end: (error) => log.push(`end ${error.message}`) }; +} + +describe("the index", () => { + it("wakes every request at the publication, out of the index first; a deleted one is not woken", () => { + const index = new UnboundIndex(); + const [a, b, c] = [request(), request(), request()]; + index.add("app.a", a); + index.add("app.a", b); + index.add("app.b", c); + index.delete("app.a", b); + index.wakeAll(); + expect([a.log, b.log, c.log]).toEqual([["wake"], [], ["wake"]]); + // Out of it: a second publication wakes nobody. + index.wakeAll(); + expect(a.log).toEqual(["wake"]); + }); + + it("wakes the requests of the names a reload serves only (carry-over: publications wake per name)", () => { + const index = new UnboundIndex(); + const [a, b, c] = [request(), request(), request()]; + index.add("app.a", a); + index.add("app.b", b); + index.add("app.c", c); + index.wake(["app.a", "app.c", "app.unknown"]); + expect([a.log, b.log, c.log]).toEqual([["wake"], [], ["wake"]]); + // The others stay indexed until their own names are served. + index.wake(["app.b"]); + expect(b.log).toEqual(["wake"]); + // Woken out of the index: a later publication wakes none of them again. + index.wakeAll(); + expect([a.log, b.log, c.log]).toEqual([["wake"], ["wake"], ["wake"]]); + }); + + it("ends the requests of a revoked principal only, and every request at the host's close", () => { + const index = new UnboundIndex(); + const guest = request({ kind: "guest" }); + const owner = request({ kind: "owner" }); + const own = request(); + for (const each of [guest, owner, own]) index.add("app.a", each); + const revoked = new ChordError({ code: "forbidden", reason: "revoked", message: "revoked" }); + index.revoke((principal) => principal.kind === "guest", revoked); + expect([guest.log, owner.log, own.log]).toEqual([["end revoked"], [], []]); + index.close(new ChordError({ code: "released", reason: "closed", message: "closed" })); + expect([owner.log, own.log]).toEqual([["end closed"], ["end closed"]]); + }); +}); + +describe("references waiting in the index", () => { + it("before the publication: loading, placed by it; a type not served then waits failed { waiting }", async () => { + const host = createHost({ onError: () => {} }); + const info = host.hold(Info, {}, context); + const missing = host.hold(Missing, { id: 1 }, context); + expect([info.state.value, missing.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); + await host.load( + [{ name: "@test/a", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + await delivered(); + expect(info.state.value).toEqual({ kind: "live" }); + expect(missing.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "app.missing is not served here" } }, + retryAt: undefined, + }); + // A retry leaves it waiting: an event wakes it, nothing else. + missing.retry(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + expect(missing.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + }); + + it("leave nothing behind once released, before and after the publication (GC)", async () => { + const host = createHost({ onError: () => {} }); + host.hold(Missing, { id: 1 }, context).release(); + await host.load([], context); + host.hold(Missing, { id: 2 }, context).release(); + const got = host.get(Missing, { id: 3 }, context, { readyTimeout: 1 }).catch(() => {}); + await got; + await expectCollected(...bindings); + expect(host).toBeDefined(); + }); + it("end at the start of the host's close, not after its slow native closes (fix round 1, Sol 6)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const closeGate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Core, (_, self) => { + self.open( + async () => 1, + () => closeGate.promise, + ); + return { n: constant(1) }; + }), + ); + await host.load([{ name: "@test/a", facet }], context); + let outcome: unknown; + void rejection(host.get(Missing, { id: 1 }, context)).then((error) => { + outcome = error; + }); + const held = host.hold(Missing, { id: 2 }, context); + const closing = host.close(context); + await delivered(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + expect(held.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + closeGate.resolve(); + await closing; + }); +}); + +describe("holds waiting in the index", () => { + /** A server not loaded yet, a raw consumer peer on a child link, and a hold of `app.missing` it sent. */ + async function early() { + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + peer.send({ + kind: "hold", + hold: 0, + type: "app.missing", + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart({}), + address: SERVER_ADDRESS, + }); + await transported(); + return { host, peer, link: a }; + } + + it("wait for the publication without a word, then are refused failed { waiting } and kept", async () => { + const { host, peer } = await early(); + expect(peer.kinds()).toEqual(["hello"]); + await host.load([], context); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + await host.close(context); + }); + + it("leave nothing behind once their link closed, before and after the publication (GC)", async () => { + const before = await early(); + before.link.close(); + await transported(); + await before.host.load([], context); + const after = await early(); + await after.host.load([], context); + await after.peer.next("failed"); + after.link.close(); + await transported(); + expect([before.host.activity.value.references, after.host.activity.value.references]).toEqual([0, 0]); + await expectCollected(...holds); + expect([before.host, after.host]).toBeDefined(); + }); +}); + +describe("what waits unserved (slice 12 §6, testPlugin)", () => { + it("lists the index's names sorted, and the host the names no plugin serves", async () => { + const index = new UnboundIndex(); + index.add("b", request()); + index.add("a", request()); + expect(index.names()).toEqual(["a", "b"]); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "info", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + const held = host.hold(Missing, { id: 1 }, context); + await delivered(); + expect((host as unknown as ChordHost).unservedNames()).toEqual(["app.missing"]); + held.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/definition.test.ts b/packages/chord/test/next/jobs/definition.test.ts new file mode 100644 index 000000000..aea86122d --- /dev/null +++ b/packages/chord/test/next/jobs/definition.test.ts @@ -0,0 +1,129 @@ +// Tests of job definitions (§15): what `defineJob` checks at declaration (names, versions, closed JSON args, a +// singleton state, concurrency, the shape of `accepts`, `persist` refused before slice 6) and the types it gives. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { bytes, int, map, obj, str, unknown } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { defineJob, type JobDefinition } from "../../../src/next/jobs/index.ts"; + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +const Keyed = defineState("app.keyed", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), + keyed: true, +}); + +const args = obj({ command: str() }); + +describe("defineJob", () => { + it("keeps its name, version, Chord's copies of its schemas, its state and concurrency", () => { + const Shell = defineJob("app.shell", { version: 2, args, result: int(), state: Progress, concurrency: 1 }); + expect(Shell).toMatchObject({ name: "app.shell", version: 2, state: Progress, concurrency: 1, accepts: {} }); + expect(Shell.args.schema).not.toBe(args); + expectTypeOf(Shell).toEqualTypeOf>(); + const Plain = defineJob("app.plain", { version: 1, args, result: str() }); + expectTypeOf(Plain).toEqualTypeOf>(); + }); + + it('takes a restricted name only: lowercase [a-z0-9@/._~-] (decision "Restricted names")', () => { + for (const name of ["@app/jobs.run_1~x-y", "a"]) + expect(defineJob(name, { version: 1, args, result: int() }).name).toBe(name); + for (const name of ["", "App.run", "app:run", "app run", "a\0b", "a\uD800", "é"]) { + expect(() => defineJob(name, { version: 1, args, result: int() })).toThrow( + `A job name ${JSON.stringify(name)}: expected lowercase [a-z0-9@/._~-]`, + ); + } + // Plugin code, not type-checked. + expect(() => defineJob(42 as never, { version: 1, args, result: int() })).toThrow("A job name 42"); + }); + + it("refuses a bad name, version, unknown option or concurrency", () => { + expect(() => defineJob("", { version: 1, args, result: int() })).toThrow("A job name"); + expect(() => defineJob("j", { version: 0, args, result: int() })).toThrow( + "j.version: expected a positive integer", + ); + // Plugin code, not type-checked. + const unknownKey = { version: 1, args, result: int(), retries: 3 } as never; + expect(() => defineJob("j", unknownKey)).toThrow('j: unknown key "retries"'); + for (const concurrency of [0, 1.5, -1]) { + expect(() => defineJob("j", { version: 1, args, result: int(), concurrency })).toThrow( + "j.concurrency: expected a positive integer", + ); + } + }); + + it("requires closed JSON args, as a method's: open objects and bytes are refused; records are fine", () => { + const open = { type: "object", properties: { a: str() } } as never; + expect(() => defineJob("j", { version: 1, args: open, result: int() })).toThrow( + "argument objects must be closed", + ); + expect(() => defineJob("j", { version: 1, args: obj({ data: bytes() }), result: int() })).toThrow( + "j.args: job arguments are JSON", + ); + expect(() => defineJob("j", { version: 1, args: obj({ tags: map(str()) }), result: unknown() })).not.toThrow(); + }); + + it("takes a singleton document definition as its state", () => { + expect(() => defineJob("j", { version: 1, args, result: int(), state: Keyed as never })).toThrow( + "j.state: expected a singleton document definition", + ); + }); + + it("checks accepts at declaration: older versions only, each with an args and a result function", () => { + const adapter = { args: (old: { cmd: string }) => ({ command: old.cmd }), result: (old: number) => old }; + expect(() => + defineJob("j", { version: 3, args, result: int(), accepts: { 1: adapter, 2: adapter } }), + ).not.toThrow(); + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: { 2: adapter } })).toThrow( + "j.accepts[2]: not an older version", + ); + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: { 0: adapter } })).toThrow( + "j.accepts[0]: not an older version", + ); + const noResult = { 1: { args: adapter.args } } as never; + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: noResult })).toThrow( + "j.accepts[1].result: expected a function", + ); + const extra = { 1: { ...adapter, state: () => null } } as never; + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: extra })).toThrow( + 'j.accepts[1]: unknown key "state"', + ); + }); + + it("takes persist with a resume rule, a persisted state and a JSON-only result (slice 6b §4)", () => { + const Persisted = defineState("p.state", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const InMemory = defineState("m.state", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const job = defineJob("j", { version: 1, args, result: int(), state: Persisted, persist: { resume: "rerun" } }); + expect(job.persist).toEqual({ resume: "rerun" }); + expect(() => + defineJob("j", { version: 1, args, result: int(), state: InMemory, persist: { resume: "abandon" } }), + ).toThrow("a persisted job's state is a persisted document"); + expect(() => defineJob("j", { version: 1, args, result: bytes(), persist: { resume: "rerun" } })).toThrow( + "a persisted job's result is JSON", + ); + // An in-memory job may return bytes. + expect(() => defineJob("j", { version: 1, args, result: bytes() })).not.toThrow(); + // @ts-expect-error: resume is required + expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow( + 'expected "rerun" or "abandon"', + ); + }); + + it("fingerprints args, result and state: a schema change without a version bump changes it", () => { + const one = defineJob("j", { version: 1, args, result: int() }); + expect(defineJob("j", { version: 1, args, result: int() }).fingerprint).toBe(one.fingerprint); + expect(defineJob("j", { version: 1, args, result: str() }).fingerprint).not.toBe(one.fingerprint); + }); +}); diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts new file mode 100644 index 000000000..d82d82b3f --- /dev/null +++ b/packages/chord/test/next/jobs/job.test.ts @@ -0,0 +1,1391 @@ +// Tests of jobs in memory (§15): starts (args, dedupe, concurrency, `tx.start`, `outcome`, when `run` starts), runs +// (the ctx, the state document and its fence, throws and aborts), finishes (cleanup, retries, blocked, dropped), +// `running`, keep-alive, the object's end, cost guards and GC. + +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { + type Context, + BACKGROUND_CONTEXT as context, + withAbortSignal, + withCancel, +} from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { arr, int, local, obj, remote, str, unknown } from "../../../src/next/contracts/index.ts"; +import { ChordError, PluginError, type Principal } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError, type Self, untilAborted } from "../../../src/next/host/index.ts"; +import { + defineJob, + type JobCtx, + type JobDefinition, + type JobHandle, + type JobImplementation, + type Outcome, + type Run, +} from "../../../src/next/jobs/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { + delivered, + expectCollected, + fakeTimers, + flushed, + framesOf, + loaded, + metered, + rejection, + Server, + type TestPrincipal, + transported, +} from "../support.ts"; + +const Log = defineState("app.log", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +/** A job without a state document. */ +const Echo = defineJob("app.echo", { version: 1, args: obj({ text: str() }), result: str() }); + +/** A job with a state document and one run at a time. */ +const Shell = defineJob("app.shell", { + version: 1, + args: obj({ command: str() }), + result: obj({ exit: int() }), + state: Progress, + concurrency: 1, +}); + +/** The host-lifetime object of most tests: its log document, and the jobs the test declares. */ +const Store = local("app.store", { version: 1, params: {}, members: { log: obj({ list: arr(str()) }) } }); + +/** The same with params: it lives while referenced or running jobs. */ +const Item = local("app.item", { version: 1, params: { id: int() }, members: { log: obj({ list: arr(str()) }) } }); + +type Declared = { + readonly self: Self; + readonly log: SingletonHandle<{ list: string[] }>; + readonly job: JobHandle; + readonly errors: HostError[]; +}; + +/** Loads a host serving `Store` with the job `definition`; `implement` gets the object's `self` and log. */ +async function declared( + definition: JobDefinition, + implement: (self: Self, log: SingletonHandle<{ list: string[] }>) => JobImplementation, + options: Parameters[1] = {}, +): Promise & { readonly host: Awaited>["host"] }> { + let found: Omit, "errors"> | undefined; + const { host, errors } = await loaded( + (h) => + h.serve(Store, (_, self) => { + const log = self.state(Log); + found = { self, log, job: self.job(definition, implement(self, log)) }; + return { log }; + }), + options, + ); + return { ...found!, errors, host }; +} + +/** Appends `entry` to the log in `tx`. */ +async function append( + tx: Parameters[0]>[0], + log: SingletonHandle<{ list: string[] }>, + entry: string, +) { + (await tx.draft(log)).list.push(entry); +} + +/** A `run` that waits for `gate`, then returns `text`. */ +function gated(gate: Promise) { + return async ({ text }: { text: string }) => { + await gate; + return text; + }; +} + +/** The reported job errors, as `kind id: message`. */ +function jobReports(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => { + if (error.kind === "job_failed" || error.kind === "job_finish_failed") { + return [`${error.kind}: ${(error.error as Error).message}`]; + } + return error.kind === "job_dropped" ? [`${error.kind} ${error.job}`] : []; + }); +} + +describe("start", () => { + it("checks the args against the definition (invalid/args) before anything is recorded", async () => { + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text })); + // Plugin code passing wrong args (not type-checked). + const wrong = { text: 1 } as unknown as { text: string }; + expect(await rejection(job.start(wrong, context))).toMatchObject({ code: "invalid", reason: "args" }); + const extra = { text: "a", more: true } as unknown as { text: string }; + expect(await rejection(job.start(extra, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(job.running.value).toEqual([]); + }); + + it("compiles the args check once per definition (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text }), { telemetry }); + const before = counts()["chord.data.compilations"] ?? 0; + for (const text of ["a", "b", "c"]) await (await job.start({ text }, counting)).outcome(context); + expect((counts()["chord.data.compilations"] ?? 0) - before).toBeLessThanOrEqual(1); + expect(counts()["chord.data.validations"]).toBeGreaterThanOrEqual(3); + }); + + it("returns the unsettled run with the same key, across transactions, whatever its args", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + // Two starts in one turn: the line serializes them, the second sees the first's record. + const [first, again] = await Promise.all([ + job.start({ text: "a" }, context, { key: "k" }), + job.start({ text: "other" }, context, { key: "k" }), + ]); + expect(again).toBe(first); + expect(await job.start({ text: "third" }, context, { key: "k" })).toBe(first); + expect(job.running.value).toHaveLength(1); + gate.resolve(); + expect(await first.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + // Settled: the key names nothing any more (Q9). + const next = await job.start({ text: "b" }, context, { key: "k" }); + expect(next.id).not.toBe(first.id); + }); + + it("commits nothing for a start that only dedupes", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context, { key: "k" }); + const commits = vi.spyOn(Batch.prototype, "commit"); + try { + expect(await job.start({ text: "a" }, context, { key: "k" })).toBe(run); + expect(commits).not.toHaveBeenCalled(); + } finally { + commits.mockRestore(); + gate.resolve(); + } + }); + + it("returns the same run for a key started twice in one transaction", async () => { + const { job, self } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const [one, two] = await self.tx(async (tx) => { + return [tx.start(job, { text: "a" }, { key: "k" }), tx.start(job, { text: "b" }, { key: "k" })]; + }, context); + expect(two).toBe(one); + expect((await one!.outcome(context)).args).toEqual({ text: "a" }); + }); + + describe("at the concurrency limit", () => { + fakeTimers(); + + it("rejects unavailable/busy: running and finishing runs count", async () => { + const runGate = Promise.withResolvers(); + let finishes = 0; + const { job } = await declared(Shell, () => ({ + run: async () => { + await runGate.promise; + return { exit: 0 }; + }, + finish: async () => { + // The first finish throws: the run stays finishing during the backoff, the line free. + if (finishes++ === 0) throw new Error("once"); + }, + })); + const run = await job.start({ command: "a" }, context); + const busy = { code: "unavailable", reason: "busy" }; + expect(await rejection(job.start({ command: "b" }, context))).toMatchObject(busy); + runGate.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["finishing"]); + expect(await rejection(job.start({ command: "b" }, context))).toMatchObject(busy); + await vi.advanceTimersByTimeAsync(1_000); + await run.outcome(context); + await (await job.start({ command: "c" }, context)).outcome(context); + }); + }); + + it("counts runs started earlier in the same transaction", async () => { + const { job, self } = await declared(Shell, () => ({ run: async () => ({ exit: 0 }) })); + const error = await rejection( + self.tx(async (tx) => { + tx.start(job, { command: "a" }); + tx.start(job, { command: "b" }); + }, context), + ); + expect(error).toMatchObject({ code: "unavailable", reason: "busy" }); + expect(job.running.value).toEqual([]); + }); + + it("lets finish chain the same job at concurrency 1: its own run is not counted", async () => { + const done: string[] = []; + const { job } = await declared(Shell, (_, log) => ({ + run: async ({ command }) => ({ exit: command.length }), + finish: async (outcome, tx) => { + done.push(outcome.args.command); + await append(tx, log, outcome.args.command); + if (outcome.args.command.length < 3) tx.start(job, { command: `${outcome.args.command}x` }); + }, + })); + await (await job.start({ command: "a" }, context)).outcome(context); + await flushed(); + await flushed(); + expect(done).toEqual(["a", "ax", "axx"]); + }); + + it("lets finish start the same job again with the same key: a new run, not the finishing one", async () => { + const gate = Promise.withResolvers(); + let successor: Run | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => { + if (text === "again") await gate.promise; + return text; + }, + finish: async (outcome, tx) => { + if (outcome.args.text === "first") successor = tx.start(job, { text: "again" }, { key: "k" }); + }, + })); + const first = await job.start({ text: "first" }, context, { key: "k" }); + expect(await first.outcome(context)).toMatchObject({ kind: "done", result: "first" }); + expect(successor!.id).not.toBe(first.id); + expect(job.running.value).toMatchObject([{ id: successor!.id, key: "k", phase: "running" }]); + // The key names the successor now. + expect(await job.start({ text: "other" }, context, { key: "k" })).toBe(successor); + gate.resolve(); + expect(await successor!.outcome(context)).toMatchObject({ kind: "done", result: "again" }); + }); + + it("tx.start commits with the transaction's other writes, both or neither", async () => { + const { job, self, log } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const run = await self.tx(async (tx) => { + await append(tx, log, "requested"); + return tx.start(job, { text: "a" }); + }, context); + expect(log.value.list).toEqual(["requested"]); + expect((await run.outcome(context)).kind).toBe("done"); + let discarded: Run | undefined; + const error = await rejection( + self.tx(async (tx) => { + await append(tx, log, "lost"); + discarded = tx.start(job, { text: "b" }); + throw new Error("changed my mind"); + }, context), + ); + expect(error).toMatchObject({ message: "changed my mind" }); + expect(log.value.list).toEqual(["requested"]); + expect(job.running.value).toEqual([]); + expect(await rejection(discarded!.outcome(context))).toMatchObject({ code: "released" }); + expect(await rejection(job.outcome(discarded!.id, context))).toMatchObject({ code: "gone" }); + }); + + it("gives a short run's handle its outcome however late the caller waits", async () => { + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const run = await job.start({ text: "quick" }, context); + await flushed(); + expect(job.running.value).toEqual([]); + expect(await run.outcome(context)).toEqual({ + id: run.id, + args: { text: "quick" }, + kind: "done", + result: "quick", + }); + }); + + it("waits by id for unsettled runs only: settled and unknown ids reject gone", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context); + const waiting = job.outcome(run.id, context); + gate.resolve(); + expect((await waiting).kind).toBe("done"); + expect(await rejection(job.outcome(run.id, context))).toMatchObject({ code: "gone" }); + expect(await rejection(job.outcome("nope", context))).toMatchObject({ code: "gone" }); + }); + + it("starts run after the commit, outside the batch, once the transaction settled", async () => { + const seen: string[] = []; + const { job, self, log } = await declared(Echo, (s, l) => ({ + run: async ({ text }) => { + seen.push(`run sees ${l.value.list.join(",")}`); + // Outside the batch and the line: a transaction of its own works. + await s.tx(async (tx) => append(tx, l, `from run ${text}`), context); + return text; + }, + })); + const run = await self.tx(async (tx) => { + const started = tx.start(job, { text: "a" }); + await append(tx, log, "written with the start"); + seen.push("transaction done"); + return started; + }, context); + await run.outcome(context); + expect(seen).toEqual(["transaction done", "run sees written with the start"]); + expect(log.value.list).toEqual(["written with the start", "from run a"]); + }); + + it("an abort between the commit and the start is marked, then the signal fires; the abort joins run (D7)", async () => { + let signalled: boolean | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + signalled = ctx.abortSignal?.aborted; + return text; + }, + })); + const watch = await job.running.watch(context); + const aborts: Promise[] = []; + // The watch's frame of the start's commit arrives before `run` starts. + watch.start(async (call) => { + if (call.kind !== "value" || aborts.length > 0) return; + for (const entry of call.value) aborts.push(job.abort(entry.id, context)); + }); + const run = await job.start({ text: "a" }, context); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(signalled).toBe(true); + expect(await Promise.all(aborts)).toEqual(["marked"]); + expect(await job.abort(run.id, context)).toBe("terminal"); + await watch.stop(); + }); + + it("creates nothing for an aborted caller; a caller aborted after the commit does not cancel the run", async () => { + const gate = Promise.withResolvers(); + let signalled: boolean | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await gate.promise; + signalled = ctx.abortSignal?.aborted; + return text; + }, + })); + const early = withCancel(context); + early.cancel(new Error("gone already")); + expect(await rejection(job.start({ text: "a" }, early.context))).toMatchObject({ message: "gone already" }); + expect(job.running.value).toEqual([]); + const caller = withCancel(context); + const run = await job.start({ text: "b" }, caller.context); + caller.cancel(new Error("caller left")); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "b" }); + expect(signalled).toBe(false); + }); + + it("rolls back the transaction's other writes when the state's initial() fails, naming the definition", async () => { + const Bad = defineState("app.bad-state", { + schema: obj({ output: str() }), + version: 1, + // Plugin code: a wrong initial value. + initial: () => ({ output: 1 }) as unknown as { output: string }, + }); + const BadJob = defineJob("app.bad", { version: 1, args: obj({ text: str() }), result: str(), state: Bad }); + const { job, self, log } = await declared(BadJob, () => ({ run: async ({ text }) => text })); + const error = await rejection( + self.tx(async (tx) => { + await append(tx, log, "lost"); + tx.start(job, { text: "a" }); + }, context), + ); + expect(error).toMatchObject({ message: "app.bad-state: initial() does not match the schema" }); + expect({ log: log.value.list, running: job.running.value }).toEqual({ log: [], running: [] }); + }); + + it("is refused before its object is live and once it ended", async () => { + let early: Promise | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const log = self.state(Log); + const job = self.job(Echo, { run: async ({ text }) => text }); + self.open( + async () => { + early = job.start({ text: "a" }, context); + return null; + }, + () => {}, + ); + return { log }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + expect(await rejection(early!)).toMatchObject({ code: "not_ready" }); + item.release(); + }); +}); + +describe("self.job", () => { + it("is declared once per job name, only while the factory runs", async () => { + let later: Self | undefined; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + later = self; + self.job(Echo, { run: async ({ text }) => text }); + self.job(Echo, { run: async ({ text }) => text }); + return { log: self.state(Log) }; + }), + ); + expect(await rejection(host.get(Item, { id: 1 }, context))).toMatchObject({ + message: expect.stringContaining('job "app.echo" is declared twice'), + }); + expect(errors.map((error) => error.kind)).toEqual(["activation_failed"]); + expect(() => later!.job(Shell, { run: async () => ({ exit: 0 }) })).toThrow( + "app.item(1): self.job is valid only in the factory", + ); + }); +}); + +describe("run", () => { + it("gets a ctx with the run's id, the host's principal for a local object, and no state without one", async () => { + let seen: Record | undefined; + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const facet = defineFacet((h) => + h.serve(Store, (_, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + seen = { jobId: ctx.jobId, principal: ctx.principal, hasState: "state" in ctx }; + expectTypeOf(ctx).toEqualTypeOf>(); + return text; + }, + }); + return { log: self.state(Log) }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const run = await job!.start({ text: "a" }, context); + await run.outcome(context); + expect(seen).toEqual({ jobId: run.id, principal: { kind: "server" }, hasState: false }); + await host.close(context); + }); + + it("reads and changes its state document; `running` carries it", async () => { + const gate = Promise.withResolvers(); + const values: string[] = []; + const { job } = await declared(Shell, () => ({ + run: async (_, ctx) => { + values.push(ctx.state.value.output); + await ctx.state.change((d) => { + d.output = "half"; + }, ctx); + values.push(ctx.state.value.output); + await gate.promise; + return { exit: 0 }; + }, + })); + const run = await job.start({ command: "a" }, context); + await flushed(); + expect(job.running.value).toEqual([ + { id: run.id, version: 1, args: { command: "a" }, state: { output: "half" }, phase: "running" }, + ]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", state: { output: "half" } }); + expect(values).toEqual(["", "half"]); + }); + + it("keeps its state documents apart from the object's own documents of the same definition", async () => { + const gate = Promise.withResolvers(); + const Many = defineJob("app.many", { version: 1, args: obj({ n: int() }), result: int(), state: Progress }); + let mine: SingletonHandle<{ output: string }> | undefined; + let job: JobHandle<{ n: number }, number, { output: string }> | undefined; + await loaded((h) => + h.serve(Store, (_, self) => { + mine = self.state(Progress); + job = self.job(Many, { + run: async ({ n }, ctx) => { + await ctx.state.change((d) => { + d.output = `run ${n}`; + }, ctx); + await gate.promise; + return n; + }, + }); + return { log: self.state(Log) }; + }), + ); + const runs = await Promise.all([job!.start({ n: 1 }, context), job!.start({ n: 2 }, context)]); + await flushed(); + expect(job!.running.value.map((entry) => entry.phase !== "blocked" && entry.state.output)).toEqual([ + "run 1", + "run 2", + ]); + expect(mine!.value).toEqual({ output: "" }); + gate.resolve(); + expect((await Promise.all(runs.map((run) => run.outcome(context)))).map((o) => o.state.output)).toEqual([ + "run 1", + "run 2", + ]); + }); + + it("refuses state changes once run settled (released), and never recreates the retired document", async () => { + let captured: JobCtx | undefined; + let inFinish: unknown; + const { job } = await declared(Shell, () => ({ + run: async (_, ctx) => { + captured = ctx; + return { exit: 0 }; + }, + // After run returned, before finish committed: refused; the outcome's state is the state at run's settle. + finish: async (outcome) => { + inFinish = [outcome.state, await rejection(captured!.state.change(() => {}, context))]; + }, + })); + await (await job.start({ command: "a" }, context)).outcome(context); + expect(inFinish).toEqual([{ output: "" }, expect.objectContaining({ code: "released" })]); + const late = captured!.state.change((d) => { + d.output = "late"; + }, context); + expect(await rejection(late)).toMatchObject({ code: "released" }); + expect(() => captured!.state.value).toThrow(ChordError); + expect(job.running.value).toEqual([]); + }); + + it("commits a change still waiting for the line when run settles, before the outcome is recorded", async () => { + const go = Promise.withResolvers(); + const line = Promise.withResolvers(); + const entered = Promise.withResolvers(); + let returned = false; + const { job, self } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await go.promise; + // Admitted while run runs; it waits behind the transaction holding the line. + void ctx.state.change((d) => { + d.output = "last words"; + }, ctx); + returned = true; + return { exit: 0 }; + }, + })); + const run = await job.start({ command: "a" }, context); + const holding = self.tx(async () => { + entered.resolve(); + await line.promise; + }, context); + await entered.promise; + go.resolve(); + await flushed(); + // run settled with its change still waiting: nothing committed, the outcome not recorded yet. + expect(returned).toBe(true); + expect(job.running.value).toMatchObject([{ phase: "running", state: { output: "" } }]); + line.resolve(); + await holding; + expect(await run.outcome(context)).toMatchObject({ kind: "done", state: { output: "last words" } }); + }); + + it("commits changes made after an abort while run stops; a throw then ends aborted, not threw", async () => { + const { job, errors } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await untilAborted(ctx); + await ctx.state.change((d) => { + d.output = "stopped cleanly"; + }, ctx); + throw new Error("interrupted"); + }, + })); + const run = await job.start({ command: "a" }, context); + await flushed(); + void job.abort(run.id, context); + expect(await run.outcome(context)).toEqual({ + id: run.id, + args: { command: "a" }, + state: { output: "stopped cleanly" }, + kind: "failed", + error: { kind: "aborted", reason: "abort" }, + }); + expect(jobReports(errors)).toEqual([]); + }); + + it("keeps a result returned after an abort", async () => { + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await untilAborted(ctx); + return text; + }, + })); + const run = await job.start({ text: "kept" }, context); + await flushed(); + void job.abort(run.id, context); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "kept" }); + }); + + it("ends threw when run throws, reported as job_failed; a plain error becomes PluginError internal", async () => { + const plugin = new PluginError("quota", "too many"); + const { job, errors } = await declared(Echo, () => ({ + run: async ({ text }) => { + if (text === "plugin") throw plugin; + throw new Error("plain"); + }, + })); + const first = await (await job.start({ text: "plugin" }, context)).outcome(context); + expect(first).toMatchObject({ kind: "failed", error: { kind: "threw", error: plugin } }); + const second = await (await job.start({ text: "plain" }, context)).outcome(context); + expect(second.kind === "failed" && second.error).toMatchObject({ kind: "threw" }); + const thrown = second.kind === "failed" && second.error.kind === "threw" ? second.error.error : undefined; + expect(thrown).toBeInstanceOf(PluginError); + expect(thrown).toMatchObject({ code: "internal", message: "plain" }); + expect(jobReports(errors)).toEqual(["job_failed: too many", "job_failed: plain"]); + expect(errors[0]).toMatchObject({ kind: "job_failed", object: "app.store", job: "app.echo", id: first.id }); + }); + + it("releases its ctx.get references when it settles", async () => { + const Other = local("app.other", { + version: 1, + params: { id: int() }, + members: { log: obj({ list: arr(str()) }) }, + }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const { host } = await loaded((h) => { + h.serve(Other, (_, self) => ({ log: self.state(Log) })); + h.serve(Store, (_, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + await ctx.get(Other, { id: 1 }); + expect(host.references(Other).value).toHaveLength(1); + return text; + }, + }); + return { log: self.state(Log) }; + }); + }); + await (await job!.start({ text: "a" }, context)).outcome(context); + expect(host.references(Other).value).toHaveLength(0); + }); + + it("that ignores its signal at the object's end is drained up to closeTimeout; its later changes are refused", async () => { + const gate = Promise.withResolvers(); + const late = Promise.withResolvers(); + const { job, host, errors } = await declared( + Shell, + () => ({ + run: async (_, ctx) => { + await gate.promise; + late.resolve( + await rejection( + ctx.state.change((d) => { + d.output = "too late"; + }, ctx), + ), + ); + return { exit: 0 }; + }, + }), + { closeTimeout: 20 }, + ); + await job.start({ command: "a" }, context); + await flushed(); + await host.close(context); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.store" }]); + gate.resolve(); + expect(await late.promise).toMatchObject({ code: "released" }); + }); + + it("may start the same job again from run (recursion): a new run", async () => { + const ids: string[] = []; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + ids.push(ctx.jobId); + if (text.length < 3) await job.start({ text: `${text}+` }, ctx); + return text; + }, + })); + await (await job.start({ text: "a" }, context)).outcome(context); + await flushed(); + await flushed(); + expect(new Set(ids).size).toBe(3); + }); +}); + +describe("finish", () => { + it("gets each outcome kind; the record, key and state document go in its commit, atomically with its writes", async () => { + const outcomes: string[] = []; + const { job, log, self } = await declared(Shell, (_, l) => ({ + run: async ({ command }, ctx) => { + await ctx.state.change((d) => { + d.output = `out ${command}`; + }, ctx); + if (command === "fail") throw new Error("no"); + return { exit: 0 }; + }, + finish: async (outcome, tx) => { + outcomes.push(`${outcome.kind} ${outcome.state.output}`); + await append(tx, l, outcome.args.command); + }, + })); + // The log entry and the removed entry arrive in one commit: whenever the log's watch sees an entry, the run is gone. + const seen: [number, number][] = []; + (await log.watch(context)).start(async (call) => { + if (call.kind === "value") seen.push([call.value.list.length, job.running.value.length]); + }); + void self; + await (await job.start({ command: "ok" }, context, { key: "k" })).outcome(context); + await (await job.start({ command: "fail" }, context, { key: "k" })).outcome(context); + await delivered(); + expect(outcomes).toEqual(["done out ok", "failed out fail"]); + expect(seen).toEqual([ + [1, 0], + [2, 0], + ]); + expect(job.running.value).toEqual([]); + }); + + it("only cleans up without a finish", async () => { + const { job } = await declared(Shell, () => ({ run: async () => ({ exit: 3 }) })); + const run = await job.start({ command: "a" }, context, { key: "k" }); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { exit: 3 } }); + expect(job.running.value).toEqual([]); + }); + + describe("throwing", () => { + fakeTimers(); + + it("is retried after 1 s doubling, each failure reported; a later success finishes the run", async () => { + const at: number[] = []; + const { job, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + at.push(Date.now()); + if (at.length < 4) throw new Error(`finish ${at.length}`); + }, + })); + const run = await job.start({ text: "a" }, context); + await vi.advanceTimersByTimeAsync(10_000); + expect(at).toEqual([0, 1_000, 3_000, 7_000]); + expect(jobReports(errors)).toEqual([ + "job_finish_failed: finish 1", + "job_finish_failed: finish 2", + "job_finish_failed: finish 3", + ]); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("blocks the run after five failures in a row: outcome, state and key kept, not counted; abort drops it", async () => { + const { job, errors } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await ctx.state.change((d) => { + d.output = "kept"; + }, ctx); + return { exit: 0 }; + }, + finish: async () => { + throw new Error("always"); + }, + })); + const run = await job.start({ command: "a" }, context, { key: "k" }); + const { context: waiting, cancel } = withCancel(context); + const waiter = rejection(run.outcome(waiting)); + const dropped = rejection(run.outcome(context)); + await vi.advanceTimersByTimeAsync(60_000); + expect(jobReports(errors)).toHaveLength(5); + expect(job.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "finish_failed" }]); + // Its waiters keep waiting; their own context ends a wait. + cancel(new Error("gave up")); + expect(await waiter).toMatchObject({ message: "gave up" }); + // Dedupe returns it; a different key starts a new run (blocked runs do not count for concurrency). + expect(await job.start({ command: "b" }, context, { key: "k" })).toBe(run); + const other = await job.start({ command: "c" }, context, { key: "other" }); + await vi.advanceTimersByTimeAsync(0); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked", "finishing"]); + void job.abort(run.id, context); + await vi.advanceTimersByTimeAsync(0); + expect(await dropped).toMatchObject({ code: "aborted" }); + expect(job.running.value.map((entry) => entry.id)).toEqual([other.id]); + expect(errors.filter((error) => error.kind === "job_dropped")).toEqual([ + { kind: "job_dropped", object: "app.store", job: "app.shell", id: run.id }, + ]); + }); + }); + + describe("backing off", () => { + fakeTimers(); + + it("leaves no timer behind when its object ends during the backoff", async () => { + const { job, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + throw new Error("stuck"); + }, + })); + const run = rejection((await job.start({ text: "a" }, context)).outcome(context)); + await vi.advanceTimersByTimeAsync(0); + expect(errors).toHaveLength(1); + expect(vi.getTimerCount()).toBe(1); + await host.close(context); + expect(vi.getTimerCount()).toBe(0); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toHaveLength(1); + expect(await run).toMatchObject({ code: "released" }); + }); + }); + + it("ignores an abort while it runs", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => gate.promise, + })); + const run = await job.start({ text: "a" }, context); + await flushed(); + void job.abort(run.id, context); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("does not run when its object ends while the outcome and finish wait for the line: no call, no report", async () => { + const go = Promise.withResolvers(); + const line = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const finished: string[] = []; + const { job, self, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => { + await go.promise; + return text; + }, + finish: async ({ args }) => void finished.push(args.text), + })); + const run = await job.start({ text: "a" }, context); + const waiting = rejection(run.outcome(context)); + const holding = self.tx(async () => { + entered.resolve(); + await line.promise; + }, context); + await entered.promise; + // run settles while the line is held: its outcome and finish queue behind it. + go.resolve(); + await flushed(); + const closing = host.close(context); + line.resolve(); + await holding.catch(() => {}); + await closing; + expect({ finished, errors }).toEqual({ finished: [], errors: [] }); + expect(await waiting).toMatchObject({ code: "released" }); + }); + + it("is drained at the object's end when it runs: its commit is refused, nothing is reported", async () => { + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const events: string[] = []; + const { job, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + entered.resolve(); + await gate.promise; + events.push("finish returned"); + }, + })); + await job.start({ text: "a" }, context); + await entered.promise; + const closing = host.close(context).then(() => void events.push("closed")); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await closing; + expect(events).toEqual(["finish returned", "closed"]); + expect(errors).toEqual([]); + }); +}); + +describe("running", () => { + it("has one entry per unsettled run, in start order, tagged by phase; a key when given", async () => { + const gates = [Promise.withResolvers(), Promise.withResolvers()]; + const finishing = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => { + await gates[Number(text)]!.promise; + return text; + }, + finish: async () => finishing.promise, + })); + const zero = await job.start({ text: "0" }, context, { key: "z" }); + const one = await job.start({ text: "1" }, context); + gates[0]!.resolve(); + await flushed(); + expect(job.running.value).toEqual([ + { id: zero.id, version: 1, key: "z", args: { text: "0" }, phase: "finishing" }, + { id: one.id, version: 1, args: { text: "1" }, phase: "running" }, + ]); + finishing.resolve(); + gates[1]!.resolve(); + await one.outcome(context); + expect(job.running.value).toEqual([]); + }); + + it("replaces only the entry whose state changed, with ops under its entry (cost guard: no walk of other runs)", async () => { + const gate = Promise.withResolvers(); + const contexts = new Map>(); + const Many = defineJob("app.many", { version: 1, args: obj({ n: int() }), result: int(), state: Progress }); + const { job } = await declared(Many, () => ({ + run: async ({ n }, ctx) => { + contexts.set(String(n), ctx); + await gate.promise; + return n; + }, + })); + for (let n = 0; n < 100; n++) await job.start({ n }, context); + await flushed(); + const before = job.running.value; + const frames = framesOf(await job.running.watch(context)); + await contexts.get("42")!.state.change((d) => { + d.output = "x"; + }, context); + await delivered(); + const after = job.running.value; + expect(after).not.toBe(before); + expect(after.filter((entry, index) => entry !== before[index])).toHaveLength(1); + expect(after[42]).toMatchObject({ state: { output: "x" } }); + expect(frames.map(([, ops]) => ops)).toEqual([[["a", [42, "state", "output"], "x"]] satisfies Op[]]); + gate.resolve(); + }); + + it("does no validation, walk or compilation for a state change (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const gate = Promise.withResolvers(); + const contexts: JobCtx[] = []; + const { job } = await declared( + Shell, + () => ({ + run: async (_, ctx) => { + contexts.push(ctx); + await gate.promise; + return { exit: 0 }; + }, + }), + { telemetry, context: counting }, + ); + await job.start({ command: "a" }, counting); + await flushed(); + const frames = framesOf(await job.running.watch(context)); + const data = () => Object.entries(counts()).filter(([name]) => name.startsWith("chord.data.")); + const before = data(); + await contexts[0]!.state.change((d) => { + d.output = "x".repeat(10_000); + }, context); + await delivered(); + expect(data()).toEqual(before); + expect(frames).toHaveLength(1); + gate.resolve(); + }); + + it("frames record changes as splices and sets of the list", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const frames = framesOf(await job.running.watch(context)); + const run = await job.start({ text: "a" }, context); + await delivered(); + gate.resolve(); + await run.outcome(context); + await delivered(); + const entry = { id: run.id, version: 1, args: { text: "a" }, phase: "running" }; + expect(frames.map(([, ops]) => ops)).toEqual([ + [["p", [], 0, 0, [entry]]], + [["s", [0], { ...entry, phase: "finishing" }]], + [["p", [], 0, 1, []]], + ]); + }); +}); + +describe("keep-alive", () => { + it("keeps an object without holders while runs run, and ends it after the last one and its linger", async () => { + const gate = Promise.withResolvers(); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + let ended = false; + const { host } = await loaded((h) => + h.serve( + Item, + (_, self) => { + job = self.job(Echo, { run: gated(gate.promise) }); + self.onLive(async (ctx) => { + await untilAborted(ctx); + ended = true; + }); + return { log: self.state(Log) }; + }, + { linger: 20 }, + ), + ); + const item = await host.get(Item, { id: 1 }, context); + await job!.start({ text: "a" }, context); + expect(host.activity.value.jobs).toBe(1); + expect(host.references(Item).value).toHaveLength(1); + item.release(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(ended).toBe(false); + expect(host.references(Item).value).toHaveLength(0); + expect(host.activity.value).toMatchObject({ jobs: 1, lingering: 0 }); + gate.resolve(); + await flushed(); + expect(host.activity.value).toMatchObject({ jobs: 0, lingering: 1 }); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(ended).toBe(true); + expect(host.activity.value).toMatchObject({ jobs: 0, lingering: 0 }); + }); + + it("keeps an object whose last run's finish chains another job: the chained run runs, then the object ends", async () => { + const A = defineJob("app.a", { version: 1, args: obj({ text: str() }), result: str() }); + const events: string[] = []; + let a: JobHandle<{ text: string }, string, undefined> | undefined; + let chained: Run | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + self.open( + async () => null, + () => void events.push("object ended"), + ); + const b = self.job(Echo, { + run: async ({ text }) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + events.push(`b ran ${text}`); + return text; + }, + }); + a = self.job(A, { + run: async ({ text }) => text, + finish: async (_outcome, tx) => { + chained = tx.start(b, { text: "chained" }); + }, + }); + return { log: self.state(Log) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await a!.start({ text: "x" }, context); + item.release(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(await chained!.outcome(context)).toMatchObject({ kind: "done", result: "chained" }); + expect(events).toEqual(["b ran chained", "object ended"]); + expect(host.activity.value.jobs).toBe(0); + }); + + it("keeps an object whose last reference goes while its start's commit publishes (a link closing as it sends)", async () => { + const View = remote("app.view", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { running: arr(unknown()) } } }, + }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + let ran = false; + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(View, (_, _principal, self) => { + job = self.job(Echo, { + run: async ({ text }) => { + ran = true; + return text; + }, + }); + // The running list as a state member, so the start's commit sends a state message. + return { variant: "v", members: { running: job.running as never } }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + let closeOnSend = false; + // The server's end of the link closes while it sends: the client's hold, the last reference, goes at once. + const failing: Link = { + send: (message) => { + if (!closeOnSend) return a.send(message); + closeOnSend = false; + a.close(); + }, + onMessage: (listener) => a.onMessage(listener), + close: () => a.close(), + onClose: (listener) => a.onClose(listener), + }; + server.accept(failing, { principal: { kind: "owner" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(View, {}, context); + await transported(); + closeOnSend = true; + const run = await job!.start({ text: "a" }, context); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(ran).toBe(true); + await client.close(context); + await server.close(context); + }); + + it("drops blocked runs, reported, when the object ends after its last run", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"], now: 0 }); + try { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const errors: HostError[] = []; + const host = createHost({ onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(Item, (_, self) => { + job = self.job(Echo, { + run: async ({ text }) => text, + finish: async () => { + throw new Error("stuck"); + }, + }); + return { log: self.state(Log) }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const item = await host.get(Item, { id: 1 }, context); + const run = await job!.start({ text: "a" }, context); + item.release(); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors.map((error) => error.kind)).toEqual([...Array(5).fill("job_finish_failed"), "job_dropped"]); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + expect(host.activity.value.jobs).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it("keeps a lingering object when a run starts during its linger", async () => { + const gate = Promise.withResolvers(); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const { host } = await loaded((h) => + h.serve( + Item, + (_, self) => { + job = self.job(Echo, { run: gated(gate.promise) }); + return { log: self.state(Log) }; + }, + { linger: 20 }, + ), + ); + (await host.get(Item, { id: 1 }, context)).release(); + expect(host.activity.value.lingering).toBe(1); + const run = await job!.start({ text: "a" }, context); + expect(host.activity.value).toMatchObject({ jobs: 1, lingering: 0 }); + await new Promise((resolve) => setTimeout(resolve, 40)); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); +}); + +describe("object end", () => { + type Revocation = { session: string }; + const Jobs = remote("app.jobs", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); + + it("revocation (R11): runs' signals fire with the end's error, the drain waits for them, no finish", async () => { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const finished: string[] = []; + let reason: unknown; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), + onError: () => {}, + }); + const facet = defineFacet((h) => + h.serve(Jobs, (_, _principal, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + await untilAborted(ctx); + reason = ctx.abortSignal?.reason; + await new Promise((resolve) => setTimeout(resolve, 10)); + finished.push("run stopped"); + return text; + }, + finish: async () => void finished.push("finish"), + }); + return { variant: "v", members: {} }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Jobs, {}, context); + const run = await job!.start({ text: "a" }, context); + await flushed(); + server.revoke({ session: "S1" }, context); + await flushed(); + expect(reason).toMatchObject({ code: "forbidden", reason: "revoked" }); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "forbidden", reason: "revoked" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + await transported(); + expect(finished).toEqual(["run stopped"]); + expect(server.activity.value.jobs).toBe(0); + await client.close(context); + await server.close(context); + }); + + it("drains a run that ends its own object before its first await: its natives stay open until it stops", async () => { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const gate = Promise.withResolvers(); + const events: string[] = []; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), + onError: () => {}, + }); + const facet = defineFacet((h) => + h.serve(Jobs, (_, _principal, self) => { + self.open( + async () => null, + () => void events.push("native closed"), + ); + job = self.job(Echo, { + run: async ({ text }) => { + server.revoke({ session: "S1" }, context); + await gate.promise; + events.push("run stopped"); + return text; + }, + }); + return { variant: "v", members: {} }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Jobs, {}, context); + const run = rejection((await job!.start({ text: "a" }, context)).outcome(context)); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await flushed(); + expect(events).toEqual(["run stopped", "native closed"]); + expect(await run).toMatchObject({ code: "forbidden", reason: "revoked" }); + await client.close(context); + await server.close(context); + }); + + it("host.close: signals fire, close waits for the run (drain), no finish, waiters rejected released", async () => { + const finished: string[] = []; + const { job, host } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await untilAborted(ctx); + await new Promise((resolve) => setTimeout(resolve, 10)); + finished.push("run stopped"); + return text; + }, + finish: async () => void finished.push("finish"), + })); + const run = await job.start({ text: "a" }, context); + await flushed(); + const waiting = rejection(run.outcome(context)); + await host.close(context); + expect(finished).toEqual(["run stopped"]); + expect(await waiting).toMatchObject({ code: "released" }); + }); +}); + +describe("GC", () => { + it("finished and dropped runs leave nothing on their object", async () => { + const refs: WeakRef[] = []; + const { job } = await declared(Echo, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx)); + return args.text; + }, + })); + await (await job.start({ text: "a" }, context)).outcome(context); + await flushed(); + await expectCollected(...refs); + }); + + describe("of dropped runs", () => { + fakeTimers(); + + it("a dropped blocked run leaves nothing on its object", async () => { + const refs: WeakRef[] = []; + const { job, errors } = await declared(Echo, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx)); + return args.text; + }, + finish: async () => { + throw new Error("stuck"); + }, + })); + const run = await job.start({ text: "a" }, context); + await vi.advanceTimersByTimeAsync(60_000); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked"]); + const dropped = run.outcome(context).then( + () => new Error("expected a rejection"), + (error: Error) => error, + ); + void job.abort(run.id, context); + await vi.advanceTimersByTimeAsync(0); + expect(await dropped).toMatchObject({ code: "aborted" }); + expect(job.running.value).toEqual([]); + // An error keeps its stack frames' receivers (the dropping transaction) until its stack is read (V8). + void (await dropped).stack; + for (const error of errors) if (error.kind === "job_finish_failed") void (error.error as Error).stack; + await expectCollected(...refs); + }); + }); + + it("keeps nothing of a cancelled wait while its run still runs", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context); + /** One wait, cancelled; returns its signal, weakly. */ + const cancelled = async (): Promise> => { + const controller = new AbortController(); + const waiting = rejection(run.outcome(withAbortSignal(controller.signal, context))); + controller.abort("gave up"); + expect(await waiting).toBe("gave up"); + return new WeakRef(controller.signal); + }; + const refs: WeakRef[] = []; + for (let i = 0; i < 5; i++) refs.push(await cancelled()); + await expectCollected(...refs); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("keeps no ended run's args or state in a job handle kept after its object ended", async () => { + const refs: WeakRef[] = []; + const { job, host } = await declared(Shell, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx.state.value)); + await untilAborted(ctx); + return { exit: 1 }; + }, + })); + await job.start({ command: "a" }, context); + await flushed(); + await host.close(context); + await expectCollected(...refs); + expect(() => job.running.value).toThrow(ChordError); + }); + + it("an ended object releases its runs, their documents and waiters", async () => { + const refs: WeakRef[] = []; + let job: JobHandle<{ command: string }, { exit: number }, { output: string }> | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + job = self.job(Shell, { + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx), new WeakRef(ctx.state.value)); + await untilAborted(ctx); + return { exit: 1 }; + }, + }); + return { log: self.state(Log) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await job!.start({ command: "a" }, context); + await flushed(); + refs.push(new WeakRef(job!)); + job = undefined; + await host.close(context); + void item; + await expectCollected(...refs); + }); +}); + +// The type of `outcome` follows the definition. +expectTypeOf["args"]>().toEqualTypeOf<{ + command: string; +}>(); +void (null as unknown as Context); diff --git a/packages/chord/test/next/jobs/persisted.test.ts b/packages/chord/test/next/jobs/persisted.test.ts new file mode 100644 index 000000000..b144a2981 --- /dev/null +++ b/packages/chord/test/next/jobs/persisted.test.ts @@ -0,0 +1,1099 @@ +// Tests of persisted jobs (slice 6b §4, D5-D8, R3): run records and state documents in the object's store, every +// transition of §4.2 across a crash (the store closed under a live host, reopened by a new one), the clean stop and +// the close writer, a seal and its retry, the restart limit and its reset, the finish failure count, a stored abort, +// deferred state. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { MAX_PENDING_OPS } from "../../../src/delta/pending.ts"; +import { arr, int, local, num, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { RETENTION_MS, Residency } from "../../../src/next/documents/residency.ts"; +import { createHost, defineFacet, type HostError, type StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineJob, + type JobHandle, + type JobImplementation, + type Outcome, + type RunState, +} from "../../../src/next/jobs/index.ts"; +import { JobRun } from "../../../src/next/jobs/run.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { delivered, expectCollected, failLive, instancesOf } from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +const Progress = defineState("import.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 8, +}); + +const Import = defineJob("notes.import", importOptions()); + +/** `Import`'s declaration, which the other versions below change. */ +function importOptions() { + return { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: Progress, + persist: { resume: "rerun" }, + } as const; +} + +/** `Import` at the next version, as a deploy changes it. */ +const ImportV2 = defineJob("notes.import", { ...importOptions(), version: 2 }); + +/** `Import` at its version with another result schema (`num` for `int`): another fingerprint. */ +const ImportReshaped = defineJob("notes.import", { ...importOptions(), result: obj({ count: num() }) }); + +const Abandoned = defineJob("notes.abandoned", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + persist: { resume: "abandon" }, +}); + +const Store = local("notes.store", { version: 1, params: {}, members: {} }); + +type ImportArgs = { readonly path: string }; +type ImportResult = { readonly count: number }; +type ImportOutcome = Outcome; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A store file opened by `open()` again after each crash or close. */ +async function storeFile() { + directory = await mkdtemp(join(tmpdir(), "chord-jobs-")); + const path = join(directory, "store.sqlite"); + return async () => (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; +} + +/** What one host over the store sees of the store object's jobs. */ +type Opened = { + readonly store: ChordStore; + readonly errors: HostError[]; + readonly imports: JobHandle; + readonly abandoned: JobHandle; + readonly finished: ImportOutcome[]; + readonly self: StoreSelf; + /** `ref.retry()` of the store object. */ + retry(): Promise; + close(): Promise; + /** A crash: the store's backend closes under the live host, which is left behind. */ + crash(): Promise; +}; + +/** A host over `store` serving `Store` with both jobs (`Import` or another declaration of it), `run` and `finish` given. */ +async function opened( + store: ChordStore, + run: JobImplementation["run"], + finish?: (outcome: ImportOutcome) => void, + declared: typeof Import = Import, +): Promise { + const errors: HostError[] = []; + const finished: ImportOutcome[] = []; + const host = createHost({ store, onError: (error) => errors.push(error), closeTimeout: 200 }); + let found: Pick | undefined; + await host.load( + [ + { + name: "notes", + facet: defineFacet((h) => + h.serve(Store, (_, self) => { + const imports = self.job(declared, { + run, + finish: async (outcome) => { + finish?.(outcome); + finished.push(outcome); + }, + }); + const abandoned = self.job(Abandoned, { + run: async () => ({ count: 0 }), + finish: async (outcome) => void finished.push(outcome as unknown as ImportOutcome), + }); + found = { imports, abandoned, self }; + return {}; + }), + ), + }, + ], + context, + ); + return { + store, + errors, + finished, + ...found!, + retry: async () => (await host.hold(Store, {}, context)).retry(), + close: async () => { + await host.close(context); + await store.close(context); + }, + crash: () => store.close(context), + }; +} + +/** The stored revisions of the run state documents, in seq order: their kinds and contents. */ +async function stateRevisions(store: ChordStore): Promise<[string, unknown][]> { + const rows = await store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + `SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.job-states' ORDER BY r.seq`, + ), + context, + ); + return rows.map(({ kind, content }) => [kind, JSON.parse(content)]); +} + +/** Waits `ms` of real time (a drain's bound passing, a wait that must not settle). */ +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** The records the store keeps of `namespace`'s documents: their kinds and keys. */ +async function stored(store: ChordStore, namespace: string): Promise { + return store.read( + (executor) => executor.all("SELECT kind, key FROM chord_documents WHERE namespace = ? ORDER BY id", namespace), + context, + ); +} + +/** The current record of run `id`. */ +async function record(store: ChordStore, id: string): Promise | undefined> { + const row = await store.read( + (executor) => + executor.get<{ content: string }>( + `SELECT r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.jobs' AND d.key = ? ORDER BY r.seq DESC LIMIT 1`, + id, + ), + context, + ); + return row === undefined ? undefined : (JSON.parse(row.content) as Record); +} + +describe("records", () => { + it("a run's record and state live in the store while it is unsettled; finish deletes the record, retires the state", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context, { key: "k" }); + await delivered(); + expect(await stored(jobs.store, "chord.jobs")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "running", + restarts: 0, + key: "k", + args: { path: "a" }, + }); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 }, state: { done: 3 } }); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("crashes and restarts (D6)", () => { + it("a run cut off while running is counted and rerun; a state change resets the count; five in a row end it", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + for (let restart = 1; restart < 5; restart++) { + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: restart }); + } + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "restart_limit", restarts: 5 } }]); + expect(jobs.errors).toContainEqual( + expect.objectContaining({ kind: "job_interrupted", job: "notes.import", id: run.id, restarts: 5 }), + ); + await jobs.close(); + }); + + it("a state change resets the restart count, stored at once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred" } }, + ); + return new Promise(() => {}); + }); + await delivered(); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 0 }); + await jobs.crash(); + }); + + it("abandon: a run cut off by a crash finishes failed { aborted, restart } without run", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async () => ({ count: 0 })); + const gate = Promise.withResolvers(); + // Its run never returns before the crash. + void gate; + const run = await jobs.abandoned.start({ path: "a" }, context); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 0 })); + await delivered(); + await delivered(); + expect(jobs.finished.filter((outcome) => outcome.id === run.id)).toMatchObject([ + { kind: "failed", error: { kind: "aborted", reason: "restart" } }, + ]); + await jobs.close(); + }); +}); + +describe("versions", () => { + it.each([ + ["another version", ImportV2], + ["the same version with another fingerprint", ImportReshaped], + ])("a run stored by %s is blocked (version): kept as stored, never run, listed blocked", async (_, declared) => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + let runs = 0; + jobs = await opened( + await open(), + async () => { + runs++; + return { count: 1 }; + }, + undefined, + declared, + ); + await delivered(); + expect(runs).toBe(0); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // Kept as stored: the crash is not counted, slice 7 maps or resolves it. + expect(await record(jobs.store, run.id)).toMatchObject({ version: 1, phase: "running", restarts: 0 }); + await jobs.close(); + }); +}); + +describe("versions with incompatible state (review regression)", () => { + it("a run of another version is blocked before its state is decoded: no migration, no activation failure", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const changed = defineState("import.progress", { + schema: obj({ done: str() }), + version: 2, + initial: () => ({ done: "new" }), + persist: true, + checkpointWhen: () => false, + migrate: () => { + throw new Error("must not migrate an unsupported run"); + }, + }); + const declared = defineJob("notes.import", { ...importOptions(), version: 2, state: changed }); + // The declaration's state type differs from `Import`'s: this host only lists the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, declared as unknown as typeof Import); + expect(jobs.errors).toEqual([]); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // An abort drops it: its record and its undecoded state document are retired. + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("versions without a state (review regression)", () => { + it("an abort of a run blocked under a version without a state retires the old state document too", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const stateless = defineJob("notes.import", { ...importOptions(), version: 2, state: undefined }); + // The declaration's state type differs from `Import`'s: this host only lists and drops the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, stateless as unknown as typeof Import); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("a seal during the reservation (review regressions)", () => { + it("a reservation refused at admission after the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The second commit is the reservation: it reaches the store only after the seal. + jobs.store.commit = (writes, fence, ctx) => { + if (++commits !== 2) return commit(writes, fence, ctx); + entered.resolve(); + return gate.promise.then(() => commit(writes, fence, ctx)); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal during reservation" })); + await jobs.retry(); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("a reservation queued behind a held commit queue past the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const held = Promise.withResolvers<() => void>(); + const commit = jobs.store.commit.bind(jobs.store); + let first = true; + // The start's commit takes the queue as it adopts (an export would): the reservation waits behind it. + jobs.store.commit = (writes, fence, ctx) => { + if (!first) return commit(writes, fence, ctx); + first = false; + const holding = { + admit: () => fence.admit(), + adopt: (seq: number, ids: readonly number[]) => { + fence.adopt(seq, ids); + void jobs.store.take(() => {}).then(held.resolve); + }, + }; + return commit(writes, holding, ctx); + }; + const run = await jobs.imports.start({ path: "p" }, context); + const release = await held.promise; + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal queued reservation" })); + await jobs.retry(); + release(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("an abort joined to a reservation that commits after the seal returns marked", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 1 })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The reservation's commit is admitted, then held past the seal (its continuation sees the attempt sealed). + jobs.store.commit = (writes, fence, ctx) => { + const committed = commit(writes, fence, ctx); + if (++commits !== 2) return committed; + entered.resolve(); + return committed.then(async (seq) => { + await gate.promise; + return seq; + }); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal after reservation" })); + gate.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); +}); + +describe("the clean stop and the close writer", () => { + it("host close writes the stopped runs back: the next start resumes them without counting", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + (_args, ctx) => + new Promise((_, reject) => ctx.abortSignal?.addEventListener("abort", () => reject(new Error("stopped")))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.close(); + const reopened = await open(); + expect(await record(reopened, run.id)).toMatchObject({ phase: "stopped", restarts: 0 }); + jobs = await opened(reopened, async () => ({ count: 7 })); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 7 } }]); + await jobs.close(); + }); + + it("a seal (a live failure) stops the run; the retry resumes it once its old invocation settled, uncounted (R3)", async () => { + const open = await storeFile(); + let invocations = 0; + const old = Promise.withResolvers(); + const jobs = await opened(await open(), async () => { + invocations++; + if (invocations === 1) { + // Ignores its signal until released. + await old.promise; + return { count: -1 }; + } + return { count: invocations }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + await delivered(); + // The retry waits for the old invocation (the drain, bounded by closeTimeout): no second run beside it. + expect(invocations).toBe(1); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + expect(jobs.finished.map(({ kind }) => kind)).toEqual(["done"]); + await jobs.close(); + }); + + it("a close writer that cannot commit leaves the records running: the next start counts a restart", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + await jobs.close(); + jobs = await opened(await open(), () => new Promise(() => {})); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.crash(); + }); +}); + +describe("invocations after a seal (R3, review regressions)", () => { + /** A store object whose first invocation ignores its signal until `old` resolves, the next ones run `next`. */ + async function sealed(next: (ctx: { readonly abortSignal?: AbortSignal }) => Promise) { + const open = await storeFile(); + const old = Promise.withResolvers(); + const signals: boolean[] = []; + let invocations = 0; + const jobs = await opened(await open(), async (_args, ctx) => { + signals.push(ctx.abortSignal?.aborted ?? false); + if (++invocations > 1) return next(ctx); + await old.promise; + const wrote = await ctx.state + .change((draft) => { + draft.done = 99; + }, ctx) + .then( + () => "accepted", + (error: unknown) => error, + ); + return { count: wrote === "accepted" ? -2 : -1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + return { jobs, run, old, signals, invocations: () => invocations }; + } + + it("an invocation replaced after the bounded drain writes nothing and records nothing", async () => { + const current = Promise.withResolvers(); + const { jobs, run, old, invocations } = await sealed(async () => { + await current.promise; + return { count: 2 }; + }); + await jobs.retry(); + // The drain (closeTimeout 200 ms) ends without the old invocation: the retry invokes the run again. + await sleepMs(300); + expect(invocations()).toBe(2); + old.resolve(); + await delivered(); + await delivered(); + expect(jobs.finished).toEqual([]); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + current.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + await jobs.close(); + }); + + it("a resumed invocation gets a fresh abort signal", async () => { + const { jobs, run, old, signals } = await sealed(async () => ({ count: 1 })); + await jobs.retry(); + old.resolve(); + await run.outcome(context); + expect(signals).toEqual([false, false]); + await jobs.close(); + }); + + it("abort joins the resumed invocation, not the first", async () => { + const second = Promise.withResolvers(); + const { jobs, run, old } = await sealed(async () => { + await second.promise; + return { count: 2 }; + }); + old.resolve(); + await jobs.retry(); + await sleepMs(20); + let settled = false; + const aborting = jobs.imports.abort(run.id, context).then((answer) => { + settled = true; + return answer; + }); + await sleepMs(50); + expect(settled).toBe(false); + second.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); + + it("resume abandon: a run the seal stopped finishes failed { aborted, restart } without a second run", async () => { + const open = await storeFile(); + const old = Promise.withResolvers(); + let calls = 0; + const declared = defineJob("notes.import", { ...importOptions(), persist: { resume: "abandon" } }); + const jobs = await opened( + await open(), + async () => { + calls++; + if (calls === 1) await old.promise; + return { count: calls }; + }, + undefined, + declared, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ + kind: "failed", + error: { kind: "aborted", reason: "restart" }, + }); + expect(calls).toBe(1); + await jobs.close(); + }); +}); + +describe("loading (review regressions)", () => { + it("a refused recovery commit loses nothing: the retry loads the runs again and counts the crash once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const store = await open(); + const refused = vi.spyOn(store, "commit").mockRejectedValueOnce(new Error("refused recovery")); + let invocations = 0; + jobs = await opened(store, () => { + invocations++; + return new Promise(() => {}); + }); + expect(jobs.errors).toContainEqual(expect.objectContaining({ kind: "activation_failed" })); + refused.mockRestore(); + await jobs.retry(); + await delivered(); + await delivered(); + expect(invocations).toBe(1); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id }]); + expect(await record(store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.close(); + }); + + it("a state document that fails to load fails the activation; the retry with fixed code loads the run", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + const failing = { fail: true }; + const ProgressV2 = defineState("import.progress", { + schema: obj({ done: int() }), + version: 2, + migrate: (old) => { + if (failing.fail) throw new Error("migrate failed"); + return old as { done: number }; + }, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, + }); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened( + await open(), + hang, + undefined, + defineJob("notes.import", { ...importOptions(), state: ProgressV2 }), + ); + expect(jobs.errors.map(({ kind }) => kind)).toContain("activation_failed"); + failing.fail = false; + await jobs.retry(); + await delivered(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + await jobs.close(); + }); +}); + +describe("finish failures", () => { + it("are counted on the record, across a crash", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("finish failed"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 1 }); + await jobs.crash(); + const failing = () => { + throw new Error("finish failed"); + }; + jobs = await opened(await open(), async () => ({ count: 1 }), failing); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 2 }); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 1 })); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 1 } }]); + await jobs.close(); + }); +}); + +describe("abort (D7)", () => { + it("is stored and joined: marked once run settled, terminal for an unknown id", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + (_args, ctx) => + new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve({ count: -1 }))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: -1 } }); + expect(await jobs.imports.abort(run.id, context)).toBe("terminal"); + await jobs.close(); + }); + + it("an abort-marked run not invoked yet finishes aborted without run after a crash", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + await jobs.crash(); + void aborting.catch(() => {}); + let ran = false; + jobs = await opened(await open(), async () => { + ran = true; + return { count: 1 }; + }); + await delivered(); + await delivered(); + expect(ran).toBe(false); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "aborted", reason: "abort" } }]); + await jobs.close(); + }); +}); + +describe("abort of a finishing run (review regression)", () => { + it("stores the mark and keeps the recorded outcome", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("retry finish"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await sleepMs(50); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing" }); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "finishing", + abort: true, + outcome: { kind: "done", result: { count: 1 } }, + }); + await jobs.crash(); + }); +}); + +describe("abort racing an outcome", () => { + it("an abort that comes as run returns keeps its result: marked or terminal, the outcome done", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 5 })); + const run = await jobs.imports.start({ path: "a" }, context); + const aborted = await jobs.imports.abort(run.id, context); + expect(["marked", "terminal"]).toContain(aborted); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 5 } }); + await jobs.close(); + }); +}); + +describe("pinning", () => { + it("an unsettled run's state never idles: past the residency retention it is in memory, read synchronously", async () => { + const open = await storeFile(); + const touched = vi.spyOn(Residency.prototype, "touch"); + const gate = Promise.withResolvers(); + const values: number[] = []; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + values.push(ctx.state.value.done); + return { count: ctx.state.value.done }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + vi.advanceTimersByTime(2 * RETENTION_MS); + } finally { + vi.useRealTimers(); + } + await delivered(); + // The run's documents never entered the idle structure the retention evicts from. + expect(touched).not.toHaveBeenCalled(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 3 } }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 } }); + expect(values).toEqual([3]); + touched.mockRestore(); + await jobs.close(); + }); +}); + +describe("deferred state (D8)", () => { + it("is adopted at once, held, and stored with the outcome", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + let seen: number | undefined; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { + persistence: { kind: "deferred", withinMs: Infinity }, + }, + ); + seen = ctx.state.value.done; + await gate.promise; + return { count: 1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(seen).toBe(1); + const state = await jobs.store.read( + (executor) => + executor.all( + "SELECT r.kind FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states'", + ), + context, + ); + expect(state).toEqual([{ kind: "base" }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ state: { done: 1 } }); + await jobs.close(); + }); + + it("merges held ops into the next immediate change: one delta on the stored revision", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + deferred, + ); + await ctx.state.change( + (draft) => { + draft.done = 2; + }, + ctx, + deferred, + ); + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + const revisions = await jobs.store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + "SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states' ORDER BY r.seq", + ), + context, + ); + expect(revisions.map(({ kind, content }) => [kind, JSON.parse(content)])).toEqual([ + ["base", { done: 0 }], + [ + "delta", + // The held sets of one leaf folded (`PendingOps`), then the change's own. + [ + ["s", ["done"], 2], + ["s", ["done"], 3], + ], + ], + ]); + gate.resolve(); + await run.outcome(context); + await jobs.close(); + }); + + it("folds held writes: 100 sets of one leaf keep one op; past the pending bound the next change stores a base", async () => { + const open = await storeFile(); + const store = await open(); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + const jobs = await opened(store, async (_args, ctx) => { + for (let n = 1; n <= 100; n++) { + await ctx.state.change( + (draft) => { + draft.done = n; + }, + ctx, + deferred, + ); + } + await ctx.state.change((draft) => { + draft.done = 101; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + expect((await stateRevisions(store)).at(-1)).toEqual([ + "delta", + [ + ["s", ["done"], 100], + ["s", ["done"], 101], + ], + ]); + await jobs.close(); + }); + + it("an overflow of held ops stores the next change as a base", async () => { + const open = await storeFile(); + const store = await open(); + const Items = defineState("import.progress", { + schema: obj({ done: int(), items: arr(int()) }), + version: 1, + initial: () => ({ done: 0, items: new Array(MAX_PENDING_OPS + 2).fill(0) }), + persist: true, + checkpointWhen: () => false, + }); + const declared = defineJob("notes.import", { ...importOptions(), state: Items }); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + // The declaration's state type differs from `Import`'s; this run writes only `Items`' shape. + const jobs = await opened( + store, + async (_args, ctx) => { + const state = ctx.state as unknown as RunState<{ done: number; items: number[] }>; + // One leaf each: nothing folds, and the queue passes its bound. + for (let n = 0; n <= MAX_PENDING_OPS; n++) { + await state.change( + (draft) => { + draft.items[n] = 1; + }, + ctx, + deferred, + ); + } + await state.change((draft) => { + draft.done = 1; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }, + undefined, + declared as unknown as typeof Import, + ); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + // A new base prunes the older revisions (`history: none`): the start's base is gone. + expect(await stateRevisions(store)).toEqual([["base", expect.objectContaining({ done: 1 })]]); + await jobs.close(); + }); + + it("is stored by the close writer at a clean stop, and after withinMs by the object's timer", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 4; + }, + ctx, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + await jobs.close(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 4 }); + await ctx.state.change( + (draft) => { + draft.done = 5; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 10 } }, + ); + return new Promise(() => {}); + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 5 }); + return { count: 0 }; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", state: { done: 5 } }]); + await jobs.close(); + }); + + it("refuses a negative withinMs", async () => { + const open = await storeFile(); + let refused: unknown; + const jobs = await opened(await open(), async (_args, ctx) => { + refused = await ctx.state + .change(() => {}, ctx, { persistence: { kind: "deferred", withinMs: -1 } }) + .catch((error: unknown) => error); + return { count: 0 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await run.outcome(context); + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await jobs.close(); + }); +}); + +describe("GC", () => { + const runs = instancesOf(JobRun, "entryFor"); + + it("settled runs, cancelled abort waits, and an ended object with a held change and its timer are collected", async () => { + const open = await storeFile(); + /** Runs one settled run and leaves one held change behind a closed host: what must be collected. */ + const ended = async (): Promise[]> => { + // Earlier tests leave crashed hosts behind: only this one's objects and runs count. + runs.splice(0); + attempted.splice(0); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 60_000 } }, + ); + return new Promise(() => {}); + }); + const settled = await jobs.abandoned.start({ path: "a" }, context); + await settled.outcome(context); + await jobs.imports.start({ path: "b" }, context); + await delivered(); + await delivered(); + const refs = [...runs.splice(0), ...attempted.splice(0)]; + await jobs.close(); + return refs; + }; + await expectCollected(...(await ended())); + const hanging = await opened(await open(), () => new Promise(() => {})); + const run = await hanging.imports.start({ path: "c" }, context); + await delivered(); + const cancel = withCancel(context); + let waiting: Promise | undefined = hanging.imports.abort(run.id, cancel.context).catch(() => {}); + const waitRef = new WeakRef(waiting); + await delivered(); + cancel.cancel(); + await waiting; + waiting = undefined; + await expectCollected(waitRef); + await hanging.close(); + }); +}); + +describe("versions resolved by a reload (slice 7b §8)", () => { + it("a persisted run blocked for its version at its load is resumed by the reload that brings its version", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const store = await open(); + const finished: ImportOutcome[] = []; + const serve = (declared: typeof Import) => + defineFacet((h) => + h.serve(Store, (_, self) => { + self.job(declared, { + run: async () => ({ count: 7 }), + finish: async (outcome) => void finished.push(outcome), + }); + return {}; + }), + ); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "notes", facet: serve(ImportV2) }], context); + await delivered(); + expect(finished).toEqual([]); + expect(await record(store, run.id)).toMatchObject({ version: 1, phase: "running" }); + await host.reload([{ name: "notes", facet: serve(Import) }], context); + await sleepMs(20); + expect(finished).toMatchObject([{ id: run.id, kind: "done", result: { count: 7 } }]); + expect(await record(store, run.id)).toBeUndefined(); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/records.test.ts b/packages/chord/test/next/jobs/records.test.ts new file mode 100644 index 000000000..a6a0e0e2b --- /dev/null +++ b/packages/chord/test/next/jobs/records.test.ts @@ -0,0 +1,441 @@ +// Tests of a job's records on its object (slice 6a §1.2, §15 rule 5): runs across attempts, cut off by an attempt's +// seal, finished on the next live attempt, and the keep-alive of runs waiting for one. + +import { describe, expect, it } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str, unknown } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, untilAborted } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineJob, type JobHandle, type Outcome, type RunningEntry } from "../../../src/next/jobs/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { delivered, flushed, instancesOf, memoryStore, rejection, restartable, storeRestartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +const Work = defineJob("app.work", { version: 1, args: obj({ text: str() }), result: str(), state: Progress }); + +type WorkHandle = JobHandle<{ text: string }, string, { output: string }>; + +/** What a test's attempts do with a run: `run` waits for its abort (or `release`), `finish` records by attempt. */ +async function working( + finish: (attempt: number, outcome: Outcome<{ text: string }, string, { output: string }>) => void | Promise, +) { + const jobs: WorkHandle[] = []; + const object = await restartable( + objects, + (attempt, self) => { + jobs.push( + self.job(Work, { + run: async ({ text }, ctx) => { + await ctx.state.change((draft) => { + draft.output = `${text} started`; + }, ctx); + await untilAborted(ctx); + return text; + }, + finish: (outcome) => finish(attempt, outcome), + }), + ); + }, + { closeTimeout: 50 }, + ); + return { ...object, jobs }; +} + +describe("runs across attempts", () => { + it("a run cut off by its attempt's seal ends aborted (restart) and is finished on the next live attempt", async () => { + const finished: string[] = []; + const { jobs, fail, retry } = await working((attempt, outcome) => { + finished.push(`${attempt}: ${outcome.kind} ${JSON.stringify(outcome)}`); + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + fail(); + await flushed(); + // No live attempt: the run waits, finishing. + expect(finished).toEqual([]); + expect((jobs[0]!.running.value as readonly RunningEntry[]).map(({ phase }) => phase)).toEqual([ + "finishing", + ]); + await retry(); + const outcome = await run.outcome(context); + expect(outcome).toEqual({ + id: run.id, + args: { text: "a" }, + state: { output: "a started" }, + kind: "failed", + error: { kind: "aborted", reason: "restart" }, + }); + expect(finished).toEqual([`2: failed ${JSON.stringify(outcome)}`]); + expect(jobs[1]!.running.value).toEqual([]); + }); + + it("refuses the cut run's changes made while it stops; its settling records nothing more", async () => { + const late = Promise.withResolvers(); + const jobs: WorkHandle[] = []; + const { restart } = await restartable(objects, (_, self) => { + const job = self.job(Work, { + run: async (_args, ctx) => { + await untilAborted(ctx); + // Accepted after an abort (§15), refused once the attempt's seal cut the run off. + await ctx.state + .change((draft) => { + draft.output = "late"; + }, context) + .then(late.resolve, late.resolve); + return "done"; + }, + }); + jobs.push(job); + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + await restart(); + const outcome = await run.outcome(context); + expect(await late.promise).toMatchObject({ code: "released" }); + expect(outcome).toMatchObject({ kind: "failed", error: { reason: "restart" }, state: { output: "" } }); + await flushed(); + expect(jobs[1]!.running.value).toEqual([]); + }); + + it("a finish sealed mid-way is not counted as a failure and runs again on the next live attempt", async () => { + const gate = Promise.withResolvers(); + const attempts: number[] = []; + const { jobs, restart, errors } = await working(async (attempt) => { + attempts.push(attempt); + if (attempt === 1) await gate.promise; + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + void jobs[0]!.abort(run.id, context); + await flushed(); + expect(attempts).toEqual([1]); + await restart(); + gate.resolve(); + // `run` returns its result after the abort: a success. + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(attempts).toEqual([1, 2]); + // No job_finish_failed: the stuck finish only outlived the drain. + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "drain_timeout"]); + }); +}); + +describe("keep-alive", () => { + it("a run waiting for a live attempt keeps a failed object no longer: it ends with its last reference", async () => { + const { jobs, fail, reference } = await working(() => {}); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + fail(); + reference.release(); + await delivered(); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + }); +}); + +describe("the object's end during an admitted commit", () => { + const Saved = defineState("app.saved", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + + it("a run started in a commit held in the SQL while the host closes ends with the object; no run is left counted", async () => { + const store = await memoryStore(); + const declared: { saved: SingletonHandle<{ n: number }>; job: WorkHandle }[] = []; + const { host, attempts } = await storeRestartable( + objects, + (_, self) => { + const job = self.job(Work, { + run: async (_args, ctx) => { + await untilAborted(ctx); + return ""; + }, + finish: () => {}, + }); + declared.push({ saved: self.state(Saved), job }); + }, + { store }, + ); + const { saved, job } = declared[0]!; + // The next commit's step waits inside the backend transaction, after its admission. + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StagedWrite = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return [{ kind: "dropIndex", table: "x", name: "y" }]; + }, + }; + return commit([...writes, held], fence, ctx); + }; + const started = attempts[0]!.tx(async (tx) => { + (await tx.draft(saved)).n = 1; + return tx.start(job, { text: "a" }); + }, context); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await delivered(); + // The object's teardown waits for the admitted step, so the run's record is adopted before the end. + expect(closed).toBe(false); + gate.resolve(); + const run = await started; + await closing; + store.commit = commit; + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + expect(host.activity.value.jobs).toBe(0); + expect(store.lastSeq).toBe(1); + await store.close(context); + }); +}); + +/** The jobs object of the reload tests: `start` starts a run of `text`, `running` lists them. */ +const Runner = local("app.runner", { + version: 1, + params: {}, + members: { start: method(obj({ text: str() }), str()), running: arr(unknown()) }, +}); + +/** Version 2 of `Work`: its args gained a priority, its result is an object; v1 runs map forwards. */ +const WorkV2 = defineJob("app.work", { + version: 2, + args: obj({ text: str(), priority: int() }), + result: obj({ text: str() }), + state: Progress, + accepts: { + 1: { args: (old: { text: string }) => ({ ...old, priority: 0 }), result: (old: string) => ({ text: old }) }, + }, +}); + +/** `WorkV2` without an adapter for version 1. */ +const WorkV2Strict = defineJob("app.work", { + version: 2, + args: obj({ text: str(), priority: int() }), + result: obj({ text: str() }), + state: Progress, +}); + +type Finished = { readonly code: string; readonly outcome: unknown }; + +/** + * A facet serving `Runner` with `job` (labelled `code` in what its finish records into `finished`); its `run` waits for + * `gates.shift()` when one is queued; with `native` the object opens a native (a reload restarts it). + */ +function runner( + code: string, + job: typeof Work | typeof WorkV2 | typeof WorkV2Strict | undefined, + finished: Finished[], + options: { readonly gates?: Promise[]; readonly native?: boolean } = {}, +) { + return defineFacet((h) => + h.serve(Runner, (_, self) => { + if (options.native) + self.open( + async () => code, + () => {}, + ); + if (job === undefined) return { start: async () => "", running: constant([]) }; + const handle = ( + self.job as (definition: unknown, implementation: unknown) => JobHandle + )(job, { + run: async (args: { text: string }, ctx: { readonly abortSignal?: AbortSignal }) => { + const gate = options.gates?.shift(); + if (gate !== undefined) await Promise.race([gate, untilAborted(ctx as never)]); + ctx.abortSignal?.throwIfAborted(); + return job.version === 1 ? `${args.text} by ${code}` : { text: `${args.text} by ${code}` }; + }, + finish: (outcome: unknown) => void finished.push({ code, outcome }), + }); + return { + start: async ({ text }: { text: string }, ctx: Context) => + (await handle.start((job.version === 1 ? { text } : { text, priority: 1 }) as never, ctx)).id, + running: handle.running as never, + }; + }), + ); +} + +describe("jobs across a reload (slice 7b §8)", () => { + it("an older run finishes on its old code across a swap; its finish runs on the new code, mapped forwards once", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", WorkV2, finished) }], context); + await flushed(); + // Listed with its args mapped forwards. + expect(jobs.running.value).toMatchObject([{ version: 1, args: { text: "a", priority: 0 }, phase: "running" }]); + gate.resolve(); + await flushed(); + expect(finished).toEqual([ + { + code: "v2", + outcome: expect.objectContaining({ + kind: "done", + args: { text: "a", priority: 0 }, + result: { text: "a by v1" }, + }), + }, + ]); + await host.close(context); + }); + + it("a run without an adapter is blocked (version) until a reload brings matching code", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + const id = await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", WorkV2Strict, finished) }], context); + gate.resolve(); + await flushed(); + expect(jobs.running.value).toEqual([{ id, version: 1, phase: "blocked", reason: "version" }]); + expect(finished).toEqual([]); + await host.reload([{ name: "@test/app", facet: runner("v1b", Work, finished) }], context); + await flushed(); + expect(finished).toEqual([ + { code: "v1b", outcome: expect.objectContaining({ kind: "done", result: "a by v1" }) }, + ]); + expect(jobs.running.value).toEqual([]); + await host.close(context); + }); + + it("a native restart aborts running in-memory runs (reload); their finish runs on the new code", async () => { + const finished: Finished[] = []; + const host = createHost({ onError: () => {} }); + const gate = Promise.withResolvers(); + await host.load( + [{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise], native: true }) }], + context, + ); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", Work, finished, { native: true }) }], context); + await flushed(); + expect(finished).toEqual([ + { + code: "v2", + outcome: expect.objectContaining({ kind: "failed", error: { kind: "aborted", reason: "reload" } }), + }, + ]); + await host.close(context); + }); + + it("a job the new code no longer declares blocks its runs (version); declared again, they go on", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", undefined, finished) }], context); + gate.resolve(); + await flushed(); + expect(finished).toEqual([]); + await host.reload([{ name: "@test/app", facet: runner("v3", Work, finished) }], context); + await flushed(); + expect(finished).toEqual([{ code: "v3", outcome: expect.objectContaining({ kind: "done", result: "a by v1" }) }]); + await host.close(context); + }); +}); + +describe("a reservation released after a swap (final review)", () => { + it("starts the old code's run with its own args, not the new version's mapping (Sol)", async () => { + const Owner = local("app.reserve-owner", { version: 1, params: {}, members: {} }); + const V1 = defineJob("app.reserve", { + version: 1, + args: obj({ n: int() }), + result: int(), + persist: { resume: "rerun" }, + }); + const V2 = defineJob("app.reserve", { + version: 2, + args: obj({ label: str() }), + result: int(), + persist: { resume: "rerun" }, + accepts: { 1: { args: (old: { n: number }) => ({ label: `mapped ${old.n}` }), result: (old: number) => old } }, + }); + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const seen: unknown[] = []; + let job: JobHandle<{ n: number }, number, undefined> | undefined; + await host.load( + [ + { + name: "@app/reserve", + facet: defineFacet((h) => + h.serve(Owner, (_, self) => { + job = self.job(V1, { + run: async (args) => { + seen.push(args.n); + return 42; + }, + }); + return {}; + }), + ), + }, + ], + context, + ); + // The run's reservation (the second commit) waits on `gate` until the swap to V2 happened. + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = store.commit.bind(store); + let writes = 0; + store.commit = (staged, fence, ctx) => { + if (++writes !== 2) return commit(staged, fence, ctx); + const held: StagedWrite = { + order: "write", + apply: async () => { + entered.resolve(); + await gate.promise; + return []; + }, + }; + return commit([...staged, held], fence, ctx); + }; + const starting = job!.start({ n: 1 }, context); + await entered.promise; + const run = await starting; + const v2 = defineFacet((h) => + h.serve(Owner, (_, self) => { + self.job(V2, { run: async () => 43 }); + return {}; + }), + ); + const reloading = host.reload([{ name: "@app/reserve", facet: v2 }], context); + await delivered(); + gate.resolve(); + await reloading; + await run.outcome(context); + // V1's handler got V1's args. + expect(seen).toEqual([1]); + store.commit = commit; + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/run.test.ts b/packages/chord/test/next/jobs/run.test.ts new file mode 100644 index 000000000..b430e62bc --- /dev/null +++ b/packages/chord/test/next/jobs/run.test.ts @@ -0,0 +1,60 @@ +// Tests of one run's record (§15): its `running` entry per phase, and its handle's wait for the outcome. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { JobRun } from "../../../src/next/jobs/run.ts"; +import { rejection } from "../support.ts"; + +describe("a run", () => { + it("has an entry per phase: args and state while running or finishing, neither when blocked, with its reason", () => { + const run = new JobRun("r1", 2, "f2", "k", { command: "ls" }, "running"); + expect(run.entryFor({ output: "" }, run.args)).toEqual({ + id: "r1", + version: 2, + key: "k", + args: { command: "ls" }, + state: { output: "" }, + phase: "running", + }); + run.phase = "finishing"; + expect(new JobRun("r2", 1, "f1", undefined, { a: 1 }, "running").entryFor(undefined, { a: 1 })).toEqual({ + id: "r2", + version: 1, + args: { a: 1 }, + phase: "running", + }); + run.phase = "blocked"; + run.blocked = "finish_failed"; + expect(run.entryFor({ output: "x" }, run.args)).toEqual({ + id: "r1", + version: 2, + phase: "blocked", + reason: "finish_failed", + }); + run.blocked = "version"; + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "blocked", reason: "version" }); + // A persisted run waiting for its reservation, or stopped, runs (again): `running`. + run.phase = "pending"; + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "running" }); + run.phase = "stopped"; + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "running" }); + // An older run's entry shows its args as the current code takes them (mapped by its job). + run.phase = "running"; + expect(run.entryFor(undefined, { command: "ls -l" })).toMatchObject({ args: { command: "ls -l" } }); + }); + + it("gives every waiter the outcome, late ones too; a waiter's context ends only its wait", async () => { + const run = new JobRun("r1", 1, "f1", undefined, {}, "running"); + const { context: waiting, cancel } = withCancel(context); + const cancelled = rejection(run.handle.outcome(waiting)); + cancel(new Error("left")); + expect(await cancelled).toMatchObject({ message: "left" }); + const outcome = { id: "r1", args: {}, state: undefined, kind: "done", result: 1 } as const; + run.resolve(outcome); + expect(await run.handle.outcome(context)).toBe(outcome); + const rejected = new JobRun("r2", 1, "f1", undefined, {}, "running"); + rejected.reject(new ChordError({ code: "aborted", message: "dropped" })); + expect(await rejection(rejected.handle.outcome(context))).toMatchObject({ code: "aborted" }); + }); +}); diff --git a/packages/chord/test/next/jobs/running.test.ts b/packages/chord/test/next/jobs/running.test.ts new file mode 100644 index 000000000..6f153ec1f --- /dev/null +++ b/packages/chord/test/next/jobs/running.test.ts @@ -0,0 +1,124 @@ +// Tests of `running`'s list (§15): each change copies the list, keeps the other entries identical and gives the ops +// of exactly that change (a state change re-pathed under its entry); one frame per batch; derives over it are marked. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { applyImmutable, type Op } from "../../../src/delta/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { Running } from "../../../src/next/jobs/running.ts"; +import { Batch, Derive } from "../../../src/next/state/internal.ts"; +import { delivered, framesOf } from "../support.ts"; + +function running() { + const batch = new Batch(() => {}, undefined); + return { batch, list: new Running(batch, "app.store job app.echo") }; +} + +describe("running's list", () => { + it("copies the list per change, keeps other entries identical, and frames apply to the previous value", async () => { + const { batch, list } = running(); + const a = { id: "a", phase: "running", state: { output: "" } }; + const b = { id: "b", phase: "running", state: { output: "" } }; + const watch = await list.watch(context); + const frames = framesOf(watch); + let previous = list.value; + batch.commit(() => { + list.add(a); + list.add(b); + }, context); + await delivered(); + const added = list.value as JsonValue[]; + expect(added).toEqual([a, b]); + expect(added).not.toBe(previous); + const b2 = { ...b, state: { output: "x" } }; + batch.commit(() => list.stateChanged(b, b2, [["a", ["output"], "x"]]), context); + await delivered(); + const a2 = { ...a, phase: "finishing" }; + batch.commit(() => list.replace(a, a2), context); + await delivered(); + batch.commit(() => list.remove(a2), context); + await delivered(); + expect(list.value).toEqual([b2]); + expect((list.value as JsonValue[])[0]).toBe(b2); + expect(frames.map(([, ops]) => ops)).toEqual([ + [ + ["p", [], 0, 0, [a]], + ["p", [], 1, 0, [b]], + ], + [["a", [1, "state", "output"], "x"]], + [["s", [0], a2]], + [["p", [], 0, 1, []]], + ] satisfies Op[][]); + // Every frame turns the previous value into the next. + previous = []; + for (const [value, ops] of frames) { + expect(applyImmutable(previous, ops)).toEqual(value); + previous = value; + } + }); + + it("re-paths every kind of state op under its entry; a replacement becomes a set", async () => { + const { batch, list } = running(); + const entry = { id: "a", state: { list: [1], text: "", n: 0 } }; + batch.commit(() => list.add(entry), context); + const frames = framesOf(await list.watch(context)); + const after = { id: "a", state: { list: [2], text: "t", n: 1 } }; + const ops: Op[] = [ + ["r", { list: [], text: "", n: 0 }], + ["s", ["n"], 1], + ["p", ["list"], 0, 0, [2]], + ["a", ["text"], "t"], + ["d", ["gone"]], + ["t", ["text"], 1], + ["m", ["list"], [0]], + ]; + batch.commit(() => list.stateChanged(entry, after, ops), context); + await delivered(); + expect(frames[0]![1]).toEqual([ + ["s", [0, "state"], { list: [], text: "", n: 0 }], + ["s", [0, "state", "n"], 1], + ["p", [0, "state", "list"], 0, 0, [2]], + ["a", [0, "state", "text"], "t"], + ["d", [0, "state", "gone"]], + ["t", [0, "state", "text"], 1], + ["m", [0, "state", "list"], [0]], + ]); + }); + + it("builds no op for a state change while unwatched (cost guard: the ops are not even walked)", () => { + const { batch, list } = running(); + const entry = { id: "a", state: { output: "" } }; + batch.commit(() => list.add(entry), context); + const unwalked = { + [Symbol.iterator]: () => { + throw new Error("walked"); + }, + } as unknown as Op[]; + const after = { id: "a", state: { output: "x" } }; + batch.commit(() => list.stateChanged(entry, after, unwalked), context); + expect(list.value).toEqual([after]); + }); + + it("keeps nothing once ended: reads throw released", async () => { + const { batch, list } = running(); + batch.commit(() => list.add({ id: "a" }), context); + const watch = await list.watch(context); + list.end(); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => list.value).toThrow("app.store job app.echo: ended"); + }); + + it("records no ops while unwatched, and marks derives over it at every change", () => { + const { batch, list } = running(); + const count = new Derive([list], ([value]) => (value as JsonValue[]).length, batch); + count.attach( + (source) => source as Running, + () => {}, + ); + expect(count.value).toBe(0); + batch.commit(() => list.add({ id: "a" }), context); + expect(count.value).toBe(1); + batch.commit(() => list.add({ id: "b" }), context); + expect(count.value).toBe(2); + }); +}); diff --git a/packages/chord/test/next/link/channel.test.ts b/packages/chord/test/next/link/channel.test.ts new file mode 100644 index 000000000..4179830af --- /dev/null +++ b/packages/chord/test/next/link/channel.test.ts @@ -0,0 +1,350 @@ +// Tests of a host's end of a link: the handshake, checks, the heartbeat, trusted links, and in-order handling while a +// message waits for an asynchronous inflate. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, remote } from "../../../src/next/contracts/index.ts"; +import { PROTOCOL_VERSION } from "../../../src/next/data/internal.ts"; +import { createHost, type HostError } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, Channel, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import { + bindNext, + expectCollected, + flushed, + forEachCodec, + linked, + metered, + Notes, + RAW_PEER, + RawPeer, + rawLinked, + rawProvider, + SERVER_ADDRESS, + Server, + testCodec, + transported, +} from "../support.ts"; + +const Thing = remote("app.thing", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { only: { state: { n: int() } } }, +}); + +const hold = { + kind: "hold", + hold: 0, + type: "app.thing", + version: 1, + fingerprint: "0000000000000000", + params: "{}", + address: SERVER_ADDRESS, +} as const; + +const problems = (errors: readonly HostError[]) => + errors.flatMap((error) => (error.kind === "protocol_error" ? [(error.error as Error).message] : [])); + +forEachCodec(() => + describe("the handshake", () => { + it("sends hello with the protocol version and the host's incarnation first", async () => { + const { peer } = await rawLinked(() => {}); + const hello = await peer.next("hello"); + expect(hello.protocol).toBe(PROTOCOL_VERSION); + expect(hello.incarnation).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + }); + + it("closes the link on traffic before hello", async () => { + const { peer, errors } = await rawLinked(() => {}, { hello: false }); + peer.send(hold); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["handshake: hold before hello"]); + }); + + it("closes the link on a second hello", async () => { + const { peer, errors } = await rawLinked(() => {}); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["handshake: a second hello"]); + }); + + it("closes the link on another protocol version", async () => { + const { peer, errors } = await rawLinked(() => {}, { hello: false }); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION + 1, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual([ + `handshake: protocol ${PROTOCOL_VERSION + 1}, this host speaks ${PROTOCOL_VERSION}`, + ]); + }); + + it("closes the link on a message failing its checks", async () => { + const { peer, errors } = await rawLinked(() => {}); + peer.sendBytes(testCodec().encode({ kind: "error", call: 0, error: { kind: "nope" } } as never)); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["message: fails the message schema"]); + }); + + it("queues a consumer's messages until the peer's hello; a get over a link that never greets waits for its deadline", async () => { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { hello: false }); + const got = client.get(Thing, {}, context, { readyTimeout: 20 }); + await flushed(); + expect(peer.kinds()).toEqual(["hello"]); + await expect(got).rejects.toThrow("app.thing: did not become live within 20 ms"); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.kinds()).toEqual(["hello", "hold", "release"]); + }); + }), +); + +describe("a codec mismatch", () => { + it("fails the first hello as a protocol error naming the expected codec", async () => { + const errors: HostError[] = []; + const server = createHost({ onError: (error) => errors.push(error) }); + await server.load([], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: binaryCodec }, context); + client.connect(b, { principal: { kind: "server" }, codec: textCodec }, context); + await transported(); + expect(problems(errors)).toEqual(["handshake: the first message is no binary message"]); + }); +}); + +describe("the heartbeat", () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + /** Advances fake time by `ms` in heartbeat ticks, letting links deliver between them. */ + async function elapse(ms: number): Promise { + for (let step = 0; step < ms; step += 1500) { + await vi.advanceTimersByTimeAsync(Math.min(1500, ms - step)); + await transported(); + } + } + + it("closes a link whose peer never sends hello after 30 s (race 30)", async () => { + const { peer } = await rawLinked(() => {}, { hello: false }); + await elapse(29_000); + expect(peer.closed).toBe(false); + await elapse(1_500); + expect(peer.closed).toBe(true); + }); + + it("pings a silent link after 15 s and closes it after 30 s without an answer: references go loading", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + await elapse(13_500); + expect(peer.kinds()).not.toContain("ping"); + await elapse(3_000); + expect(peer.kinds().filter((kind) => kind === "ping")).toEqual(["ping"]); + await elapse(15_000); + expect(peer.closed).toBe(true); + expect(notes.state.value).toEqual({ kind: "loading" }); + }); + + it("sends no pings on a busy link", async () => { + const { peer } = await rawLinked(() => {}); + for (let second = 0; second < 60; second += 5) { + peer.send({ kind: "release", hold: 99 }); + await elapse(5_000); + } + expect(peer.kinds()).toEqual(["hello"]); + expect(peer.closed).toBe(false); + }); + + it("answers each ping with a pong at once; two sides pinging at once both answer", async () => { + const { client, server } = await linked(() => {}); + await elapse(60_000); + // Both sides pinged at 15 s and were answered: neither closed the link. + const notes = client.hold(Notes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed" }); + expect(server).toBeDefined(); + const { peer } = await rawLinked(() => {}); + peer.send({ kind: "ping" }); + expect(await peer.next("pong")).toEqual({ kind: "pong" }); + }); + + it("runs one timer per host, only while it has links: none per message", async () => { + const host = createHost({ onError: () => {} }); + await host.load([], context); + expect(vi.getTimerCount()).toBe(0); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const [c, d] = memoryLinkPair(); + host.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peers = [new RawPeer(b), new RawPeer(d)]; + for (let round = 0; round < 50; round++) peers[0]!.send({ kind: "release", hold: round }); + await transported(); + expect(vi.getTimerCount()).toBe(1); + for (const peer of peers) peer.close(); + await transported(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("takes its numbers from the host's options", async () => { + const host = createHost({ onError: () => {}, heartbeat: { ping: 1_000, close: 2_000 } }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await vi.advanceTimersByTimeAsync(1_100); + await transported(); + expect(peer.kinds()).toEqual(["hello", "ping"]); + await vi.advanceTimersByTimeAsync(1_000); + await transported(); + expect(peer.closed).toBe(true); + }); +}); + +describe("trusted links", () => { + it("skip the message schema check: no chord.link.checks, the codec and the owners' checks still run", async () => { + const run = async (trusted: boolean) => { + const { context: counting, counts } = await metered(); + const host = createHost({ onError: () => {} }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, trusted, codec: testCodec() }, counting); + const peer = new RawPeer(b); + peer.send({ kind: "release", hold: 1 }); + peer.sendBytes(new Uint8Array([0xff])); + await transported(); + return { checks: counts()["chord.link.checks"] ?? 0, closed: peer.closed }; + }; + expect(await run(false)).toEqual({ checks: 2, closed: true }); + expect(await run(true)).toEqual({ checks: 0, closed: true }); + }); +}); + +describe("in-order handling", () => { + /** A channel whose handler handles `release` of hold 0 asynchronously (until `finish`), every other message at once. */ + async function holding() { + const [a, b] = memoryLinkPair(); + const handled: string[] = []; + const reports: unknown[] = []; + let finish: { resolve(): void; reject(error: Error): void } | undefined; + let closed = false; + const handler = { + opened: () => {}, + closed: () => { + closed = true; + }, + message: (message: Message): void | Promise => { + const name = message.kind === "release" ? `release ${message.hold}` : message.kind; + if (message.kind !== "release" || message.hold !== 0) { + handled.push(name); + return; + } + const { promise, resolve, reject } = Promise.withResolvers(); + finish = { resolve, reject }; + return promise.then(() => void handled.push(name)); + }, + }; + const options = { + codec: testCodec(), + incarnation: RAW_PEER, + calls: 1, + trusted: true, + report: (e: unknown) => reports.push(e), + }; + new Channel(a, handler, options, context).start(); + const peer = new RawPeer(b); + await peer.next("hello"); + return { peer, handled, reports, finish: () => finish!, isClosed: () => closed }; + } + + forEachCodec(() => { + it("holds every later message, a ping included, until the asynchronous one was handled, then handles them in order", async () => { + const { peer, handled, finish } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + peer.send({ kind: "ping" }); + peer.send({ kind: "release", hold: 2 }); + await transported(); + expect(handled).toEqual([]); + expect(peer.kinds()).toEqual([]); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0", "release 1", "release 2"]); + expect(peer.kinds()).toEqual(["pong"]); + }); + + it("keeps the rest behind a held message that holds again, in order, a long queue included", async () => { + const { peer, handled, finish } = await holding(); + const release = (hold: number) => testCodec().encode({ kind: "release", hold }); + peer.sendBytes(release(0)); + peer.sendBytes(release(1)); + peer.sendBytes(release(0)); + // Beyond what a spread into function arguments takes: the queue is read by index. + const many = release(2); + for (let i = 0; i < 70_000; i++) peer.sendBytes(many); + await transported(); + expect(handled).toEqual([]); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0", "release 1"]); + finish().resolve(); + await transported(); + expect(handled.length).toBe(70_003); + expect(handled.slice(0, 4)).toEqual(["release 0", "release 1", "release 0", "release 2"]); + }); + + it("drops the held message's successors when the link closes meanwhile (race 13)", async () => { + const { peer, handled, finish, isClosed } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + await transported(); + peer.close(); + await transported(); + expect(isClosed()).toBe(true); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0"]); + }); + + it("keeps no held message once the link closed (GC)", async () => { + const { peer, finish, refs } = await (async () => { + const { peer, finish } = await holding(); + peer.send({ kind: "release", hold: 0 }); + const held = testCodec().encode({ kind: "release", hold: 1 }); + peer.sendBytes(held); + await transported(); + peer.close(); + await transported(); + return { peer, finish, refs: [new WeakRef(held)] }; + })(); + await expectCollected(...refs); + finish().resolve(); + expect(peer.closed).toBe(true); + }); + + it("treats a rejected asynchronous handling as a protocol error: reported, the link closes", async () => { + const { peer, handled, finish, reports } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + await transported(); + finish().reject(new Error("corrupt")); + await transported(); + expect(peer.closed).toBe(true); + expect(reports).toEqual([new Error("corrupt")]); + expect(handled).toEqual([]); + }); + }); +}); diff --git a/packages/chord/test/next/link/errors.test.ts b/packages/chord/test/next/link/errors.test.ts new file mode 100644 index 000000000..2303a00b9 --- /dev/null +++ b/packages/chord/test/next/link/errors.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { ChordError, PluginError } from "../../../src/next/data/index.ts"; +import { errorFromWire, wireError } from "../../../src/next/link/internal.ts"; + +describe("errors on the wire", () => { + it("carry a ChordError as its JSON, without stack or cause", () => { + const error = new ChordError({ code: "invalid", reason: "args", message: "bad" }, { cause: new Error("x") }); + const wire = wireError(error); + expect(wire).toEqual({ kind: "chord", error: { code: "invalid", reason: "args", message: "bad" } }); + const back = errorFromWire(JSON.parse(JSON.stringify(wire))); + expect(back).toBeInstanceOf(ChordError); + expect(back).toMatchObject({ code: "invalid", reason: "args", message: "bad" }); + }); + + it("carry a PluginError's code, message and details only; the consumer rethrows a PluginError", () => { + const error = new PluginError("quota", "too many notes", { limit: 3 }); + const wire = wireError(error); + expect(wire).toEqual({ kind: "plugin", code: "quota", message: "too many notes", details: { limit: 3 } }); + const back = errorFromWire(wire); + expect(back).toBeInstanceOf(PluginError); + expect(back).toMatchObject({ code: "quota", message: "too many notes", details: { limit: 3 } }); + expect(wireError(new PluginError("c", "m"))).toEqual({ kind: "plugin", code: "c", message: "m" }); + }); + + it("carry anything else as the plugin code internal with its message", () => { + expect(wireError(new TypeError("boom"))).toEqual({ kind: "plugin", code: "internal", message: "boom" }); + expect(wireError("plain")).toEqual({ kind: "plugin", code: "internal", message: "plain" }); + }); + + it("throw on a malformed ChordError envelope (a protocol error)", () => { + expect(() => errorFromWire({ kind: "chord", error: { code: "nope", message: "m" } })).toThrow( + "Invalid ChordError JSON", + ); + }); +}); diff --git a/packages/chord/test/next/link/fixtures/ipc-echo.ts b/packages/chord/test/next/link/fixtures/ipc-echo.ts new file mode 100644 index 000000000..6764f8992 --- /dev/null +++ b/packages/chord/test/next/link/fixtures/ipc-echo.ts @@ -0,0 +1,10 @@ +// A forked child for the IPC link test: echoes every message of its parent link, and exits when the link closes. +import process from "node:process"; +import { ipcLink } from "../../../../src/next/link/index.ts"; + +const link = ipcLink(process as unknown as Parameters[0]); +link.onMessage((message) => { + if (message[0] === 0xff) process.exit(0); + link.send(message); +}); +link.onClose(() => process.exit(0)); diff --git a/packages/chord/test/next/link/ipc.test.ts b/packages/chord/test/next/link/ipc.test.ts new file mode 100644 index 000000000..da9e87d21 --- /dev/null +++ b/packages/chord/test/next/link/ipc.test.ts @@ -0,0 +1,143 @@ +import { fork } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { type IpcPort, ipcLink } from "../../../src/next/link/index.ts"; +import { flushed } from "../support.ts"; + +/** A port playing a Node IPC channel by hand: what was sent, its listeners, and what the test makes happen. */ +class FakePort implements IpcPort { + connected = true; + readonly sent: Uint8Array[] = []; + /** What `send` returns, or throws when an error. */ + next: boolean | Error = true; + callbacks: ((error: Error | null) => void)[] = []; + disconnects = 0; + readonly #message = new Set<(message: unknown) => void>(); + readonly #disconnect = new Set<() => void>(); + + send(message: Uint8Array, callback: (error: Error | null) => void): boolean { + if (this.next instanceof Error) throw this.next; + this.sent.push(message); + this.callbacks.push(callback); + return this.next; + } + + on(event: "message" | "disconnect", listener: (message: unknown) => void): this { + if (event === "message") this.#message.add(listener); + else this.#disconnect.add(listener as () => void); + return this; + } + + removeListener(event: "message" | "disconnect", listener: (message: unknown) => void): this { + if (event === "message") this.#message.delete(listener); + else this.#disconnect.delete(listener as () => void); + return this; + } + + disconnect(): void { + this.disconnects++; + this.connected = false; + for (const listener of [...this.#disconnect]) listener(); + } + + receive(message: unknown): void { + for (const listener of [...this.#message]) listener(message); + } + + get listeners(): number { + return this.#message.size + this.#disconnect.size; + } +} + +describe("IPC links over a fake port", () => { + it("deliver Uint8Array messages in order, queued until the listener is set; other messages are not the link's", () => { + const port = new FakePort(); + const link = ipcLink(port); + port.receive(new Uint8Array([1])); + port.receive({ start: "the app's own" }); + port.receive(new Uint8Array([2])); + const received: number[] = []; + link.onMessage((message) => received.push(message[0]!)); + port.receive(new Uint8Array([3])); + expect(received).toEqual([1, 2, 3]); + }); + + it("treat send returning false as backpressure, not a close", () => { + const port = new FakePort(); + const link = ipcLink(port); + let closed = 0; + link.onClose(() => closed++); + port.next = false; + link.send(new Uint8Array([1])); + link.send(new Uint8Array([2])); + expect(port.sent).toHaveLength(2); + expect(closed).toBe(0); + }); + + it.each([ + [ + "a thrown send", + (port: FakePort) => { + port.next = new Error("channel closed"); + }, + ], + ["an error in the send callback", (port: FakePort) => port.callbacks[0]!(new Error("EPIPE"))], + ["a disconnect", (port: FakePort) => port.disconnect()], + ])("close once on %s; listeners removed, later messages dropped (race 27)", (_, fail) => { + const port = new FakePort(); + const link = ipcLink(port); + let closed = 0; + const received: Uint8Array[] = []; + link.onClose(() => closed++); + link.onMessage((message) => received.push(message)); + link.send(new Uint8Array([1])); + fail(port); + link.send(new Uint8Array([2])); + port.disconnect(); + port.receive(new Uint8Array([3])); + expect(closed).toBe(1); + expect(received).toEqual([]); + expect(port.listeners).toBe(0); + }); + + it("disconnect the port on close, once; a port not connected gives a closed link", () => { + const port = new FakePort(); + const link = ipcLink(port); + link.close(); + link.close(); + expect(port.disconnects).toBe(1); + let closed = false; + ipcLink(port).onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + }); +}); + +describe("IPC links to a forked child", () => { + it("carry Uint8Array messages both ways and close when the child exits mid-send (race 27)", async () => { + const child = fork(fileURLToPath(new URL("./fixtures/ipc-echo.ts", import.meta.url)), [], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + const link = ipcLink(child); + const echoed = new Promise((resolve) => link.onMessage(resolve)); + let closes = 0; + const closed = new Promise((resolve) => + link.onClose(() => { + closes++; + resolve(); + }), + ); + link.send(new Uint8Array([1, 2, 3])); + const message = await echoed; + expect(message).toBeInstanceOf(Uint8Array); + expect([...message]).toEqual([1, 2, 3]); + // The child exits on this one; the sends after it race the exit. + link.send(new Uint8Array([0xff])); + for (let send = 0; send < 20; send++) link.send(new Uint8Array(1024)); + await closed; + await flushed(); + expect(closes).toBe(1); + }, 20_000); +}); diff --git a/packages/chord/test/next/link/link.test.ts b/packages/chord/test/next/link/link.test.ts new file mode 100644 index 000000000..0f720f196 --- /dev/null +++ b/packages/chord/test/next/link/link.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { expectCollected, flushed } from "../support.ts"; + +const bytes = (...values: number[]) => new Uint8Array(values); + +describe("memory links", () => { + it("deliver in order, one task later, and queue what arrives before the listener", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.send(bytes(1)); + a.send(bytes(2)); + await flushed(); + b.onMessage((message) => got.push(message[0]!)); + expect(got).toEqual([1, 2]); + a.send(bytes(3)); + expect(got).toEqual([1, 2]); + await flushed(); + expect(got).toEqual([1, 2, 3]); + }); + + it("close this end at once and the peer one task later; send after close is a no-op", async () => { + const [a, b] = memoryLinkPair(); + const closes: string[] = []; + const got: number[] = []; + a.onClose(() => closes.push("a")); + b.onClose(() => closes.push("b")); + b.onMessage((message) => got.push(message[0]!)); + a.close(); + expect(closes).toEqual(["a"]); + a.send(bytes(1)); + await flushed(); + expect(closes).toEqual(["a", "b"]); + b.send(bytes(2)); + await flushed(); + expect(got).toEqual([]); + const late: string[] = []; + a.onClose(() => late.push("late")); + expect(late).toEqual(["late"]); + }); + + it("drop messages still in flight to a closed end", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.onMessage((message) => got.push(message[0]!)); + b.send(bytes(1)); + a.close(); + await flushed(); + expect(got).toEqual([]); + }); + + it("stop delivering queued messages once a listener closed the link", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.send(bytes(1)); + a.send(bytes(2)); + await flushed(); + b.onMessage((message) => { + got.push(message[0]!); + b.close(); + }); + expect(got).toEqual([1]); + }); + + it("keep no listener once closed", async () => { + const [a, b] = memoryLinkPair(); + const refs = (() => { + const listener = { onMessage: () => {} }; + a.onMessage(listener.onMessage); + a.onClose(listener.onMessage); + return [new WeakRef(listener.onMessage)]; + })(); + a.close(); + await flushed(); + await expectCollected(...refs); + expect(b).toBeDefined(); + }); +}); + +describe("a closed memory link", () => { + it("keeps no listener set after its close: accepting it keeps no host", async () => { + const [a, b] = memoryLinkPair(); + a.close(); + await flushed(); + const refs = await (async () => { + const host = createHost({ onError: () => {} }); + await host.load([], BACKGROUND_CONTEXT); + host.accept(a, { principal: { kind: "owner" } }, BACKGROUND_CONTEXT); + return [new WeakRef(host)]; + })(); + await expectCollected(...refs); + expect([a, b]).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/link/message.test.ts b/packages/chord/test/next/link/message.test.ts new file mode 100644 index 000000000..35be68873 --- /dev/null +++ b/packages/chord/test/next/link/message.test.ts @@ -0,0 +1,629 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + binaryCodec, + checkMessage, + DeflatedPart, + describeMessage, + jsonPart, + type Message, + parsePart, + parsePayload, + payloadPart, + textCodec, +} from "../../../src/next/link/internal.ts"; +import type { Part } from "../../../src/next/link/part.ts"; +import { forEachCodec, testCodec } from "../support.ts"; + +/** A part as JSON text: a received part's bytes decoded (a deflated part stays as it is). */ +function partText(part: Part): Part { + return part instanceof Uint8Array ? new TextDecoder().decode(part) : part; +} + +const incarnation = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; + +/** The address a hold names: `Server()` on the wire. */ +const server = { kind: "server", params: {} }; +const error = { kind: "chord", error: { code: "released", message: "gone" } } as const; + +/** One message of every kind, with the optional fields both present and absent; parts are JSON text. */ +const EVERY_KIND: Message[] = [ + { kind: "hello", protocol: 1, incarnation, calls: 256 }, + { + kind: "hold", + hold: 300, + type: "app.notes", + version: 2, + fingerprint: "0123456789abcdef", + params: '{"id":1}', + address: server, + }, + { + kind: "hold", + hold: 1, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: '{"session":"S1"}', + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "guest" }, + incarnation, + }, + { + kind: "hold", + hold: 2, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + incarnation, + }, + { kind: "release", hold: 0 }, + { kind: "call", call: 3, hold: 0, method: 1, args: '{"text":"x"}' }, + { kind: "call", call: 4, hold: 0, method: 0 }, + { kind: "abort", call: 3 }, + { + kind: "bound", + holds: [{ hold: 0, variant: 1, members: [{ member: 0, value: '[1,"a"]' }], failed: [{ member: 1, error }] }], + }, + { + kind: "bound", + holds: [ + { hold: 0, variant: 0, members: [], failed: [] }, + { hold: 5, variant: 2, members: [{ member: 1, value: "null" }], failed: [] }, + ], + }, + { + kind: "state", + holds: [ + { hold: 0, members: [{ member: 2, ops: '[["s",["n"],1]]' }], failed: [], retired: [] }, + { hold: 1, members: [], failed: [{ member: 0, error }], retired: [] }, + // Retired members (decision "One retirement mechanism"), alone and with a failure. + { hold: 2, members: [{ member: 1, ops: '[["r",{}]]' }], failed: [], retired: [1, 3] }, + { hold: 3, members: [], failed: [{ member: 0, error }], retired: [2] }, + ], + }, + { + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { + kind: "failed", + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + retryIn: 0, + }, + // Not the object's fault (dependency model §2): no backoff window, an event wakes it. + { + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "contract_mismatch", message: "m" } } }, + }, + // The wire error format in `failed` and `gone`: a plugin's error travels too. + { kind: "failed", hold: 0, cause: { kind: "broken", error: { kind: "plugin", code: "config", message: "m" } } }, + { kind: "gone", hold: 1, error: { kind: "plugin", code: "config", message: "m", details: { at: 1 } } }, + { + kind: "failed", + hold: 9, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + retryIn: 60_000, + }, + // With its hop budget (audit A5): the first host's, and the last one passed on. + { kind: "retry", hold: 300, hops: 16 }, + { kind: "retry", hold: 300, hops: 1 }, + { kind: "loading", hold: 300 }, + { kind: "result", call: 3, value: '{"n":1}' }, + { kind: "result", call: 3 }, + { kind: "error", call: 3, error: { kind: "plugin", code: "c", message: "m", details: [1] } }, + { kind: "gone", hold: 0, error: { kind: "chord", error: { code: "released", message: "m" } } }, + { kind: "ack" }, + { kind: "ack", origin: 1 }, + { kind: "closed", origin: 300 }, + { + kind: "hold", + origin: 2, + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + }, + { kind: "release", origin: 2, hold: 3 }, + { kind: "call", origin: 2, call: 1, hold: 3, method: 0, args: "[1]" }, + { kind: "abort", origin: 2, call: 1 }, + { kind: "retry", origin: 2, hold: 3, hops: 16 }, + { kind: "bound", origin: 2, holds: [{ hold: 3, variant: 0, members: [], failed: [] }] }, + { kind: "state", origin: 2, holds: [] }, + { + kind: "failed", + origin: 2, + hold: 3, + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { kind: "loading", origin: 2, hold: 3 }, + { kind: "result", origin: 2, call: 1 }, + { kind: "error", origin: 2, call: 1, error: { kind: "plugin", code: "c", message: "m" } }, + { kind: "gone", origin: 2, hold: 3, error: { kind: "chord", error: { code: "released", message: "m" } } }, + { + kind: "hold", + hold: 3, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + chain: [incarnation, incarnation], + limit: 65_536, + }, + { + kind: "hold", + hold: 4, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + limit: 0, + }, + { kind: "revoke", revocation: { grant: "g1" } }, + { kind: "ping" }, + { kind: "pong" }, +]; + +/** A decoded message with every part as its text, to compare with one built from text parts. */ +function plain(message: Message): Message { + switch (message.kind) { + case "hold": + return { ...message, params: partText(message.params) }; + case "call": + return message.args === undefined ? message : { ...message, args: partText(message.args) }; + case "result": + return message.value === undefined ? message : { ...message, value: partText(message.value) }; + case "bound": + return { + ...message, + holds: message.holds.map((entry) => ({ + ...entry, + members: entry.members.map(({ member, value }) => ({ member, value: partText(value) })), + })), + }; + case "state": + return { + ...message, + holds: message.holds.map((entry) => ({ + ...entry, + members: entry.members.map(({ member, ops }) => ({ member, ops: partText(ops) })), + })), + }; + default: + return message; + } +} + +forEachCodec(() => { + describe("messages", () => { + it.each(EVERY_KIND)("round-trip %j and pass the message schema", (message) => { + const codec = testCodec(); + const decoded = checkMessage(codec.decode(codec.encode(message)), context); + expect(plain(decoded)).toEqual(message); + }); + + it("round-trip deflated parts in every part's place, still deflated (a relay forwards them so)", () => { + const codec = testCodec(); + const deflated = () => new DeflatedPart(new Uint8Array([0, 1, 2, 255])); + const messages: Message[] = [ + { + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: deflated(), + address: server, + }, + { kind: "call", call: 0, hold: 0, method: 0, args: deflated() }, + { kind: "result", call: 0, value: deflated(), segments: [new Uint8Array([7])] }, + { + kind: "bound", + holds: [{ hold: 0, variant: 0, members: [{ member: 0, value: deflated() }], failed: [] }], + }, + { kind: "state", holds: [{ hold: 0, members: [{ member: 0, ops: deflated() }], failed: [], retired: [] }] }, + ]; + for (const message of messages) { + const decoded = checkMessage(codec.decode(codec.encode(message)), context); + expect(decoded).toEqual(message); + } + }); + + it("round-trip parts received as bytes (a relay forwards them as they came)", () => { + const codec = testCodec(); + const message: Message = { kind: "result", call: 0, value: new TextEncoder().encode('{"n":1}') }; + expect(plain(codec.decode(codec.encode(message)))).toEqual({ kind: "result", call: 0, value: '{"n":1}' }); + }); + + it("carry byte segments beside a payload, as views of the message", () => { + const codec = testCodec(); + const data = new Uint8Array(1024).fill(7); + const thumbs = [new Uint8Array([1, 2]), new Uint8Array([3])]; + const { json, segments } = payloadPart({ name: "a", data, thumbs }, true); + const message: Message = { kind: "call", call: 0, hold: 0, method: 0, args: json, segments }; + const bytes = codec.encode(message); + const decoded = codec.decode(bytes) as Extract; + expect(decoded.segments).toHaveLength(3); + expect(decoded.segments![0]!.buffer).toBe(bytes.buffer); + const args = parsePayload(decoded.args as Uint8Array, decoded.segments) as { + data: Uint8Array; + thumbs: Uint8Array[]; + }; + expect(args.data).toEqual(data); + expect(args.data.buffer).toBe(bytes.buffer); + expect(args.thumbs).toEqual(thumbs); + // The segments travel raw: the message is barely larger than the bytes it carries. + expect(bytes.byteLength).toBeLessThan(1024 + 3 + 100); + }); + + it("refuse an empty message, an unknown kind and trailing bytes", () => { + const codec = testCodec(); + const release = codec.encode({ kind: "release", hold: 1 }); + const trailing = new Uint8Array([...release, ...(codec === binaryCodec ? [0] : [0x20, 0x31])]); + expect(() => codec.decode(new Uint8Array())).toThrow(); + expect(() => + codec.decode(codec === binaryCodec ? new Uint8Array([99]) : new TextEncoder().encode('["?"]')), + ).toThrow("unknown kind"); + expect(() => codec.decode(trailing)).toThrow(); + }); + + it("refuse an origin on a kind without one, an origin 0, and a closed without one (the message schema)", () => { + const codec = testCodec(); + const check = (message: unknown) => () => + checkMessage(codec.decode(codec.encode(message as Message)), context); + expect(check({ kind: "ping", origin: 1 })).toThrow("message schema"); + expect(check({ kind: "ack", origin: 0 })).toThrow("message schema"); + const closed = codec === binaryCodec ? new Uint8Array([14]) : new TextEncoder().encode('["C"]'); + expect(() => checkMessage(codec.decode(closed), context)).toThrow("message schema"); + }); + + it("keep a leading U+FEFF in strings and parts (it is data, not a byte order mark)", () => { + const codec = testCodec(); + const message: Message = { + kind: "hold", + hold: 0, + type: "\uFEFFthing", + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart({ "\uFEFFkey": "\uFEFFvalue" }), + address: server, + }; + const decoded = codec.decode(codec.encode(message)) as Extract; + expect(plain(decoded)).toEqual(message); + expect(parsePart(decoded.params as string | Uint8Array)).toEqual({ "\uFEFFkey": "\uFEFFvalue" }); + const result: Message = { kind: "result", call: 0, value: jsonPart("\uFEFF") }; + expect(plain(codec.decode(codec.encode(result)))).toEqual(result); + }); + + it("leave the envelope to the message schema: a misshapen error passes the codec and fails the schema", () => { + const codec = testCodec(); + const decoded = codec.decode(codec.encode({ kind: "error", call: 0, error: { kind: "nope" } } as never)); + expect(() => checkMessage(decoded, context)).toThrow("message: fails the message schema"); + expect(() => checkMessage({ kind: "release", hold: -1 }, context)).toThrow( + "message: fails the message schema", + ); + }); + }); +}); + +describe("the binary codec", () => { + it("writes ids as varints, the incarnation as 16 and the fingerprint as 8 raw bytes, no outer length", () => { + expect(binaryCodec.encode({ kind: "hello", protocol: 1, incarnation, calls: 1 })).toHaveLength(1 + 1 + 16 + 1); + expect([...binaryCodec.encode({ kind: "release", hold: 300 })]).toEqual([2, 0xac, 0x02]); + const hold = binaryCodec.encode({ + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + }); + // kind, id, type, version, fingerprint, params, the address's kind and params, the optional fields' flags + expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 7 + 3 + 1); + expect([...binaryCodec.encode({ kind: "ack" })]).toEqual([13]); + // An origin: the kind byte's high bit, then the origin as a varint. + expect([...binaryCodec.encode({ kind: "ack", origin: 300 })]).toEqual([13 | 0x80, 0xac, 0x02]); + expect([...binaryCodec.encode({ kind: "closed", origin: 1 })]).toEqual([14 | 0x80, 1]); + }); + + it("writes a failure's cause as one byte, 0 transient, 1 broken, 2 waiting (dependency model §2); refuses another", () => { + const error = { kind: "chord", error: { code: "unavailable", message: "m" } } as const; + const failed = (kind: "transient" | "broken" | "waiting"): Message => ({ + kind: "failed", + hold: 0, + cause: { kind, error }, + }); + expect( + ["transient", "broken", "waiting"].map((kind) => binaryCodec.encode(failed(kind as "waiting"))[2]), + ).toEqual([0, 1, 2]); + const unknown = binaryCodec.encode(failed("waiting")); + unknown[2] = 3; + expect(() => binaryCodec.decode(unknown)).toThrow("an unknown failure kind"); + }); + + it("costs a state entry without retirements no byte: the failure count is doubled, plus one when they follow", () => { + const state = (retired: number[]): Message => ({ + kind: "state", + holds: [{ hold: 0, members: [], failed: [], retired }], + }); + // kind, entry count, hold, member count, failures (0 doubled) + expect([...binaryCodec.encode(state([]))]).toEqual([6, 1, 0, 0, 0]); + // the low bit set, then the retired count and member + expect([...binaryCodec.encode(state([3]))]).toEqual([6, 1, 0, 0, 1, 1, 3]); + const first = binaryCodec.decode(binaryCodec.encode(state([]))); + const second = binaryCodec.decode(binaryCodec.encode(state([]))); + const retired = (message: Message) => (message as Extract).holds[0]!.retired; + // No allocation per message: decoded entries without retirements share one empty list. + expect(retired(first)).toBe(retired(second)); + }); + + it("prefixes a part with its length doubled, plus one when it is deflated", () => { + const plain = binaryCodec.encode({ kind: "result", call: 0, value: "12" }); + const deflated = binaryCodec.encode({ kind: "result", call: 0, value: new DeflatedPart(new Uint8Array([7, 8])) }); + // kind, call, payload flags, prefix, bytes + expect([...plain]).toEqual([8, 0, 1, 4, 0x31, 0x32]); + expect([...deflated]).toEqual([8, 0, 1, 5, 7, 8]); + }); + + it("decodes parts as views of the message, never parsing them: a malformed part decodes", () => { + const part = new TextEncoder().encode("{not json"); + const bytes = binaryCodec.encode({ kind: "result", call: 0, value: part }); + const decoded = binaryCodec.decode(bytes) as Extract; + const value = decoded.value as Uint8Array; + expect(value.buffer).toBe(bytes.buffer); + expect(() => parsePart(value)).toThrow(); + }); + + it("refuses varints past a safe integer and lengths past the message", () => { + expect(() => binaryCodec.decode(new Uint8Array([2, ...new Array(8).fill(0xff), 0x7f]))).toThrow( + "a varint does not fit", + ); + expect(() => binaryCodec.decode(new Uint8Array([1, 0, 9, 0x61]))).toThrow("the message ends early"); + }); + + it("refuses a varint of more than 8 bytes, which would read as no number", () => { + const state = new Uint8Array([6, ...new Array(150).fill(0x80), 0]); + expect(() => binaryCodec.decode(state)).toThrow("a varint does not fit"); + }); + + it("refuses unknown hold flags", () => { + const hold = binaryCodec.encode({ + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + }); + const forged = hold.slice(); + forged[forged.length - 1] = 32; + expect(() => binaryCodec.decode(forged)).toThrow("unknown hold flags"); + }); + + it("returns a right-sized message: a large one keeps no larger backing store", () => { + const value = jsonPart("x".repeat(500_000)); + // Small fields after the large value: they must not double the buffer the message keeps. + const members = [ + { member: 0, value }, + { member: 1, value: "1" }, + ]; + const bytes = binaryCodec.encode({ kind: "bound", holds: [{ hold: 0, variant: 0, members, failed: [] }] }); + expect(bytes.buffer.byteLength - bytes.byteLength).toBeLessThanOrEqual(4096); + const small = binaryCodec.encode({ kind: "release", hold: 1 }); + expect(small.buffer.byteLength).toBe(small.byteLength); + // A message of about two thirds of the starting buffer keeps no spare room either. + const medium = binaryCodec.encode({ kind: "result", call: 0, value: jsonPart("x".repeat(160)) }); + expect(medium.buffer.byteLength).toBe(medium.byteLength); + }); + + it("refuses invalid UTF-8 in strings", () => { + expect(() => binaryCodec.decode(new Uint8Array([1, 0, 1, 0xff]))).toThrow(); + }); + + it("writes retry as its kind, hold and hops, loading as its kind and hold, and failed's retryIn as one more than it (0: absent)", () => { + expect([...binaryCodec.encode({ kind: "retry", hold: 300, hops: 16 })]).toEqual([11, 0xac, 0x02, 16]); + expect([...binaryCodec.encode({ kind: "loading", hold: 1 })]).toEqual([12, 1]); + const cause = { kind: "broken", error: { kind: "chord", error: { code: "released", message: "" } } } as const; + const absent = binaryCodec.encode({ kind: "failed", hold: 0, cause }); + const present = binaryCodec.encode({ kind: "failed", hold: 0, cause, retryIn: 0 }); + expect(absent.at(-1)).toBe(0); + expect(present.at(-1)).toBe(1); + }); +}); + +describe("the text codec", () => { + const text = (message: Message) => new TextDecoder().decode(textCodec.encode(message)); + + it("writes one JSON tuple per message with short tags and positional fields, parts spliced in", () => { + expect(text({ kind: "release", hold: 3 })).toBe('["R",3]'); + expect(text({ kind: "call", call: 1, hold: 0, method: 2 })).toBe('["c",1,0,2]'); + const members = [{ member: 0, value: "5" }]; + expect(text({ kind: "bound", holds: [{ hold: 0, variant: 1, members, failed: [] }] })).toBe( + '["b",[[0,1,[[0,[0,5]]]]]]', + ); + expect(text({ kind: "retry", hold: 2, hops: 16 })).toBe('["T",2,16]'); + expect(text({ kind: "loading", hold: 2 })).toBe('["l",2]'); + const cause = { + kind: "transient", + error: { kind: "chord", error: { code: "unavailable", message: "m" } }, + } as const; + expect(text({ kind: "failed", hold: 2, cause })).toBe( + '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}}]', + ); + expect(text({ kind: "failed", hold: 2, cause, retryIn: 750 })).toBe( + '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}},750]', + ); + expect(text({ kind: "failed", hold: 2, cause: { ...cause, kind: "waiting" } })).toBe( + '["f",2,"w",{"kind":"chord","error":{"code":"unavailable","message":"m"}}]', + ); + expect(() => textCodec.decode(new TextEncoder().encode('["f",2,"x",{}]'))).toThrow("an unknown failure kind"); + expect(text({ kind: "ack" })).toBe('["a"]'); + // An origin wraps the message's own tuple. + expect(text({ kind: "release", origin: 4, hold: 3 })).toBe('["@",4,["R",3]]'); + expect(text({ kind: "closed", origin: 4 })).toBe('["@",4,["C"]]'); + }); + + it("tags every part with its codec, [0, json] or [1, base64] for a deflated one, so no value is reserved", () => { + const message: Message = { kind: "result", call: 1, value: new DeflatedPart(new Uint8Array([1, 2, 3])) }; + expect(text(message)).toBe('["r",1,[1,"AQID"]]'); + expect(textCodec.decode(textCodec.encode(message))).toEqual(message); + // A value of any shape is data, the one-key `$deflated` object the first builder reserved included. + const shaped: Message = { kind: "result", call: 1, value: '{"$deflated":"AQID"}' }; + expect(text(shaped)).toBe('["r",1,[0,{"$deflated":"AQID"}]]'); + expect(textCodec.decode(textCodec.encode(shaped))).toEqual(shaped); + expect(text({ kind: "revoke", revocation: { grant: "g" } })).toBe('["V",{"grant":"g"}]'); + }); + + it("refuses a part that is no [codec, payload] pair, or of an unknown codec", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["r",1,{"n":1}]')).toThrow("no [codec, payload] pair"); + expect(() => decode('["r",1,[2,"AQID"]]')).toThrow("unknown codec"); + expect(() => decode('["r",1,[1,5]]')).toThrow("unknown codec"); + }); + + it("carries a binary link's deflated part over a text hop and back unchanged (a relay between codecs)", () => { + const part = new DeflatedPart(new Uint8Array([9, 8, 7])); + const message: Message = { + kind: "state", + holds: [{ hold: 0, members: [{ member: 0, ops: part }], failed: [], retired: [] }], + }; + const overText = textCodec.decode(textCodec.encode(binaryCodec.decode(binaryCodec.encode(message)))); + expect(binaryCodec.decode(binaryCodec.encode(overText))).toEqual(message); + }); + + it("leaves out a hold's absent optional fields at the end, null in the middle", () => { + const hold = { + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + } as const; + const head = '["H",0,"t",1,"0123456789abcdef",[0,{}],["server",{}]'; + expect(text(hold)).toBe(`${head}]`); + expect(text({ ...hold, incarnation })).toBe(`${head},null,"${incarnation}"]`); + expect(text({ ...hold, limit: 5 })).toBe(`${head},null,null,null,5]`); + }); + + it("puts byte segments after the tuple and a newline, their lengths the tuple's last field", () => { + const bytes = textCodec.encode({ + kind: "result", + call: 0, + value: '{"$bytes":0}', + segments: [new Uint8Array([1, 2])], + }); + expect([...bytes]).toEqual([...new TextEncoder().encode('["r",0,[0,{"$bytes":0}],[2]]\n'), 1, 2]); + }); + + it("refuses a tuple of the wrong arity and segments without a payload carrying them", () => { + expect(() => textCodec.decode(new TextEncoder().encode('["R",1,2]'))).toThrow('a "R" message with 2 fields'); + expect(() => textCodec.decode(new TextEncoder().encode('["s",[[0,[],[],[],9]]]'))).toThrow( + "a state entry with 5 fields", + ); + expect(() => textCodec.decode(new TextEncoder().encode('["s",[[0]]]'))).toThrow("a state entry with 1 fields"); + expect(() => textCodec.decode(new Uint8Array([...new TextEncoder().encode('["R",1]\n'), 1]))).toThrow( + "byte segments without a payload", + ); + expect(() => textCodec.decode(new TextEncoder().encode('["@",1,["@",2,["a"]]]'))).toThrow("wraps one message"); + expect(() => textCodec.decode(new TextEncoder().encode('["@",1]'))).toThrow("wraps one message"); + }); + + it("refuses a hold without its address: every hold names one", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}]]')).toThrow('a "H" message with 5 fields'); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],null]')).toThrow("expected a list"); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],["server"]]')).toThrow("an address is a pair"); + const hold = { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }; + expect(() => checkMessage(hold, context)).toThrow("message schema"); + }); + + it("refuses a retry without its hop budget, or with one outside 1..16 (audit A5)", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["T",2]')).toThrow('a "T" message with 1 fields'); + for (const hops of [0, 17]) { + expect(() => checkMessage({ kind: "retry", hold: 2, hops }, context)).toThrow("message schema"); + } + }); +}); + +describe("describeMessage", () => { + it("prints a message in the text tuple form", () => { + expect(describeMessage({ kind: "abort", call: 7 })).toBe('["x",7]'); + expect(describeMessage({ kind: "result", call: 0, value: '{"$bytes":0}', segments: [new Uint8Array(3)] })).toBe( + '["r",0,[0,{"$bytes":0}],[3]]', + ); + }); +}); + +describe("a codec mismatch", () => { + it("decodes the other codec's hello as no message of its own", () => { + const hello: Message = { kind: "hello", protocol: 1, incarnation, calls: 256 }; + expect(() => binaryCodec.decode(textCodec.encode(hello))).toThrow(); + expect(() => textCodec.decode(binaryCodec.encode(hello))).toThrow(); + }); +}); + +describe("the binary codec's strings", () => { + it("survive the buffer growing around them, at every offset near the initial size", () => { + for (let length = 0; length < 400; length++) { + // A long value of three-byte characters (filling what it reserved), then a short one that lands at every offset around the buffer's first growth. + const members = [ + { member: 0, value: jsonPart("€".repeat(length)) }, + { member: 1, value: "1" }, + ]; + const message: Message = { kind: "bound", holds: [{ hold: 1, variant: 0, members, failed: [] }] }; + expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); + } + }); +}); + +describe("the binary codec's long strings", () => { + const texts = [ + ["ASCII", "x".repeat(10_000)], + ["three-byte characters", "€".repeat(10_000)], + ["surrogate pairs", "😀".repeat(5_000)], + ["mixed, at the in-place threshold", `${"a".repeat(4_093)}😀€`], + ["a leading U+FEFF", `\uFEFF${"y".repeat(5_000)}`], + ]; + + it.each(texts)("round-trip %s, encoded in place behind a padded length", (_, text) => { + const message: Message = { kind: "result", call: 0, value: jsonPart(text) }; + const bytes = binaryCodec.encode(message); + expect(plain(binaryCodec.decode(bytes))).toEqual(message); + expect(bytes.buffer.byteLength - bytes.byteLength).toBeLessThanOrEqual(4096); + }); +}); + +describe("the binary codec's short strings", () => { + it.each([0, 1, 41, 42, 43])("round-trip at %i three-byte characters, around the one-byte length", (length) => { + const type = "€".repeat(length); + const message: Message = { + kind: "hold", + hold: 0, + type, + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart(type), + address: server, + }; + expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); + }); +}); diff --git a/packages/chord/test/next/link/part.test.ts b/packages/chord/test/next/link/part.test.ts new file mode 100644 index 000000000..83d76f54e --- /dev/null +++ b/packages/chord/test/next/link/part.test.ts @@ -0,0 +1,196 @@ +// Tests of parts on links: the limits per part (in UTF-8 bytes), compression at the origin, opening a part where it is +// owned, and the asynchronous inflate with its limit, its corrupt outcome and its stop when the link closes. + +import { describe, expect, it } from "vitest"; +import { + COMPRESS_ABOVE, + type Compression, + DeflatedPart, + inflateAsync, + linkLimits, + openPart, + originPart, + oversized, +} from "../../../src/next/link/part.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { metered } from "../support.ts"; + +const UTF8 = new TextEncoder(); + +const NONE = { part: Number.POSITIVE_INFINITY, decoded: Number.POSITIVE_INFINITY }; + +/** A link that never closes. */ +const NEVER = new AbortController().signal; + +/** A JSON text of `length` characters that deflates well. */ +const json = (length: number) => JSON.stringify("x".repeat(length - 2)); + +/** What `fn` threw. */ +function thrown(fn: () => unknown): unknown { + try { + fn(); + } catch (error) { + return error; + } + throw new Error("expected a throw"); +} + +describe("limits per part", () => { + it("default to none on trusted links and to 16 MiB on the wire, 64 MiB decoded on untrusted ones", () => { + expect(linkLimits(true)).toEqual(NONE); + expect(linkLimits(false)).toEqual({ part: 16 * 1024 * 1024, decoded: 64 * 1024 * 1024 }); + expect(linkLimits(false, { part: 10 })).toEqual({ part: 10, decoded: 64 * 1024 * 1024 }); + expect(linkLimits(true, { decoded: 5 })).toEqual({ part: Number.POSITIVE_INFINITY, decoded: 5 }); + }); + + it("check a part's size on the wire with one comparison, nothing parsed", () => { + expect(oversized("[1,2]", 5)).toBeUndefined(); + expect(oversized("[1,2,3]", 5)?.toJSON()).toEqual({ + code: "too_large", + reason: "part", + message: "a part of 7 bytes is over the wire limit of 5 bytes", + }); + expect(oversized(new Uint8Array(6), 5)?.code).toBe("too_large"); + expect(oversized(new DeflatedPart(new Uint8Array(5)), 5)).toBeUndefined(); + }); + + it("count text in UTF-8 bytes, not UTF-16 units (a text link bounds what a binary link does)", () => { + const text = JSON.stringify("界".repeat(6000)); + expect(UTF8.encode(text).byteLength).toBe(18_002); + expect(oversized(text, 18_002)).toBeUndefined(); + expect(oversized(text, 18_001)?.message).toBe("a part of 18002 bytes is over the wire limit of 18001 bytes"); + // Two-byte characters and surrogate pairs (four bytes for two units). + expect(oversized('"é😀"', 8)).toBeUndefined(); + expect(oversized('"é😀"', 7)?.code).toBe("too_large"); + }); +}); + +describe("compression at the origin", () => { + it("deflates a JSON part above 16 KiB when the host compresses, and counts it", async () => { + const { context, counts } = await metered(); + const small = json(COMPRESS_ABOVE / 3); + expect(originPart(small, nodeCompression, context)).toBe(small); + const large = json(COMPRESS_ABOVE + 1); + const part = originPart(large, nodeCompression, context); + expect(part).toBeInstanceOf(DeflatedPart); + expect((part as DeflatedPart).bytes.byteLength).toBeLessThan(200); + expect(counts()["chord.link.deflates"]).toBe(1); + }); + + it("measures the threshold in UTF-8 bytes: a text of few units but many bytes is compressed", async () => { + const { context } = await metered(); + const text = JSON.stringify("界".repeat(6000)); + expect(text.length).toBeLessThan(COMPRESS_ABOVE); + expect(originPart(text, nodeCompression, context)).toBeInstanceOf(DeflatedPart); + // Within the threshold in bytes: its bytes, encoded once (the codec copies them). + const within = json(COMPRESS_ABOVE); + expect(originPart(within, nodeCompression, context)).toEqual(UTF8.encode(within)); + }); + + it("sends a part as it is without a compressor, and its bytes when deflating does not make it smaller", async () => { + const { context } = await metered(); + const large = json(COMPRESS_ABOVE * 2); + expect(originPart(large, undefined, context)).toBe(large); + const useless: Compression = { + deflate: (bytes) => bytes, + inflate: (bytes) => ({ kind: "bytes", bytes }), + }; + expect(originPart(large, useless, context)).toEqual(UTF8.encode(large)); + }); +}); + +describe("opening a part", () => { + it("returns a plain part within its limits; past either it throws too_large/part", async () => { + const { context } = await metered(); + expect(openPart("[1]", NONE, undefined, context)).toBe("[1]"); + expect(() => openPart("[1,2]", { part: 4, decoded: 100 }, undefined, context)).toThrow( + "a part of 5 bytes is over the wire limit of 4 bytes", + ); + // Not deflated: its decoded size is its size. + expect(() => openPart(UTF8.encode("[1,2]"), { part: 100, decoded: 4 }, undefined, context)).toThrow( + "a part of 5 bytes is over the decoded limit of 4 bytes", + ); + }); + + it("inflates a deflated part synchronously within the decoded limit, and counts it", async () => { + const { context, counts } = await metered(); + const text = json(100_000); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(text))); + const opened = openPart(part, { part: 1000, decoded: 100_000 }, nodeCompression, context); + expect(new TextDecoder().decode(opened as Uint8Array)).toBe(text); + expect(counts()["chord.link.inflates"]).toBe(1); + }); + + it("refuses a compression bomb: a small part inflating past the decoded limit", async () => { + const { context } = await metered(); + const bomb = new DeflatedPart(nodeCompression.deflate(new Uint8Array(10 * 1024 * 1024))); + expect(bomb.bytes.byteLength).toBeLessThan(50_000); + const error = thrown(() => openPart(bomb, { part: 1_000_000, decoded: 1024 * 1024 }, nodeCompression, context)); + expect(error).toMatchObject({ code: "too_large", reason: "part" }); + }); + + it("checks the wire size before inflating anything", async () => { + const { context, counts } = await metered(); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(100_000)))); + const error = thrown(() => openPart(part, { part: 10, decoded: 1_000_000 }, nodeCompression, context)); + expect(error).toMatchObject({ code: "too_large", message: expect.stringContaining("wire limit") }); + expect(counts()["chord.link.inflates"]).toBeUndefined(); + }); + + it("opens what an asynchronous inflate left on the part", async () => { + const { context } = await metered(); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(1000)))); + await inflateAsync(part, 2000, NEVER, context); + expect(openPart(part, { part: 1000, decoded: 2000 }, undefined, context)).toHaveLength(1000); + const bomb = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(1000)))); + await inflateAsync(bomb, 999, NEVER, context); + expect(() => openPart(bomb, { part: 1000, decoded: 999 }, undefined, context)).toThrow("decoded limit of 999"); + }); + + it("throws a plain error for a deflated part nothing inflated, or no raw deflate stream (a protocol error)", async () => { + const { context } = await metered(); + const part = new DeflatedPart(new Uint8Array([255, 255, 255])); + expect(() => openPart(part, NONE, undefined, context)).toThrow("a deflated part this host did not inflate"); + expect(() => openPart(part, NONE, nodeCompression, context)).toThrow("no raw deflate stream"); + expect(thrown(() => openPart(part, NONE, nodeCompression, context))).not.toMatchObject({ code: "too_large" }); + }); +}); + +describe("the asynchronous inflate", () => { + it("inflates with DecompressionStream, counting output bytes", async () => { + const { context, counts } = await metered(); + const text = json(200_000); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(text))); + await inflateAsync(part, Number.POSITIVE_INFINITY, NEVER, context); + expect(part.inflated?.kind).toBe("bytes"); + expect(new TextDecoder().decode(openPart(part, NONE, undefined, context) as Uint8Array)).toBe(text); + expect(counts()["chord.link.inflates"]).toBe(1); + }); + + it("stops past the limit: a bomb leaves too_large, never the whole output; a limit of 0 too", async () => { + const { context } = await metered(); + const bomb = new DeflatedPart(nodeCompression.deflate(new Uint8Array(64 * 1024 * 1024))); + await inflateAsync(bomb, 1024 * 1024, NEVER, context); + expect(bomb.inflated).toEqual({ kind: "too_large" }); + const one = new DeflatedPart(nodeCompression.deflate(new Uint8Array(1))); + await inflateAsync(one, 0, NEVER, context); + expect(one.inflated).toEqual({ kind: "too_large" }); + }); + + it("leaves corrupt for data that is no raw deflate stream", async () => { + const { context } = await metered(); + const part = new DeflatedPart(new Uint8Array([255, 255, 255])); + await inflateAsync(part, 100, NEVER, context); + expect(part.inflated?.kind).toBe("corrupt"); + }); + + it("stops at once when its link closes (the signal), leaving no outcome", async () => { + const { context } = await metered(); + const closing = new AbortController(); + const part = new DeflatedPart(nodeCompression.deflate(new Uint8Array(64 * 1024 * 1024))); + const inflating = inflateAsync(part, Number.POSITIVE_INFINITY, closing.signal, context); + closing.abort(); + await inflating; + expect(part.inflated).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next/link/values.test.ts b/packages/chord/test/next/link/values.test.ts new file mode 100644 index 000000000..84656b516 --- /dev/null +++ b/packages/chord/test/next/link/values.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { jsonPart, parsePart, parsePayload, payloadPart, resolveSegments } from "../../../src/next/link/values.ts"; + +describe("parts", () => { + it("are JSON text at their origin, parsed from text or from UTF-8 bytes where they are owned", () => { + expect(jsonPart({ a: [1, "x"] })).toBe('{"a":[1,"x"]}'); + expect(jsonPart(undefined)).toBe("null"); + expect(parsePart('{"a":1}')).toEqual({ a: 1 }); + expect(parsePart(new TextEncoder().encode('{"a":"\u00e9"}'))).toEqual({ a: "\u00e9" }); + }); + + it("that are no strict UTF-8 JSON throw (a protocol error where they are parsed)", () => { + expect(() => parsePart("{")).toThrow(); + expect(() => parsePart(new Uint8Array([0x22, 0xff, 0x22]))).toThrow(); + }); +}); + +describe("payloads", () => { + it("marked as carrying bytes write each byte leaf as a segment referenced by {$bytes: i}", () => { + const data = new Uint8Array(1024).fill(7); + const thumbs = [new Uint8Array([1, 2]), new Uint8Array([3])]; + const { json, segments } = payloadPart({ name: "chart.png", data, thumbs }, true); + expect(json).toBe('{"name":"chart.png","data":{"$bytes":0},"thumbs":[{"$bytes":1},{"$bytes":2}]}'); + expect(segments).toEqual([data, ...thumbs]); + expect(segments![0]).toBe(data); + const read = parsePayload(json, segments) as { data: Uint8Array; thumbs: Uint8Array[] }; + expect(read.data).toBe(data); + expect(read.thumbs).toEqual(thumbs); + }); + + it("marked as carrying bytes refuse the reserved key $bytes with invalid/args", () => { + let thrown: unknown; + try { + payloadPart({ meta: { $bytes: 0 }, data: new Uint8Array(1) }, true); + } catch (error) { + thrown = error; + } + expect(isChordError(thrown, "invalid") && thrown.reason).toBe("args"); + }); + + it("not marked are plain JSON, $bytes included; undefined is null", () => { + expect(payloadPart({ $bytes: 0 }, false)).toEqual({ json: '{"$bytes":0}', segments: undefined }); + expect(payloadPart(undefined, false).json).toBe("null"); + expect(parsePayload('{"$bytes":0}', undefined)).toEqual({ $bytes: 0 }); + }); + + it.each([ + ["an index past the segments", { $bytes: 1 }], + ["a non-integer index", { $bytes: "0" }], + ["other keys next to $bytes", { $bytes: 0, x: 1 }], + ])("refuse a malformed reference: %s", (_, value) => { + expect(() => resolveSegments({ a: [value] }, [new Uint8Array(1)])).toThrow( + "payload: a malformed byte segment reference", + ); + }); + + it("define a resolved key __proto__ as an own property", () => { + const parsed = JSON.parse('{"__proto__":{"$bytes":0}}'); + const resolved = resolveSegments(parsed, [new Uint8Array(1)]) as object; + expect(Object.getPrototypeOf(resolved)).toBe(Object.prototype); + expect(Object.getOwnPropertyDescriptor(resolved, "__proto__")?.value).toEqual(new Uint8Array(1)); + }); +}); diff --git a/packages/chord/test/next/link/websocket.test.ts b/packages/chord/test/next/link/websocket.test.ts new file mode 100644 index 000000000..c67212b48 --- /dev/null +++ b/packages/chord/test/next/link/websocket.test.ts @@ -0,0 +1,159 @@ +import type { AddressInfo } from "node:net"; +import { describe, expect, it } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type WebSocketLike, webSocketLink } from "../../../src/next/link/index.ts"; +import { Notes, Server, serveNotes, type TestPrincipal, transported } from "../support.ts"; + +/** A socket playing a WebSocket by hand. */ +class FakeSocket implements WebSocketLike { + readyState: number; + binaryType = "blob"; + readonly sent: Uint8Array[] = []; + closes = 0; + readonly #listeners = new Map void>>(); + + constructor(readyState: number) { + this.readyState = readyState; + } + + send(data: Uint8Array): void { + this.sent.push(data); + } + + close(): void { + this.closes++; + this.readyState = 3; + } + + addEventListener(type: string, listener: (event: { readonly data: unknown }) => void): void { + const listeners = this.#listeners.get(type) ?? new Set(); + listeners.add(listener); + this.#listeners.set(type, listeners); + } + + removeEventListener(type: string, listener: (event: { readonly data: unknown }) => void): void { + this.#listeners.get(type)?.delete(listener); + } + + emit(type: string, data?: unknown): void { + if (type === "open") this.readyState = 1; + for (const listener of [...(this.#listeners.get(type) ?? [])]) listener({ data }); + } + + get listeners(): number { + return [...this.#listeners.values()].reduce((count, listeners) => count + listeners.size, 0); + } +} + +describe("WebSocket links over a fake socket", () => { + it("buffer messages sent while connecting and send them on open, as binary frames (race 29)", () => { + const socket = new FakeSocket(0); + const link = webSocketLink(socket); + expect(socket.binaryType).toBe("arraybuffer"); + link.send(new Uint8Array([1])); + link.send(new Uint8Array([2])); + expect(socket.sent).toEqual([]); + socket.emit("open"); + expect(socket.sent.map((message) => message[0])).toEqual([1, 2]); + const received: number[] = []; + link.onMessage((message) => received.push(message[0]!)); + socket.emit("message", new Uint8Array([7]).buffer); + expect(received).toEqual([7]); + }); + + it("give a closed link for a socket closing or closed already", () => { + for (const state of [2, 3]) { + let closed = false; + webSocketLink(new FakeSocket(state)).onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + } + }); + + it.each([ + ["close", (socket: FakeSocket) => socket.emit("close")], + ["error", (socket: FakeSocket) => socket.emit("error")], + ["a text frame", (socket: FakeSocket) => socket.emit("message", "text")], + ])("close once on %s, listeners removed", (_, fail) => { + const socket = new FakeSocket(1); + const link = webSocketLink(socket); + let closes = 0; + link.onClose(() => closes++); + fail(socket); + socket.emit("close"); + link.send(new Uint8Array([1])); + expect(closes).toBe(1); + expect(socket.sent).toEqual([]); + expect(socket.listeners).toBe(0); + }); +}); + +/** A `ws` server on a free loopback port whose connections a server host accepts, and a URL to it. */ +async function loopback(options: { readonly maxPayload?: number } = {}) { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const wss = new WebSocketServer({ port: 0, host: "127.0.0.1", maxPayload: options.maxPayload }); + await new Promise((resolve) => wss.once("listening", resolve)); + wss.on("connection", (socket) => server.accept(webSocketLink(socket), { principal: { kind: "owner" } }, context)); + const url = `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`; + return { server, wss, url }; +} + +describe("WebSocket links over a loopback ws server", () => { + it("survive closing a socket that still connects: ws's late error is handled", async () => { + const errors: unknown[] = []; + const record = (error: unknown) => errors.push(error); + process.on("uncaughtException", record); + try { + const socket = new WebSocket("ws://127.0.0.1:1"); + let closed = false; + const link = webSocketLink(socket); + link.onClose(() => { + closed = true; + }); + link.close(); + expect(closed).toBe(true); + await new Promise((resolve) => socket.once("close", () => resolve())); + expect(errors).toEqual([]); + expect(socket.listenerCount("error")).toBe(0); + } finally { + process.off("uncaughtException", record); + } + }); + + it("carry a live reference both ways; the client's hello waits for open", async () => { + const { server, wss, url } = await loopback(); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + tab.connect(webSocketLink(new WebSocket(url)), { principal: { kind: "server" } }, context); + const notes = await tab.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "over ws" }, context)).toBe(1); + await transported(); + expect(notes.notes.value).toEqual([{ text: "over ws" }]); + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + }); + + it("close the connection on a frame above maxPayload (1009): references go loading (race 28)", async () => { + const { server, wss, url } = await loopback({ maxPayload: 4096 }); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const socket = new WebSocket(url); + const code = new Promise((resolve) => socket.once("close", resolve)); + tab.connect(webSocketLink(socket), { principal: { kind: "server" } }, context); + const notes = await tab.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + const call = notes.echo({ data: new Uint8Array(10_000) }, context).catch((error: unknown) => error); + expect(await code).toBe(1009); + expect(await call).toMatchObject({ code: "unavailable" }); + expect(notes.state.value).toEqual({ kind: "loading" }); + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + }); +}); diff --git a/packages/chord/test/next/registries/entries.test.ts b/packages/chord/test/next/registries/entries.test.ts new file mode 100644 index 000000000..88bd61003 --- /dev/null +++ b/packages/chord/test/next/registries/entries.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + keyedEntries, + orderedEntries, + type RegistryConflict, + type RegistryTable, +} from "../../../src/next/registries/index.ts"; + +const table: RegistryTable<{ readonly routes?: Readonly>; readonly items?: readonly string[] }> = + [ + { plugin: "@a/one", version: 1, value: { routes: { "/x": "one", "/y": "one" }, items: ["late", "first"] } }, + { plugin: "@b/two", type: "b.store", version: 1, value: { routes: { "/x": "two" }, items: ["second"] } }, + { plugin: "@c/three", version: 1, value: {} }, + ]; + +describe("keyedEntries", () => { + it("maps every entry's keys with their plugin; a duplicate is reported and the first by plugin name keeps it", () => { + const conflicts: RegistryConflict[] = []; + const map = keyedEntries( + table, + (value) => value.routes, + (conflict) => conflicts.push(conflict), + ); + expect([...map]).toEqual([ + ["/x", { value: "one", plugin: "@a/one" }], + ["/y", { value: "one", plugin: "@a/one" }], + ]); + expect(conflicts).toEqual([{ message: '"/x" is contributed twice', plugins: ["@a/one", "@b/two"] }]); + }); +}); + +describe("orderedEntries", () => { + it("lists every entry's items in table order, with their plugin and type", () => { + expect(orderedEntries(table, (value) => value.items)).toEqual([ + { value: "late", plugin: "@a/one" }, + { value: "first", plugin: "@a/one" }, + { value: "second", plugin: "@b/two", type: "b.store" }, + ]); + }); + + it("sorts by `orderOf`, table order kept among equals", () => { + const order = (item: string) => (item === "first" ? 0 : 1); + expect(orderedEntries(table, (value) => value.items, order).map(({ value }) => value)).toEqual([ + "first", + "late", + "second", + ]); + }); +}); diff --git a/packages/chord/test/next/registries/record.test.ts b/packages/chord/test/next/registries/record.test.ts new file mode 100644 index 000000000..f9678df4b --- /dev/null +++ b/packages/chord/test/next/registries/record.test.ts @@ -0,0 +1,752 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineRegistry, + keyedEntries, + type Registry, + type RegistryTable, + type TableItem, +} from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; +import type { NativeWatchCall } from "../../../src/watch.ts"; +import { + delivered, + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + storeRestartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** A registry whose composition is its table as `plugin:type=value` strings. */ +const Names = defineRegistry("app.names", { version: 1 }); + +const listed = (table: RegistryTable): readonly string[] => + table.map(({ plugin, type, value }) => `${plugin}:${type ?? ""}=${value}`); + +/** A host-lifetime object contributing a writable it sets with `set`. */ +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter` contributing a writable of `R`'s contribution type, `initial` first, `set(n)` makes `of(n)`. */ +function setter(R: Registry, initial: C, of: (n: number) => C) { + return (host: Parameters[0][string]>[0]) => + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(R, value); + return { + set: async (n: number) => { + value.set(of(n)); + return null; + }, + }; + }); +} + +function kinds(errors: readonly HostError[]): string[] { + return errors.map((error) => error.kind); +} + +describe("the table", () => { + it("is sorted by plugin name, then type, a facet first: never load order", async () => { + const { host } = await loadedPlugins({ + "@z/late": (_, self) => self.contribute(Names, "z"), + "@a/early": (host, self) => { + self.contribute(Names, "a"); + setter(Names, "store", String)(host); + }, + "@m/provider": (host) => host.serve(Names, { compose: listed }), + }); + const value = await host.registry(Names, context); + expect(value.value).toEqual(["@a/early:=a", "@a/early:app.setter=store", "@z/late:=z"]); + await host.close(context); + }); + + it("adapts an older contribution once per value; a newer one, or an older one without an adapter, is left out and reported once", async () => { + const V3 = defineRegistry("app.versions", { version: 3 }); + const V2 = defineRegistry<{ old: string }, unknown>("app.versions", { version: 2 }); + const V1 = defineRegistry("app.versions", { version: 1 }); + const V4 = defineRegistry("app.versions", { version: 4 }); + const adapted: string[] = []; + const { host, errors } = await loadedPlugins({ + "@app/provider": (host) => + host.serve(V3, { + compose: listed, + accepts: { + 2: { + contribution: (old: { old: string }) => { + adapted.push(old.old); + return `adapted ${old.old}`; + }, + }, + }, + }), + "@app/one": (_, self) => self.contribute(V1, "one"), + "@app/two": setter(V2, { old: "first" }, (n) => ({ old: `${n}` })), + "@app/three": (_, self) => self.contribute(V3, "three"), + "@app/four": (_, self) => self.contribute(V4, "four"), + }); + const value = await host.registry(V3, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted first"]); + expect(errors).toEqual([ + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/four", version: 4 }, + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/one", version: 1 }, + ]); + const setters = await host.get(Setter, {}, context); + await setters.set(5, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted 5"]); + // Each value adapted once, though the table was built again; no refusal reported twice. + expect(adapted).toEqual(["first", "5"]); + expect(kinds(errors)).toEqual(["contribution_refused", "contribution_refused"]); + await host.close(context); + }); + + it("takes a contribution declared before its provider's setup into the first composition", async () => { + let composed: readonly string[] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Names, "first"), + "@app/provider": (host) => + host.serve(Names, { + compose: (table) => { + composed = listed(table); + return composed; + }, + }), + }); + expect(composed).toEqual(["@app/a:=first"]); + await host.close(context); + }); +}); + +describe("compose", () => { + it("composes every served registry once at the end of load's setups, an empty table too, before any object activates", async () => { + const { telemetry, counts } = await metered(); + const Other = defineRegistry("app.other", { version: 1 }); + const Reader = local("app.reader", { version: 1, params: {}, members: { seen: int() } }); + let seen: readonly string[] | undefined; + const { host } = await loadedPlugins( + { + "@app/provider": (host, self) => { + host.serve(Names, { compose: listed }); + host.serve(Other, { compose: (table) => table.length }); + self.contribute(Names, "a"); + host.serve(Reader, (_, self) => { + const names = self.registry(Names); + self.open( + async () => { + seen = names.value; + }, + () => {}, + ); + return { seen: self.derive([self.registry(Other)], ([n]) => n) }; + }); + }, + }, + { telemetry }, + ); + expect(seen).toEqual(["@app/provider:=a"]); + expect((await host.get(Reader, {}, context)).seen.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(2); + await host.close(context); + }); + + it("is lazy: a change marks the table, a read composes it once; no compose for an unchanged table", async () => { + const { telemetry, counts } = await metered(); + const Flags = defineRegistry("app.flags", { version: 1 }); + const Doc = defineState("app.flag", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Flagger = local("app.flagger", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins( + { + "@app/flags": (host) => { + host.serve(Flags, { compose: (table) => table.filter(({ value }) => value).length }); + host.serve(Flagger, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Flags, + self.derive([doc], ([d]) => d.n > 0), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const flags = await host.registry(Flags, context); + const flagger = await host.get(Flagger, {}, context); + expect(flags.value).toBe(1); + const base = counts()["chord.registry.composes"]!; + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // The derive changes, its value does not: the table stays, nothing composes. + await flagger.set(2, context); + await flagger.set(3, context); + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // Nobody reads it: nothing composes until a read. + await flagger.set(-1, context); + await flagger.set(-2, context); + expect(counts()["chord.registry.composes"]).toBe(base); + expect(flags.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("composes a watched registry in the batch of the commit that changed a contribution, once per batch", async () => { + const { telemetry, counts } = await metered(); + const Doc = defineState("app.names", { + schema: obj({ a: int(), b: int() }), + version: 1, + initial: () => ({ a: 0, b: 0 }), + }); + const Two = local("app.two", { version: 1, params: {}, members: { both: method(int()), sum: int() } }); + const Sum = defineRegistry<{ readonly a: number; readonly b: number }, number>("app.sum", { version: 1 }); + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { compose: (table) => table.reduce((n, { value }) => n + value.a + value.b, 0) }); + host.serve(Two, (_, self) => { + const doc = self.state(Doc); + self.contribute(Sum, doc); + return { + sum: self.derive([self.registry(Sum)], ([n]) => n), + both: async (n: number, ctx) => { + await self.tx(async (tx) => { + const d = await tx.draft(doc); + d.a = n; + d.b = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const two = await host.get(Two, {}, context); + const sums: number[] = []; + const watch = await two.sum.watch(context); + watch.start(async (call) => { + if (call.kind === "value") sums.push(call.value); + }); + const base = counts()["chord.registry.composes"]!; + await two.both(2, context); + await delivered(); + // One commit, both fields: one compose in its batch, one frame of the derive over it. + expect(sums).toEqual([4]); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("reports a conflict once per episode", async () => { + const Routes = defineRegistry>, ReadonlyMap>>( + "app.routes", + { version: 1 }, + ); + const { host, errors } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Routes, { "/x": "a" }), + "@app/b": setter(Routes, { "/x": "b" } as Readonly>, (n) => ({ + "/x": "b", + [`/${n}`]: "b", + })), + "@app/http": (host) => + host.serve(Routes, { compose: (table, { report }) => keyedEntries(table, (c) => c, report) }), + }); + const routes = await host.registry(Routes, context); + expect(routes.value.get("/x")).toEqual({ value: "a", plugin: "@app/a" }); + const setters = await host.get(Setter, {}, context); + await setters.set(1, context); + void routes.value; + await setters.set(2, context); + void routes.value; + expect(errors).toEqual([ + { + kind: "registry_conflict", + registry: "app.routes", + message: '"/x" is contributed twice', + plugins: ["@app/a", "@app/b"], + }, + ]); + await host.close(context); + }); + + it("a throwing compose: reported once, a failed call, `.value` keeps the last good composition; the next good one heals", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/count": (host) => { + host.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + setter(Count, 1, (n) => n)(host); + }, + }); + const count = await host.registry(Count, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + watch.start(async (call) => void calls.push(call)); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + await setters.set(-2, context); + await delivered(); + expect(count.value).toBe(1); + expect(errors).toEqual([{ kind: "registry_failed", registry: "app.count", error: new Error("negative: -1") }]); + await setters.set(1, context); + await delivered(); + // The heal is a value call, also when its value is the last good one. + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative: -1") }, + { kind: "value", value: 1 }, + ]); + await setters.set(3, context); + await delivered(); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3 }); + await host.close(context); + }); + + it("before any good composition, `.value` throws the error and `host.registry` rejects with it at its readyTimeout", async () => { + const Broken = defineRegistry("app.broken", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/broken": (host) => + host.serve(Broken, { + compose: () => { + throw new Error("bug"); + }, + }), + }); + await expect(host.registry(Broken, context, { readyTimeout: 5 })).rejects.toThrow("bug"); + expect(kinds(errors)).toEqual(["registry_failed"]); + await host.close(context); + }); +}); + +describe("contributions", () => { + it("a watchable contribution that fails is left out and reported once, and is back once it heals", async () => { + const Doc = defineState("app.n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Failing = local("app.failing", { version: 1, params: {}, members: { set: method(int()) } }); + const { host, errors } = await loadedPlugins({ + "@app/names": (host, self) => { + host.serve(Names, { compose: listed }); + self.contribute(Names, "facet"); + host.serve(Failing, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Names, + self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("negative"); + return `n${d.n}`; + }), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }); + const names = await host.registry(Names, context); + const failing = await host.get(Failing, {}, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n1"]); + await failing.set(-1, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(-2, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(2, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n2"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([ + { + kind: "contribution_failed", + registry: "app.names", + plugin: "@app/names", + type: "app.failing", + error: new Error("negative"), + }, + ]); + await host.close(context); + }); + + it("a contribution adapter that throws leaves that contribution out (`contribution_failed`)", async () => { + const V2 = defineRegistry("app.adapted", { version: 2 }); + const V1 = defineRegistry("app.adapted", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/provider": (host, self) => { + host.serve(V2, { + compose: listed, + accepts: { + 1: { + contribution: () => { + throw new Error("cannot adapt"); + }, + }, + }, + }); + self.contribute(V2, "current"); + }, + "@app/old": (_, self) => self.contribute(V1, "old"), + }); + expect((await host.registry(V2, context)).value).toEqual(["@app/provider:=current"]); + expect(errors).toEqual([ + { kind: "contribution_failed", registry: "app.adapted", plugin: "@app/old", error: new Error("cannot adapt") }, + ]); + await host.close(context); + }); + + it("catches a ring of registries at its first composition: the contributors go live, the rest works (audit A2)", async () => { + const A = defineRegistry("app.ring-a", { version: 1 }); + const B = defineRegistry("app.ring-b", { version: 1 }); + const ToA = local("app.to-a", { version: 1, params: {}, members: {} }); + const ToB = local("app.to-b", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/ring": (h, self) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(ToA, (_, self) => { + self.contribute(A, self.registry(B)); + return {}; + }); + h.serve(ToB, (_, self) => { + self.contribute(B, self.registry(A)); + return {}; + }); + self.contribute(A, "plain"); + self.contribute(B, "plain"); + }, + }); + const live = [await host.get(ToA, {}, context), await host.get(ToB, {}, context)]; + expect(live.map((reference) => reference.state.value)).toEqual([{ kind: "live" }, { kind: "live" }]); + // A composes first: B composes for it and its contribution of A, read while A composes, is left out. A has its + // plain entry and B's value, B its plain entry. + expect([(await host.registry(A, context)).value, (await host.registry(B, context)).value]).toEqual([2, 1]); + expect(errors).toEqual([ + { + kind: "contribution_failed", + registry: "app.ring-b", + plugin: "@app/ring", + type: "app.to-b", + error: expect.objectContaining({ code: "invalid", reason: "cycle" }), + }, + ]); + await host.close(context); + }); + + it("keeps a ring's later recompositions bounded: a change composes each registry once, then nothing (audit A2)", async () => { + const A = defineRegistry("app.later-a", { version: 1 }); + const B = defineRegistry("app.later-b", { version: 1 }); + const Ring = local("app.later-ring", { version: 1, params: {}, members: {} }); + const Back = local("app.later-back", { version: 1, params: {}, members: {} }); + const Extra = local("app.later-extra", { version: 1, params: {}, members: {} }); + let composes = 0; + let extra: Writable | undefined; + const sum = (table: RegistryTable): number => { + composes++; + return table.reduce((total, { value }) => total + (typeof value === "number" ? value : 0), 0); + }; + const { host, errors } = await loadedPlugins({ + "@app/later": (h) => { + h.serve(A, { compose: sum }); + h.serve(B, { compose: sum }); + h.serve(Ring, (_, self) => { + self.contribute(A, self.registry(B)); + return {}; + }); + h.serve(Back, (_, self) => { + self.contribute(B, self.registry(A)); + return {}; + }); + h.serve(Extra, (_, self) => { + extra = self.writable(1); + self.contribute(B, extra); + return {}; + }); + }, + }); + await host.get(Ring, {}, context); + await host.get(Back, {}, context); + await host.get(Extra, {}, context); + const a = await host.registry(A, context); + const b = await host.registry(B, context); + const watches = [await a.watch(context), await b.watch(context)]; + await flushed(); + const before = composes; + extra!.set(5); + await flushed(); + // B changed and composes first now: it reads A, whose contribution of B, read while B composes, is cut instead. + expect(composes - before).toBe(2); + expect([a.value, b.value]).toEqual([0, 5]); + expect(errors.map((error) => (error.kind === "contribution_failed" ? error.registry : error.kind))).toEqual([ + "app.later-b", + "app.later-a", + ]); + // Nothing composes on by itself. + await flushed(); + expect(composes - before).toBe(2); + for (const watch of watches) await watch.stop(); + await host.close(context); + }); + + it("catches a ring through a derive, and one declared within one factory, the same way (audit A2)", async () => { + const A = defineRegistry("app.both-a", { version: 1 }); + const B = defineRegistry("app.both-b", { version: 1 }); + const Both = local("app.both", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/both": (h) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(Both, (_, self) => { + self.contribute(A, self.registry(B)); + self.contribute( + B, + self.derive([self.registry(A)], ([n]) => n as number), + ); + return {}; + }); + }, + }); + expect((await host.get(Both, {}, context)).state.value).toEqual({ kind: "live" }); + await host.registry(A, context); + await host.registry(B, context); + const cut = errors.filter((error) => error.kind === "contribution_failed"); + expect(cut.length).toBeGreaterThan(0); + for (const error of cut) expect(error).toMatchObject({ error: { code: "invalid", reason: "cycle" } }); + expect(failures(errors)).toEqual([]); + await host.close(context); + }); + + it("follows only Chord watchables: anything else is a plain value, whatever methods it has (Sol)", async () => { + const Natives = defineRegistry<{ watch(): string; readonly value: string }, readonly string[]>("app.natives", { + version: 1, + }); + const native = { value: "a handle", watch: () => "a native operation" }; + const { host } = await loadedPlugins({ + "@app/x": (h, self) => { + h.serve(Natives, { compose: (table) => table.map(({ value }) => value.watch()) }); + self.contribute(Natives, native); + }, + }); + expect((await host.registry(Natives, context)).value).toEqual(["a native operation"]); + await host.close(context); + }); + + it("keeps an `undefined` contribution in the table (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.undefined", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h, self) => { + h.serve(Nothing, { compose: (table) => table.length }); + self.contribute(Nothing, undefined); + }, + }); + expect((await host.registry(Nothing, context)).value).toBe(1); + await host.close(context); + }); + + it("leaves out a registry's value contributed to another while that registry's compose throws (Sol, Astra)", async () => { + const Source = defineRegistry("app.source", { version: 1 }); + const Target = defineRegistry("app.target", { version: 1 }); + const Bridge = local("app.bridge", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Source, { + compose: (table) => { + const n = table.reduce((sum, { value }) => sum + value, 0); + if (n < 0) throw new Error("negative"); + return n; + }, + }); + h.serve(Target, { compose: (table) => table.map(({ value }) => value) }); + setter(Source, 1, (n) => n)(h); + h.serve(Bridge, (_, self) => { + self.contribute(Target, self.registry(Source)); + return {}; + }); + }, + }); + const target = await host.registry(Target, context); + expect(target.value).toEqual([1]); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // The source's readers keep its last good value; as a contribution it is failed: left out, reported. + expect((await host.registry(Source, context)).value).toBe(1); + expect(target.value).toEqual([]); + expect(errors.find((error) => error.kind === "contribution_failed")).toMatchObject({ + registry: "app.target", + plugin: "@app/a", + type: "app.bridge", + error: new Error("negative"), + }); + await setters.set(3, context); + expect(target.value).toEqual([3]); + await host.close(context); + }); + + it("a followed contribution whose runtime ended its observer stays with its last value (R4: the source keeps it)", async () => { + let end: (() => void) | undefined; + const Follower = local("app.follower", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Names, { compose: listed }); + h.serve(Follower, (_, self) => { + self.contribute( + Names, + self.follow(async (observer) => { + end = () => observer.end({ kind: "ended" }); + return { value: "followed", detach: () => {} }; + }), + ); + return {}; + }); + }, + }); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + end!(); + await flushed(); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([]); + await host.close(context); + }); + + it("an object's contribution is in the table from live until it fails, again once retried, until it ends", async () => { + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Names, `attempt ${attempt}`), + { serve: (host) => host.serve(Names, { compose: listed }) }, + ); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@test/app:app.store=attempt 1"]); + fail(); + expect(names.value).toEqual([]); + await retry(); + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + await host.close(context); + // Closed: nothing composes any more; a value a reader keeps reads the last composition. + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + }); +}); + +describe("the last good composition (decision 5)", () => { + it("is dropped when a contribution it was built from leaves: `.value` throws until `compose` succeeds", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const compose = (table: RegistryTable): number => { + if (table.length === 0) throw new Error("no contributor"); + return table[0]!.value; + }; + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Count, attempt), + { + serve: (host) => host.serve(Count, { compose }), + }, + ); + const count = await host.registry(Count, context); + expect(count.value).toBe(1); + // Its contributor failed: the last good value was built from its contribution (a native may have closed with it). + fail(); + expect(() => count.value).toThrow("no contributor"); + await retry(); + expect(count.value).toBe(2); + await host.close(context); + }); +}); + +describe("the table's memory", () => { + it("keeps nothing of a closed host: records, tables and followed contributions go with it", async () => { + const Doc = defineState("app.kept", { schema: obj({ on: bool() }), version: 1, initial: () => ({ on: true }) }); + const Kept = local("app.kept", { version: 1, params: {}, members: {} }); + const refs = await (async () => { + const { host } = await loadedPlugins({ + "@app/kept": (host) => { + host.serve(Names, { compose: listed }); + host.serve(Kept, (_, self) => { + self.contribute( + Names, + self.derive([self.state(Doc)], ([d]) => `${d.on}`), + ); + return {}; + }); + }, + }); + const names = new WeakRef(await host.registry(Names, context)); + await host.close(context); + return [names, new WeakRef(host)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("a reload's new contribution adapter (fix round 1)", () => { + it("remaps an unchanged contributor with the new provider's adapter", async () => { + const R = defineRegistry("astra.contrib", { version: 2 }); + const Old = defineRegistry("astra.contrib", { version: 1 }); + const contribution = defineFacet((_, self) => self.contribute(Old, 7)); + const provider = (factor: number) => + defineFacet((h) => + h.serve(R, { + compose: (entries) => entries.reduce((n, e) => n + e.value, 0), + accepts: { 1: { contribution: (n) => n * factor, value: (n) => n } }, + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@astra/provider", facet: provider(2) }, + { name: "@astra/contribution", facet: contribution }, + ], + context, + ); + const value = await host.registry(R, context); + expect(value.value).toBe(14); + await host.reload( + [ + { name: "@astra/provider", facet: provider(3) }, + { name: "@astra/contribution", facet: contribution }, + ], + context, + ); + const actual = value.value; + await host.close(context); + expect(actual).toBe(21); + }); +}); + +/** A registry its plugin serves: never listed as undeclared. */ +const Served = defineRegistry("app.served", { version: 1 }); + +describe("registries without a provider (slice 12 §6, testPlugin)", () => { + it("lists a registry with contributions and no provider, with the newest version contributed", async () => { + const Older = defineRegistry("app.names", { version: 1 }); + const Newer = defineRegistry("app.names", { version: 3 }); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "a", facet: defineFacet((_host, self) => self.contribute(Older, "a")) }, + { name: "b", facet: defineFacet((_host, self) => self.contribute(Newer, "b")) }, + { name: "c", facet: defineFacet((h) => h.serve(Served, { compose: () => [] })) }, + ], + context, + ); + expect((host as unknown as ChordHost).undeclaredRegistries()).toEqual([{ name: "app.names", version: 3 }]); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/registry.test.ts b/packages/chord/test/next/registries/registry.test.ts new file mode 100644 index 000000000..f1d31373e --- /dev/null +++ b/packages/chord/test/next/registries/registry.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry, type Registry, type RegistryValue } from "../../../src/next/registries/index.ts"; +import { loadedPlugins } from "../support.ts"; + +describe("tokens", () => { + it("carry a name, a version and the types only; names and versions are checked at declaration", () => { + const Http = defineRegistry<{ readonly path: string }, readonly string[]>("chord.http", { version: 2 }); + expect(Http).toEqual({ kind: "registry", name: "chord.http", version: 2 }); + expectTypeOf(Http).toEqualTypeOf>(); + expect(() => defineRegistry("Chord.Http", { version: 1 })).toThrow('registry "Chord.Http": expected lowercase'); + expect(() => defineRegistry("chord.http", { version: 0 })).toThrow( + "chord.http.version: expected a positive integer", + ); + }); +}); + +describe("providers", () => { + const Names = defineRegistry("app.names", { version: 1 }); + + it("one per host: a second serve leaves its plugin out at load, reported", async () => { + const host = createHost({ onError: () => {} }); + const serve = defineFacet((h) => h.serve(Names, { compose: (table) => table.length })); + const { failed } = await host.load( + [ + { name: "@app/a", facet: serve }, + { name: "@app/b", facet: serve }, + ], + context, + ); + expect(failed).toEqual([ + { + plugin: "@app/b", + error: expect.objectContaining({ + message: 'plugin "@app/b": registry app.names is served twice (plugin "@app/a")', + }), + }, + ]); + await host.close(context); + }); + + it("a provider without compose leaves its plugin out at load (plugin code is not type-checked)", async () => { + const { host, failed } = await loadedPlugins({ "@app/a": (h) => h.serve(Names, {} as never) }); + expect(failed).toEqual([ + { + plugin: "@app/a", + error: expect.objectContaining({ message: 'plugin "@app/a": registry app.names: compose is a function' }), + }, + ]); + await host.close(context); + }); + + it("a registry whose provider comes with `load` heals a `host.registry` made before it", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Names, context); + await host.load( + [{ name: "@app/a", facet: defineFacet((h) => h.serve(Names, { compose: (t) => t.length })) }], + context, + ); + const value = await waiting; + expectTypeOf(value).toEqualTypeOf>(); + expect(value.value).toBe(0); + await host.close(context); + }); + + it("are served only while the setup runs", async () => { + let later: Parameters[0]>[0] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + later = h; + }, + }); + expect(() => later!.serve(Names, { compose: (t) => t.length })).toThrow( + "@app/a: host.serve is valid only in the facet's setup", + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/value.test.ts b/packages/chord/test/next/registries/value.test.ts new file mode 100644 index 000000000..9b102ebe2 --- /dev/null +++ b/packages/chord/test/next/registries/value.test.ts @@ -0,0 +1,540 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import type { FollowObserver } from "../../../src/next/host/follow.ts"; +import { createHost, defineFacet, type Native } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry, type Registry } from "../../../src/next/registries/index.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + rejection, + settledHeap, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The provider's version 2: contributions are numbers, the value their sum. */ +const Sum = defineRegistry("app.sum", { version: 2 }); + +/** A reader built against version 1: its value was a plain number. */ +const SumV1 = defineRegistry("app.sum", { version: 1 }); + +/** A reader built against a version the provider does not know yet. */ +const SumV3 = defineRegistry("app.sum", { version: 3 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter`, contributing a writable number to `Sum`. */ +function setter(host: Parameters[0]>[0], initial = 1): void { + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(Sum, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +const total = (table: readonly { readonly value: number }[]) => ({ + total: table.reduce((n, { value }) => n + value, 0), +}); + +describe("older readers (S2)", () => { + it("get the composition mapped back with accepts[old].value, once per composition", async () => { + const { telemetry, counts } = await metered(); + const mapped: number[] = []; + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { + compose: total, + accepts: { + 1: { + value: (current) => { + mapped.push(current.total); + return current.total; + }, + }, + }, + }); + setter(host); + }, + }, + { telemetry }, + ); + const old = await host.registry(SumV1, context); + const current = await host.registry(Sum, context); + expect(old.value).toBe(1); + expect(old.value).toBe(1); + expect(current.value).toEqual({ total: 1 }); + await (await host.get(Setter, {}, context)).set(4, context); + expect(old.value).toBe(4); + expect(mapped).toEqual([1, 4]); + expect(counts()["chord.registry.adapts"]).toBe(2); + await host.close(context); + }); + + it("without a value mapping, and for a reader newer than the provider, park readers failed { waiting } (contract_mismatch)", async () => { + const Newer = local("app.newer", { version: 1, params: {}, members: { total: int() } }); + const Older = local("app.older", { version: 1, params: {}, members: { total: int() } }); + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total }); + host.serve(Newer, (_, self) => ({ total: self.derive([self.registry(SumV3)], ([v]) => v.total) })); + host.serve(Older, (_, self) => ({ total: self.derive([self.registry(SumV1)], ([v]) => v) })); + }, + }); + const quick = { readyTimeout: 5 }; + expect(await rejection(host.registry(SumV1, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.registry(SumV3, context, quick))).toMatchObject({ code: "contract_mismatch" }); + // Not their fault: waiting (a reload serving the version wakes them), no backoff, nothing failed. + expect(failures(errors)).toEqual([]); + expect(host.hold(Newer, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + message: "waiting for registry app.sum: registry app.sum: version 3 is not served here (version 2 is)", + }, + }, + retryAt: undefined, + }); + await host.close(context); + }); + + it("drops an older reader's last good mapping after a contributor it was built from failed live (fix round 1, Astra 2)", async () => { + const R = defineRegistry("app.mapped", { version: 2 }); + const Old = defineRegistry("app.mapped", { version: 1 }); + const Contributor = local("app.contributor", { version: 1, params: {}, members: { n: int() } }); + let observer: FollowObserver | undefined; + const { host } = await loadedPlugins({ + "@app/mapping": (h) => { + h.serve(R, { + compose: (table) => table.reduce((n, e) => n + e.value, 0), + accepts: { + 1: { + value: (n) => { + if (n === 0) throw new Error("old version cannot map the empty table"); + return n; + }, + }, + }, + }); + h.serve(Contributor, (_, self) => { + self.contribute(R, 42); + return { + n: self.follow(async (followed) => { + observer = followed; + return { value: 42, detach() {} }; + }), + }; + }); + }, + }); + const old = await host.registry(Old, context); + expect(old.value).toBe(42); + observer!.end({ kind: "failed", error: new Error("runtime ended") }); + await flushed(); + expect(() => old.value).toThrow("old version cannot map the empty table"); + await host.close(context); + }); + + it("never hands out a mapped native whose contributor failed and closed it (fix round 1, Sol 2)", async () => { + type Resource = { closed: boolean }; + const Current = defineRegistry, readonly Native[]>("app.resources", { version: 2 }); + const Old = defineRegistry, Native>("app.resources", { version: 1 }); + const Owner = local("app.owner", { version: 1, params: {}, members: {} }); + const resource = { closed: false }; + let failRuntime: () => void = () => {}; + const { host } = await loadedPlugins({ + "@app/resources": (h) => { + h.serve(Current, { + compose: (table) => table.map(({ value }) => value), + accepts: { + 1: { + value: (current) => { + if (current.length === 0) throw new Error("no current resource"); + return current[0]!; + }, + }, + }, + }); + h.serve(Owner, (_, self) => { + const native = self.open( + async () => resource, + () => { + resource.closed = true; + }, + ); + self.contribute(Current, native); + self.follow(async (observer) => { + failRuntime = () => observer.end({ kind: "failed", error: new Error("runtime closed") }); + return { value: 1, detach: () => {} }; + }); + return {}; + }); + }, + }); + const value = await host.registry(Old, context); + expect(value.value.value).toBe(resource); + failRuntime(); + await flushed(); + expect(resource.closed).toBe(true); + expect(() => value.value).toThrow("no current resource"); + await host.close(context); + }); + + it("maps an `undefined` composition (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.nothing", { version: 2 }); + const NothingV1 = defineRegistry("app.nothing", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h) => h.serve(Nothing, { compose: () => undefined, accepts: { 1: { value: () => "mapped" } } }), + }); + expect((await host.registry(NothingV1, context)).value).toBe("mapped"); + await host.close(context); + }); + + it("a failed mapping heals when the composition is the one mapped before (Astra)", async () => { + const Count = defineRegistry("app.count", { version: 2 }); + const CountV1 = defineRegistry("app.count", { version: 1 }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/count": (h) => { + h.serve(Count, { + compose: (table) => table.reduce((n, { value }) => n + value, 0), + accepts: { + 1: { + value: (n) => { + if (n < 0) throw new Error("negative"); + return n; + }, + }, + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, + }); + const old = await host.registry(CountV1, context); + const calls: NativeWatchCall[] = []; + old.observe({ advance: (call) => calls.push(call), end: () => {} }, context); + const counter = await host.get(Counter, {}, context); + await counter.set(-1, context); + // Back to the composition mapped before (the same number): the mapping heals. + await counter.set(1, context); + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative") }, + { kind: "value", value: 1 }, + ]); + await host.close(context); + }); +}); + +describe("readers (S3)", () => { + const Reader = local("app.reader", { version: 1, params: {}, members: { total: int() } }); + const serveReader = ( + host: Parameters[0]>[0], + R: Registry = Sum, + ) => host.serve(Reader, (_, self) => ({ total: self.derive([self.registry(R)], ([v]) => v.total) })); + + it("activate once the registry has a value; a reader of a registry nobody serves parks failed { waiting }, `load` waits for nothing", async () => { + const started = Date.now(); + const { host, errors } = await loadedPlugins({ "@app/reader": (host) => serveReader(host) }); + expect(Date.now() - started).toBeLessThan(1_000); + expect(errors).toEqual([]); + expect(host.hold(Reader, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + code: "unavailable", + message: "waiting for registry app.sum: registry app.sum: no plugin serves it here", + cause: { code: "unavailable", reason: "undeclared" }, + }, + }, + }); + expect(await rejection(host.registry(Sum, context, { readyTimeout: 5 }))).toMatchObject({ + code: "unavailable", + reason: "undeclared", + }); + await host.close(context); + }); + + it("a reader parked on a first `compose` that threw goes on at the first good composition", async () => { + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + serveReader(host); + }, + "@app/setter": (host) => setter(host, 0), + }); + expect(failures(errors)).toEqual([]); + const held = host.hold(Reader, {}, context); + expect(held.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const setters = await host.get(Setter, {}, context); + await setters.set(3, context); + // Woken by the composition: no timer of its own. + const reader = await host.get(Reader, {}, context, { readyTimeout: 500 }); + held.release(); + expect(reader.total.value).toBe(3); + await host.close(context); + }); + + it("host.registry before the publication waits for it, then resolves with the provider it served", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Sum, context); + await host.load([{ name: "@app/sum", facet: defineFacet((h) => h.serve(Sum, { compose: total })) }], context); + expect((await waiting).value).toEqual({ total: 0 }); + await host.close(context); + }); + + it("host.registry waits through no value until a first good composition, woken by it, its wait then gone", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + setter(host, 0); + }, + }); + const waiting = host.registry(Sum, context, { readyTimeout: 500 }); + await flushed(); + await (await host.get(Setter, {}, context)).set(5, context); + expect((await waiting).value).toEqual({ total: 5 }); + await host.close(context); + }); + + it("once live, a later throwing `compose` changes nothing for the reader: `.value` keeps the last good one", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total < 0) throw new Error("negative"); + return sum; + }, + }); + serveReader(host); + setter(host, 2); + }, + }); + const reader = await host.get(Reader, {}, context); + const setters = await host.get(Setter, {}, context); + expect(reader.total.value).toBe(2); + await setters.set(-1, context); + expect(reader.total.value).toBe(2); + expect(reader.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("readers' memory", () => { + it("an ended reader is collected while the registry lives: its derive and its read leave the value", async () => { + const Reader = local("app.reader", { version: 1, params: { id: int() }, members: { total: int() } }); + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total, accepts: { 1: { value: ({ total }) => total } } }); + host.serve(Reader, (_, self) => ({ + total: self.derive([self.registry(Sum), self.registry(SumV1)], ([v, old]) => v.total + old), + })); + setter(host); + }, + }); + const refs = await (async () => { + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.total.value).toBe(2); + reader.release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return objects.filter((ref) => ref.deref()?.label === "app.reader(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await (await host.get(Setter, {}, context)).set(5, context); + await expectCollected(...refs); + await host.close(context); + }); +}); + +describe("values' memory", () => { + it("drops an older version's mapping once its last reader ended (Sol)", async () => { + const New = defineRegistry("app.version-gc", { version: 2 }); + const Old = defineRegistry("app.version-gc", { + version: 1, + }); + const Reader = local("app.old-reader", { version: 1, params: { id: int() }, members: { n: int() } }); + let mapped: WeakRef | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(New, { + compose: () => 1, + accepts: { + 1: { + value: (n) => { + const value = { n, payload: new Uint8Array(1_000_000) }; + mapped = new WeakRef(value); + return value; + }, + }, + }, + }); + h.serve(Reader, (_, self) => ({ n: self.derive([self.registry(Old)], ([v]) => v.n) })); + }, + }); + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.n.value).toBe(1); + reader.release(); + await flushed(); + await expectCollected(mapped!); + await host.close(context); + }); + + it("keeps no old composition once its readers stopped waiting and nobody follows it (Opus)", async () => { + const Big = defineRegistry("app.big", { version: 1 }); + let first = true; + const made: WeakRef[] = []; + const Waiting = local("app.waiting", { version: 1, params: {}, members: {} }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Big, { + compose: (table) => { + if (first) throw new Error("first"); + const value = { payload: new Array(1000).fill(table.length) }; + made.push(new WeakRef(value)); + return value; + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(0); + self.contribute(Big, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + h.serve(Waiting, (_, self) => { + void self.registry(Big); + return {}; + }); + }, + }); + const counter = await host.get(Counter, {}, context); + first = false; + // The waiting reader's composition (pulled in the batch of this set) wakes it. + await counter.set(1, context); + await flushed(); + await counter.set(2, context); + void (await host.registry(Big, context)).value; + await settledHeap(); + // Only the current composition is alive. + expect(made.map((ref) => ref.deref() !== undefined)).toEqual([false, true]); + await host.close(context); + }); + + it("a value an app keeps after `host.close` keeps no closed host, and reads its last composition (Sol, Astra)", async () => { + const Kept = defineRegistry("app.kept", { version: 1 }); + const signal = new AbortController(); + const { weak, value } = await (async () => { + const { host } = await loadedPlugins({ "@app/a": (h) => h.serve(Kept, { compose: () => 7 }) }); + const value = await host.registry(Kept, context); + value.observe({ advance: () => {}, end: () => {} }, withAbortSignal(signal.signal, context)); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + // The observation ended with the host: its abort listener went with it. + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), value }; + })(); + await expectCollected(weak); + expect(value.value).toBe(7); + }); +}); + +describe("observe", () => { + it("calls a synchronous observer inside the batch that composed; it counts as watching", async () => { + const Labels = defineRegistry("app.labels", { version: 1 }); + const Label = local("app.label", { version: 1, params: {}, members: { set: method(str()) } }); + const { host } = await loadedPlugins({ + "@app/labels": (host) => { + host.serve(Labels, { compose: (table) => table.map(({ value }) => value).join(",") }); + host.serve(Label, (_, self) => { + const label = self.writable("a"); + self.contribute(Labels, label); + return { + set: async (text: string) => { + label.set(text); + return null; + }, + }; + }); + }, + }); + const labels = await host.registry(Labels, context); + const calls: (NativeWatchCall | WatchEnd)[] = []; + const { context: cancellable, cancel } = withCancel(context); + const observation = labels.observe( + { advance: (call) => calls.push(call), end: (end) => calls.push(end) }, + cancellable, + ); + expect(observation.value).toBe("a"); + const label = await host.get(Label, {}, context); + await label.set("b", context); + // Already called: synchronously, in the set's batch. + expect(calls).toEqual([{ kind: "value", value: "b" }]); + cancel(); + await label.set("c", context); + expect(calls).toEqual([{ kind: "value", value: "b" }, { kind: "aborted" }]); + const again = labels.observe({ advance: (call) => calls.push(call), end: (end) => calls.push(end) }, context); + again.detach(); + await label.set("d", context); + expect(calls).toHaveLength(2); + labels.observe({ advance: () => {}, end: (end) => calls.push(end) }, context); + await host.close(context); + expect(calls.at(-1)).toEqual({ kind: "ended" }); + }); +}); + +describe("a reload's new value adapter (fix round 1)", () => { + it("uses a new value adapter even when compose returns the same primitive", async () => { + const R = defineRegistry("astra.registry", { version: 2 }); + const Old = defineRegistry("astra.registry", { version: 1 }); + const host = createHost({ onError: () => {} }); + const facet = (factor: number) => + defineFacet((h) => h.serve(R, { compose: () => 7, accepts: { 1: { value: (n) => n * factor } } })); + await host.load([{ name: "@astra/reg", facet: facet(2) }], context); + const old = await host.registry(Old, context); + expect(old.value).toBe(14); + await host.reload([{ name: "@astra/reg", facet: facet(3) }], context); + const actual = old.value; + await host.close(context); + expect(actual).toBe(21); + }); +}); diff --git a/packages/chord/test/next/remote/calls.test.ts b/packages/chord/test/next/remote/calls.test.ts new file mode 100644 index 000000000..79f3981da --- /dev/null +++ b/packages/chord/test/next/remote/calls.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { CallWindow, type QueuedCall } from "../../../src/next/remote/calls.ts"; +import { expectCollected } from "../support.ts"; + +/** A call of `hold` that records its name in `sent` when it goes out. */ +function callOf(name: string, sent: string[], hold = 0): QueuedCall { + return { hold, send: () => void sent.push(name) }; +} + +describe("the call window", () => { + it("sends nothing before the peer's hello, then up to its N; the rest waits", () => { + const sent: string[] = []; + const window = new CallWindow(); + for (const name of ["a", "b", "c"]) window.submit(callOf(name, sent)); + expect(sent).toEqual([]); + window.opened(2); + expect(sent).toEqual(["a", "b"]); + window.freed(); + expect(sent).toEqual(["a", "b", "c"]); + }); + + it("sends waiting calls in order as slots free", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + for (const name of ["a", "b", "c", "d"]) window.submit(callOf(name, sent)); + for (let slot = 0; slot < 3; slot++) window.freed(); + expect(sent).toEqual(["a", "b", "c", "d"]); + }); + + it("never preempts: a call that waits does not overtake while every slot is taken", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + window.submit(callOf("long", sent)); + window.submit(callOf("short", sent)); + expect(sent).toEqual(["long"]); + }); + + it("takes out a cancelled waiting call, and every waiting call of a hold that rebinds", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + window.submit(callOf("first", sent)); + const cancelled = callOf("cancelled", sent); + window.submit(cancelled); + const rebinding = [callOf("hold 1 a", sent, 1), callOf("hold 1 b", sent, 1)]; + for (const call of rebinding) window.submit(call); + window.submit(callOf("last", sent)); + window.remove(cancelled); + // A second remove of the same call changes nothing. + window.remove(cancelled); + expect(window.removeHold(1)).toEqual(rebinding); + window.freed(); + expect(sent).toEqual(["first", "last"]); + expect(window.clear()).toEqual([]); + }); + + it("keeps no call that went out or was taken", async () => { + const window = new CallWindow(); + window.opened(1); + const refs = (() => { + const sent = callOf("a", []); + const next = callOf("b", []); + const waiting = callOf("c", []); + for (const call of [sent, next, waiting]) window.submit(call); + window.freed(); + window.remove(waiting); + return [new WeakRef(sent), new WeakRef(waiting)]; + })(); + await expectCollected(...refs); + expect(window).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/remote/consumer.test.ts b/packages/chord/test/next/remote/consumer.test.ts new file mode 100644 index 000000000..5f7005a68 --- /dev/null +++ b/packages/chord/test/next/remote/consumer.test.ts @@ -0,0 +1,722 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, map, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { PluginError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { DeflatedPart, jsonPart, type Part, wireError } from "../../../src/next/link/internal.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + bindNext, + EDITOR, + expectCollected, + forEachCodec, + framesOf, + instancesOf, + linked, + metered, + Notes, + RawPeer, + rawParent, + rawProvider, + rejection, + SERVER_ADDRESS, + Server, + serveNotes, + testCodec, + transported, +} from "../support.ts"; + +/** Every remote object bound while this file runs, weakly. */ +const remoteObjects = instancesOf(RemoteObject, "hold"); + +forEachCodec(() => { + describe("references over a link", () => { + it("are live and hydrated when got, with the variant of the link's principal", async () => { + const { client, server } = await linked((h) => void serveNotes(h), { + child: { kind: "guest", session: "S1" }, + }); + await addNote(server, "S1", "a"); + const notes = await client.get(Notes, { session: "S1" }, context); + expect(notes.variant).toBe("reader"); + expect(notes.notes.value).toEqual([{ text: "a" }]); + expect(notes.count.value).toBe(1); + const frames = framesOf(await notes.notes.watch(context)); + await addNote(server, "S1", "b"); + await transported(); + expect(frames).toEqual([[[{ text: "a" }, { text: "b" }], [["p", [], 1, 0, [{ text: "b" }]]]]]); + expect(notes.count.value).toBe(2); + }); + + it("apply one state message as one batch: a derive over two members never sees half a commit", async () => { + const View = local("app.view", { version: 1, params: {}, members: { pair: str() } }); + const { server } = await linked((h) => void serveNotes(h)); + const seen: string[] = []; + const client = createHost({ onError: () => {} }); + // Connected first (W6): the view's factory references a server type. + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await client.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => { + const notes = self.get(Notes, { session: "S1" }); + return { + pair: self.derive([notes.notes, notes.count], ([list, count]) => { + seen.push(`${list.length}/${count}`); + return `${list.length}/${count}`; + }), + }; + }), + ), + }, + ], + context, + ); + const view = await client.get(View, {}, context); + const frames = framesOf(await view.pair.watch(context)); + await addNote(server, "S1", "a"); + await addNote(server, "S1", "b"); + await transported(); + expect(frames.map(([value]) => value)).toEqual(["1/1", "2/2"]); + expect(seen.every((pair) => pair.split("/")[0] === pair.split("/")[1])).toBe(true); + }); + + it("call methods: results, ChordErrors, PluginErrors and other throws", async () => { + const { client } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "x" }, context)).toBe(1); + const chord = await rejection(notes.fail({ how: "chord" }, context)); + expect(chord).toEqual({ code: "forbidden", message: "not yours" }); + const plugin = await notes.fail({ how: "plugin" }, context).catch((error: unknown) => error); + expect(plugin).toBeInstanceOf(PluginError); + expect(plugin).toMatchObject({ code: "quota", message: "too many", details: { limit: 1 } }); + expect(await notes.fail({ how: "plain" }, context).catch((error: unknown) => error)).toMatchObject({ + name: "PluginError", + code: "internal", + message: "plain", + }); + expect(await rejection(notes.add({ text: 1 } as never, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("carry bytes as raw segments both ways", async () => { + const { client } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + const data = new Uint8Array([1, 2, 3]); + const echoed = await notes.echo({ data }, context); + expect(echoed).toBeInstanceOf(Uint8Array); + expect([...echoed]).toEqual([1, 2, 3]); + }); + + it("cancel a call: the caller is rejected at once, the provider's handler aborted", async () => { + let probe: ReturnType | undefined; + const { client } = await linked((h) => { + probe = serveNotes(h); + }); + const notes = await client.get(Notes, { session: "S1" }, context); + const { context: cancellable, cancel } = withCancel(context); + const call = notes.wait(cancellable); + await transported(); + cancel(new Error("user cancelled")); + await expect(call).rejects.toThrow("user cancelled"); + await transported(); + expect(probe!.waits[0]!.abortSignal?.aborted).toBe(true); + }); + + it("release: pending calls reject released, the provider aborts their handlers and lets the object go", async () => { + let probe: ReturnType | undefined; + const { client } = await linked((h) => { + probe = serveNotes(h); + }); + const notes = await client.get(Notes, { session: "S1" }, context); + const watch = await notes.notes.watch(context); + const call = notes.wait(context); + await transported(); + notes.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.notes(S1): released" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + await transported(); + expect(probe!.waits[0]!.abortSignal?.aborted).toBe(true); + // The object ended with its last reference: a new get runs the factory again. + await client.get(Notes, { session: "S1" }, context); + expect(probe!.factories).toEqual(["S1 owner", "S1 owner"]); + }); + + it("a closing link rejects sent calls unavailable; references go loading, gets keep waiting", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const call = notes.wait(context); + const pending = client.get(Notes, { session: "S2" }, context, { readyTimeout: 50 }); + await peer.next("call"); + peer.close(); + expect(await rejection(call)).toEqual({ code: "unavailable", message: "the link closed" }); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(await rejection(pending)).toEqual({ + code: "unavailable", + message: "app.notes(S2): did not become live within 50 ms", + }); + }); + }); + + describe("races at the consumer", () => { + it("ignores a bound and a state for a hold it released, and a result crossing its cancel", async () => { + const { client, peer, errors } = await rawProvider(); + const late = client.get(Notes, { session: "S1" }, context, { readyTimeout: 10 }); + const { hold } = await peer.next("hold"); + await expect(late).rejects.toThrow("did not become live"); + const members = [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); + peer.send({ + kind: "state", + holds: [ + { hold, members: [{ member: EDITOR.state.count!, ops: jsonPart([["r", 1]]) }], failed: [], retired: [] }, + ], + }); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const { context: cancellable, cancel } = withCancel(context); + const call = notes.wait(cancellable); + const { call: id } = await peer.next("call"); + cancel(new Error("cancelled")); + await expect(call).rejects.toThrow("cancelled"); + await peer.next("abort"); + peer.send({ kind: "result", call: id }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + }); + + describe("protocol errors at the consumer", () => { + it.each([ + ["wire ops failing Delta's checks", [{ member: EDITOR.state.count!, ops: '[["x"]]' }]], + ["a member index outside the variant", [{ member: 9, ops: '[["r",1]]' }]], + [ + "a member twice", + [ + { member: EDITOR.state.count!, ops: '[["r",1]]' }, + { member: EDITOR.state.count!, ops: '[["r",2]]' }, + ], + ], + ["a member's changes that are no list", [{ member: EDITOR.state.count!, ops: "1" }]], + ["ops that are no JSON", [{ member: EDITOR.state.count!, ops: "[" }]], + ["ops that do not apply to the member", [{ member: EDITOR.state.count!, ops: '[["s",["x"],1]]' }]], + ])("close the link: %s", async (_, members) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + peer.send({ kind: "state", holds: [{ hold, members, failed: [], retired: [] }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + // Nothing of the message was applied; the reference waits for the next link. + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(notes.count.value).toBe(0); + }); + + it.each([ + ["a result that is no JSON", { kind: "result", call: 0, value: "{" }], + ["a $bytes reference without its segment", { kind: "result", call: 0, value: '{"$bytes":0}', segments: [] }], + ["a malformed ChordError envelope", { kind: "error", call: 0, error: { kind: "chord", error: { code: 1 } } }], + ] as const)("reject the call and close the link on a malformed reply: %s", async (_, reply) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const call = notes.wait(context); + await peer.next("call"); + peer.send(reply); + // The text codec cannot even decode a result that is no JSON: the link closes and the call with it. + expect(await rejection(call)).toMatchObject({ code: "unavailable" }); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + + it("ignore a gone for a hold it does not know before reading its error (Q1)", async () => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + await got; + peer.send({ kind: "gone", hold: 9, error: { kind: "chord", error: { code: "nope" } } }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + + it("close the link on a malformed error envelope", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "failed", + hold, + cause: { kind: "broken", error: { kind: "chord", error: { code: "nope", message: "m" } } }, + }); + await transported(); + expect(peer.closed).toBe(true); + expect(notes.state.value).toEqual({ kind: "loading" }); + }); + }); + + describe("the wire format of a hold", () => { + it("is the snapshot by member, then each commit's changed members with their ops", async () => { + const { server } = await linked((h) => void serveNotes(h)); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + peer.send({ + kind: "hold", + hold: 7, + type: Notes.name, + version: Notes.version, + fingerprint: Notes.fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); + const [bound] = (await peer.next("bound")).holds; + expect(bound!.variant).toBe(EDITOR.variant); + expect(EDITOR.named(bound!.members)).toEqual({ notes: [], count: 0 }); + await addNote(server, "S1", "a"); + const state = await peer.next("state"); + expect(state.holds).toMatchObject([{ hold: 7, failed: [] }]); + expect(EDITOR.named(state.holds[0]!.members)).toEqual({ + notes: [["p", [], 0, 0, [{ text: "a" }]]], + count: [["r", 1]], + }); + }); + }); + + describe("cost guards at the consumer", () => { + it("applies state without validating, walking or copying it: unchanged parts stay the same objects", async () => { + const { context: counting, counts } = await metered(); + const { server } = await linked((h) => void serveNotes(h)); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, counting); + await addNote(server, "S1", "a"); + const notes = await client.get(Notes, { session: "S1" }, context); + const first = notes.notes.value[0]; + await addNote(server, "S1", "b"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "a" }, { text: "b" }]); + expect(notes.notes.value[0]).toBe(first); + expect(counts()["chord.data.validations"]).toBeUndefined(); + expect(counts()["chord.data.walks"]).toBeUndefined(); + }); + }); + + describe("GC at the consumer", () => { + it("a closed link retained by the test, with its host, keeps no hold, replica or released reference", async () => { + const { client, link, refs } = await (async () => { + const { client, peer } = await rawProvider(); + const made = remoteObjects.length; + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const refs: WeakRef[] = [new WeakRef(resolveWatchable(notes.notes)!), new WeakRef(notes)]; + refs.push(...remoteObjects.slice(made)); + expect(refs).toHaveLength(3); + peer.close(); + await transported(); + // Across the drop the host keeps the reference's object for the next link; released, nothing stays. + notes.release(); + return { client, link: peer, refs }; + })(); + await expectCollected(...refs); + expect([client, link]).toBeDefined(); + }); + }); + + describe("one state message over several holds", () => { + const Pair = local("app.pair", { version: 1, params: {}, members: { pair: str() } }); + + /** A client whose `Pair` derives over the counts of two holds (S1, S2), both bound by a raw provider. */ + async function pair() { + const seen: string[] = []; + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(Pair, (_, self) => { + const one = self.get(Notes, { session: "S1" }); + const two = self.get(Notes, { session: "S2" }); + return { + pair: self.derive([one.count, two.count], ([x, y]) => { + seen.push(`${x}/${y}`); + return `${x}/${y}`; + }), + }; + }), + ); + const loading = client.load([{ name: "@test/pair", facet }], context); + await bindNext(peer); + await bindNext(peer); + await loading; + const view = await client.get(Pair, {}, context); + const frames = framesOf(await view.pair.watch(context)); + return { client, peer, seen, frames, errors, view }; + } + + it("is applied as one batch: a derive over both holds never sees half of it, a failed member included", async () => { + const { peer, seen, frames, view } = await pair(); + const failed = [{ member: EDITOR.state.notes!, error: wireError(new PluginError("derive", "notes threw")) }]; + const count = [{ member: EDITOR.state.count!, ops: '[["r",1]]' }]; + peer.send({ + kind: "state", + holds: [ + { hold: 0, members: count, failed: [], retired: [] }, + { hold: 1, members: count, failed, retired: [] }, + ], + }); + await transported(); + expect(frames.map(([value]) => value)).toEqual(["1/1"]); + expect(seen).toEqual(["0/0", "1/1"]); + expect(view.pair.value).toBe("1/1"); + }); + + it("with a malformed later entry applies nothing and closes the link", async () => { + const { peer, seen, frames, errors } = await pair(); + peer.send({ + kind: "state", + holds: [ + { hold: 0, members: [{ member: EDITOR.state.count!, ops: '[["r",1]]' }], failed: [], retired: [] }, + { hold: 1, members: [{ member: EDITOR.state.count!, ops: '[["x"]]' }], failed: [], retired: [] }, + ], + }); + await transported(); + expect(frames).toEqual([]); + expect(seen).toEqual(["0/0"]); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + }); + + describe("$bytes outside methods with bytes", () => { + it("is plain data: {$bytes: 0} round-trips unchanged through a method without bytes", async () => { + const Echo = remote("app.echo", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { methods: { echo: method(map(int()), map(int())) } } }, + }); + const { client } = await linked((h) => + h.serve(Echo, () => ({ variant: "v", members: { echo: async (args) => args } })), + ); + const echo = await client.get(Echo, {}, context); + expect(await echo.echo({ $bytes: 0 }, context)).toEqual({ $bytes: 0 }); + }); + }); +}); + +forEachCodec(() => { + describe("the call window at the consumer", () => { + /** A client connected to a raw provider announcing one concurrent call, and two bound references. */ + async function narrow() { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { calls: 1 }); + const one = client.get(Notes, { session: "S1" }, context); + const first = await bindNext(peer); + const two = client.get(Notes, { session: "S2" }, context); + const second = await bindNext(peer); + return { peer, one: await one, two: await two, holds: [first, second] as const }; + } + + /** The calls the provider received, by hold. */ + const calls = (peer: RawPeer) => + peer.received.flatMap((message) => (message.kind === "call" ? [message.hold] : [])); + + it("frees a slot once when a hold with a call in flight and one waiting is released", async () => { + const { peer, one, two, holds } = await narrow(); + const sent = one.wait(context); + const waiting = one.wait(context); + const other = [two.wait(context), two.wait(context)]; + await transported(); + expect(calls(peer)).toEqual([holds[0]]); + one.release(); + expect(await rejection(sent)).toMatchObject({ code: "released" }); + expect(await rejection(waiting)).toMatchObject({ code: "released" }); + await transported(); + expect(calls(peer)).toEqual([holds[0], holds[1]]); + expect(other).toHaveLength(2); + }); + + it("frees a slot once on gone", async () => { + const { peer, one, two, holds } = await narrow(); + const sent = one.wait(context); + const other = [two.wait(context), two.wait(context)]; + await transported(); + peer.send({ + kind: "gone", + hold: holds[0], + error: { kind: "chord", error: { code: "unavailable", message: "revoked" } }, + }); + expect(await rejection(sent)).toMatchObject({ message: "revoked" }); + await transported(); + expect(calls(peer)).toEqual([holds[0], holds[1]]); + expect(other).toHaveLength(2); + }); + + it("rejects a call still waiting when its hold rebinds: unavailable (race 26)", async () => { + const { peer, one, holds } = await narrow(); + const sent = one.wait(context); + const waiting = one.wait(context); + await transported(); + peer.send({ kind: "loading", hold: holds[0] }); + expect(await rejection(waiting)).toEqual({ code: "unavailable", message: `hold ${holds[0]}: rebinding` }); + const { call } = await peer.next("call"); + peer.send({ kind: "result", call }); + expect(await sent).toBe(null); + }); + }); +}); + +forEachCodec(() => { + describe("limits per part at the consumer", () => { + /** `[member, value]` of `Notes`'s editor variant: `notes` and `count`, the notes' JSON given as text. */ + const members = (notes: string): { member: number; value: Part }[] => [ + { member: EDITOR.state.notes!, value: notes }, + { member: EDITOR.state.count!, value: "1" }, + ]; + + it("rejects a call whose result is past the link's limits with too_large/part; the link stays", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + await transported(); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const added = notes.add({ text: "x" }, context); + const { call } = await peer.next("call"); + peer.send({ kind: "result", call, value: jsonPart("y".repeat(60)) }); + expect(await rejection(added)).toMatchObject({ code: "too_large", reason: "part" }); + const again = notes.add({ text: "x" }, context); + peer.send({ kind: "result", call: (await peer.next("call")).call, value: "2" }); + expect(await again).toBe(2); + }); + + it("fails a hold whose snapshot is past the limits broken, releases it, and binds the message's others", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const big = client.hold(Notes, { session: "S1" }, context); + const small = client.hold(Notes, { session: "S2" }, context); + const [one, two] = [await peer.next("hold"), await peer.next("hold")]; + const holds = [ + { + hold: one.hold, + variant: EDITOR.variant, + members: members(jsonPart([{ text: "z".repeat(60) }])), + failed: [], + }, + { hold: two.hold, variant: EDITOR.variant, members: members('[{"text":"a"}]'), failed: [] }, + ]; + peer.send({ kind: "bound", holds }); + expect(await peer.next("release")).toEqual({ kind: "release", hold: one.hold }); + expect(await peer.next("ack")).toEqual({ kind: "ack" }); + expect(big.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "too_large" } }, + }); + expect(small.notes.value).toEqual([{ text: "a" }]); + // No automatic re-hold; a retry holds again. + await transported(); + expect(peer.kinds()).toEqual([]); + big.retry(); + expect(big.state.value.kind).toBe("loading"); + const again = await peer.next("hold"); + expect(again.hold).toBeGreaterThan(two.hold); + }); + + it("acknowledges a message waiting for its asynchronous inflate only once applied, in order (race 14)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const list = [{ text: "q".repeat(30_000) }]; + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode(JSON.stringify(list)))); + const bound = { hold, variant: EDITOR.variant, members: [...members("[]")], failed: [] }; + bound.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [bound] }); + const ops = jsonPart([["r", 2]]); + peer.send({ + kind: "state", + holds: [{ hold, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }], + }); + await peer.next("ack"); + expect(notes.notes.value).toEqual(list); + await peer.next("ack"); + expect(notes.count.value).toBe(2); + }); + + it("drops a message whose asynchronous inflate is in flight when the link closes, and those behind it (race 13)", async () => { + const { client, peer, errors } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const list = [{ text: "q".repeat(30_000) }]; + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode(JSON.stringify(list)))); + const bound = { hold, variant: EDITOR.variant, members: [...members("[]")], failed: [] }; + bound.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [bound] }); + peer.close(); + for (let i = 0; i < 20; i++) await new Promise((resolve) => setTimeout(resolve, 1)); + expect(notes.state.value.kind).toBe("loading"); + expect(errors).toEqual([]); + }); + + it("inflates nothing it gave up (Q1): a corrupt deflated result of an unknown call, an entry of a released hold (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { client, peer, errors } = await rawProvider({ compression }); + const kept = client.hold(Notes, { session: "S1" }, context); + const released = client.hold(Notes, { session: "S2" }, context); + const [a, b] = [await bindNext(peer), await bindNext(peer)]; + await transported(); + released.release(); + await peer.next("release"); + const corrupt = () => new DeflatedPart(new Uint8Array([255, 255, 255])); + peer.send({ kind: "result", call: 999, value: corrupt() }); + const ops = jsonPart([["r", 4]]); + peer.send({ + kind: "state", + holds: [ + { hold: b, members: [{ member: EDITOR.state.count!, ops: corrupt() }], failed: [], retired: [] }, + { hold: a, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }, + ], + }); + await peer.next("ack"); + expect(kept.count.value).toBe(4); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("checks a deflated part's wire size before inflating it: corrupt data past the limit only fails its hold", async () => { + for (const compression of [nodeCompression, undefined]) { + const { client, peer, errors } = await rawProvider({ compression, limits: { part: 10 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const value = new DeflatedPart(new Uint8Array(20).fill(255)); + peer.send({ + kind: "bound", + holds: [ + { + hold, + variant: EDITOR.variant, + members: [ + { member: EDITOR.state.notes!, value }, + { member: EDITOR.state.count!, value: "1" }, + ], + failed: [], + }, + ], + }); + expect(await peer.next("release")).toEqual({ kind: "release", hold }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + expect(errors).toEqual([]); + } + }); + + it("keeps a hold failed for a part past the limits off the next link until retry, offline retry included", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "bound", + holds: [ + { hold, variant: EDITOR.variant, members: members(jsonPart([{ text: "z".repeat(60) }])), failed: [] }, + ], + }); + await peer.next("release"); + peer.close(); + await transported(); + const next = await rawParent(client, { part: 50 }); + await transported(); + // A reconnect holds nothing again: the failure is final until retry. + expect(next.kinds()).toEqual([]); + expect(notes.state.value.kind).toBe("failed"); + next.close(); + await transported(); + // A retry while offline holds it on the next link. + notes.retry(); + expect(notes.state.value.kind).toBe("loading"); + const third = await rawParent(client); + expect((await third.next("hold")).type).toBe(Notes.name); + }); + + it("stops an asynchronous inflate at once when the link closes: nothing of the link stays (GC)", async () => { + // Decompression that never yields: only the close ends it. + const streams: WeakRef[] = []; + class Stalled { + readonly writable: WritableStream; + readonly readable: ReadableStream; + constructor() { + const stream = new TransformStream({ transform() {} }); + [this.writable, this.readable] = [stream.writable, stream.readable]; + streams.push(new WeakRef(this)); + } + } + vi.stubGlobal("DecompressionStream", Stalled); + try { + const refs = await (async () => { + const { client, peer } = await rawProvider(); + client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode("[]"))); + const entry = { hold, variant: EDITOR.variant, members: members("[]"), failed: [] }; + entry.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [entry] }); + await transported(); + expect(streams).toHaveLength(1); + peer.close(); + await transported(); + await client.close(context); + return [new WeakRef(client), ...streams]; + })(); + await expectCollected(...refs); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("fails only the hold of a state part past the limits (race 10); the others of the message apply", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const one = client.hold(Notes, { session: "S1" }, context); + const two = client.hold(Notes, { session: "S2" }, context); + const [a, b] = [await bindNext(peer), await bindNext(peer)]; + await transported(); + const ops = (count: number, pad: number) => jsonPart([["r", count], ...Array(pad).fill(["r", count])]); + peer.send({ + kind: "state", + holds: [ + { hold: a, members: [{ member: EDITOR.state.count!, ops: ops(5, 10) }], failed: [], retired: [] }, + { hold: b, members: [{ member: EDITOR.state.count!, ops: ops(7, 0) }], failed: [], retired: [] }, + ], + }); + expect(await peer.next("release")).toEqual({ kind: "release", hold: a }); + await transported(); + expect(one.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + expect(two.count.value).toBe(7); + expect(peer.closed).toBe(false); + }); + }); +}); diff --git a/packages/chord/test/next/remote/hold.test.ts b/packages/chord/test/next/remote/hold.test.ts new file mode 100644 index 000000000..2250a0a8f --- /dev/null +++ b/packages/chord/test/next/remote/hold.test.ts @@ -0,0 +1,1369 @@ +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type * as Delta from "../../../src/delta/index.ts"; +import { decoder } from "../../../src/delta/index.ts"; +import { bool, evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type FacetHost, type Host, type Self } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant, type Writable } from "../../../src/next/state/index.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + fakeTimers, + flushed, + forEachCodec, + instancesOf, + linked, + PlacedNotes, + partValue, + RawPeer, + rawLinked, + rejection, + relayed, + SERVER_ADDRESS, + Server, + serveNotes, + statesOf, + type TestPrincipal, + testCodec, + transported, + valuesOf, + wireNames, +} from "../support.ts"; + +/** Every Delta encoder made while this file runs, weakly: the holds' member encoders among them. */ +const encoders = vi.hoisted(() => [] as WeakRef[]); + +vi.mock("../../../src/delta/index.ts", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + encoder: () => { + const made = original.encoder(); + encoders.push(new WeakRef(made)); + return made; + }, + }; +}); + +/** Every hold started while this file runs, weakly. */ +const holds = instancesOf(Hold, "start"); + +const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +/** `half` throws once `n` is 2: a member failing after the object is live (W1). */ +const Halves = remote("app.halves", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }), half: int() }, methods: { set: method(int()), wait: method() } } }, +}); + +/** The wire's indexes of `Halves`. */ +const HALVES = wireNames(Halves, "v"); + +/** A server serving `Halves` and a raw peer; `set` writes `n` through hold 0, the object of the link's principal. */ +async function halves() { + const linked = await rawLinked((h) => + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + wait: (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve(null))), + }, + }; + }), + ); + let calls = 0; + const set = async (value: number) => { + const call = calls++; + linked.peer.send({ kind: "call", call, hold: 0, method: HALVES.methods.set!, args: jsonPart(value) }); + expect((await linked.peer.next("result")).call).toBe(call); + }; + return { ...linked, set }; +} + +function hold(peer: RawPeer, id: number, Type: { name: string; version: number; fingerprint: string }) { + const { name: type, version, fingerprint } = Type; + peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); +} + +/** The provider object behind a hold, reached through the host's own lookup (to end it by hand). */ +function providerObject(server: unknown) { + const host = server as ChordHost; + const { name: type, version, fingerprint } = Halves; + const request = { type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }; + const served = host.serving(request, {}, { kind: "owner" }, context); + if (served.kind !== "object") throw new Error("Halves is served here"); + return served.object; +} + +/** A remote type whose activation never completes: its open waits for its abort. */ +const Pending = remote("app.pending", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +function servePending(h: Parameters[0]>[0]) { + h.serve(Pending, (_, _principal, self) => { + self.open( + (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }); +} + +forEachCodec(() => { + describe("the parts of a hold", () => { + it("are its members, each with its own encoder: the snapshot by member, then the changed members' ops", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + expect(HALVES.named((await peer.next("bound")).holds[0]!.members)).toEqual({ n: { n: 0 }, half: 0 }); + await set(1); + const first = HALVES.named((await peer.next("state")).holds[0]!.members); + expect(first).toEqual({ n: [["s", ["n"], 1]], half: [["r", 0.5]] }); + await set(4); + // The member's encoder interns a path on its second use. + const n = decoder(); + n.decode(first.n as unknown[]); + const second = HALVES.named((await peer.next("state")).holds[0]!.members); + expect(n.decode(second.n as unknown[])).toEqual([["s", ["n"], 4]]); + }); + }); + + describe("failed members", () => { + it("are listed in the header of the state that carries the failure, never in its part; they recover in the same hold", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + hold(peer, 1, Halves); + await peer.next("bound"); + await peer.next("bound"); + await set(2); + const state = await peer.next("state"); + const failed = [ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ]; + // Race 12: each hold's entry carries its healthy members' ops and the failure. + expect(state.holds.map(({ hold, failed }) => ({ hold, failed }))).toEqual([ + { hold: 0, failed }, + { hold: 1, failed }, + ]); + for (const entry of state.holds) expect(HALVES.named(entry.members)).toEqual({ n: [["s", ["n"], 2]] }); + await set(3); + const next = await peer.next("state"); + expect(next.holds.map(({ hold, failed }) => ({ hold, failed }))).toEqual([ + { hold: 0, failed: [] }, + { hold: 1, failed: [] }, + ]); + // The derive healed: its next value replaces the root. + expect(HALVES.named(next.holds[0]!.members).half).toEqual([["r", 1.5]]); + }); + + it("reach a client's watch over the link as one failure call, then the healed value: the watch stays open", async () => { + const { server, client } = await linked((h) => + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + wait: async () => null, + }, + }; + }), + ); + const halves = await client.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + const calls = changesOf(await halves.half.watch(context)); + await halves.set(2, context); + await transported(); + expect(calls).toEqual([{ kind: "failed", error: expect.objectContaining({ message: "half threw" }) }]); + await halves.set(6, context); + await transported(); + expect(halves.half.value).toBe(3); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + await client.close(context); + await server.close(context); + }); + + it("are listed in bound for a hold made after the failure", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + await peer.next("bound"); + await set(2); + await peer.next("state"); + hold(peer, 1, Halves); + const [bound] = (await peer.next("bound")).holds; + expect(bound!.failed).toEqual([ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ]); + expect(HALVES.named(bound!.members)).toEqual({ n: { n: 2 } }); + // Followed though it failed at the snapshot: it recovers in this hold. + await set(4); + const next = await peer.next("state"); + expect(HALVES.named(next.holds.find(({ hold }) => hold === 1)!.members).half).toEqual([["r", 2]]); + }); + }); + + describe("transient failures", () => { + // Timers stand still: the failed object's backoff window does not end while the test runs. + fakeTimers(); + + it("a new hold of an object failed transiently is sent the failure at once", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const { peer } = await rawLinked((h) => + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + hold(peer, 0, Flaky); + const cause = { + kind: "transient", + error: { kind: "chord", error: { code: "unavailable", message: "app.flaky: offline" } }, + }; + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause }); + hold(peer, 1, Flaky); + // The object stays failed for its backoff window (1 s); the new hold does not wait for the next attempt. + expect(await peer.next("failed")).toMatchObject({ hold: 1, cause }); + }); + }); + + // A hold whose object ends sends `gone` and nothing for its pending calls (race 4): through revocation, the path + // that ends a held object (`host/revocation.test.ts`, "aborts running calls"). + + describe("GC", () => { + it("a closed link retained by the test keeps no hold, encoder or object", async () => { + const { server, refs, link } = await (async () => { + const { server } = await halves(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const [held, made] = [holds.length, encoders.length]; + hold(peer, 0, Halves); + await peer.next("bound"); + const refs: WeakRef[] = [new WeakRef(providerObject(server)), ...holds.slice(held)]; + refs.push(...encoders.slice(made)); + expect(refs).toHaveLength(4); + peer.close(); + await transported(); + return { server, refs, link: a }; + })(); + await expectCollected(...refs); + expect([server, link]).toBeDefined(); + }); + + it("a released pending hold is kept neither by its object, still activating for another hold, nor by its waits", async () => { + const { server, peer, refs } = await (async () => { + const { server, peer } = await rawLinked((h) => servePending(h)); + hold(peer, 0, Pending); + await transported(); + const held = holds.length; + hold(peer, 1, Pending); + await transported(); + const refs = holds.slice(held); + expect(refs).toHaveLength(1); + peer.send({ kind: "release", hold: 1 }); + await transported(); + return { server, peer, refs }; + })(); + await expectCollected(...refs); + expect([server, peer]).toBeDefined(); + }); + + it("a pending activation on a closed link keeps no hold", async () => { + const { server, link, refs } = await (async () => { + const { server } = await rawLinked((h) => servePending(h)); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const held = holds.length; + hold(peer, 0, Pending); + await transported(); + const refs = holds.slice(held); + peer.close(); + await transported(); + return { server, link: a, refs }; + })(); + await expectCollected(...refs); + expect([server, link]).toBeDefined(); + }); + + it("a hold released while waiting for load is not kept by the wait", async () => { + const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); + const { server, refs } = await (async () => { + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + readyTimeout: 2_000, + onError: () => {}, + }); + const facet = defineFacet((h) => { + servePending(h); + // A host-lifetime object whose open never resolves: `load` waits for it (until its readyTimeout). + h.serve(Slow, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: constant(1) }; + }); + }); + void server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const held = holds.length; + hold(peer, 0, Pending); + await transported(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + return { server, refs: holds.slice(held) }; + })(); + expect(refs).toHaveLength(1); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); + +/** A remote type whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ +const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +function serveFlaky(h: Parameters[0]>[0], failing: { left: number; broken: boolean }) { + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + throw failing.broken ? broken(new Error("offline")) : new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }); +} + +/** A second raw peer on `server`, stamped with the same principal as `rawLinked`'s: it holds the same objects. */ +function secondPeer(server: Host): RawPeer { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + return new RawPeer(b); +} + +const offline = { code: "unavailable", message: "app.flaky: offline" }; + +forEachCodec(() => { + describe("notices of a failed object", () => { + fakeTimers(); + + it("its holds get failed with the rest of its window, loading for each attempt, then bound (race 6)", async () => { + const { peer } = await rawLinked((h) => serveFlaky(h, { left: 2, broken: false })); + hold(peer, 0, Flaky); + expect(await peer.next("failed")).toEqual({ + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: offline } }, + retryIn: 1_000, + }); + await vi.advanceTimersByTimeAsync(600); + hold(peer, 1, Flaky); + expect(await peer.next("failed")).toMatchObject({ hold: 1, retryIn: 400 }); + await vi.advanceTimersByTimeAsync(400); + await transported(); + expect(peer.kinds()).toEqual(["hello", "loading", "loading", "failed", "failed"]); + expect((await peer.next("failed")).retryIn).toBe(2_000); + await vi.advanceTimersByTimeAsync(2_000); + await peer.next("bound"); + await peer.next("bound"); + expect(peer.kinds()).toEqual(["hello", "loading", "loading", "failed", "loading", "loading"]); + }); + + it("a retry from one consumer renews a broken object for every holder: loading, then bound (races 12, 13)", async () => { + const failing = { left: 1, broken: true }; + const { server, peer } = await rawLinked((h) => serveFlaky(h, failing)); + const other = secondPeer(server); + hold(peer, 0, Flaky); + hold(other, 0, Flaky); + const broken = { + kind: "failed", + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: offline } }, + }; + expect([await peer.next("failed"), await other.next("failed")]).toEqual([broken, broken]); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + for (const consumer of [peer, other]) { + expect(await consumer.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect((await consumer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + } + }); + + it("a retry skips a transient failure's window for every holder (race 12)", async () => { + const { server, peer } = await rawLinked((h) => serveFlaky(h, { left: 1, broken: false })); + const other = secondPeer(server); + hold(peer, 0, Flaky); + await peer.next("failed"); + hold(other, 0, Flaky); + await other.next("failed"); + other.send({ kind: "retry", hold: 0, hops: 16 }); + for (const consumer of [peer, other]) { + await consumer.next("loading"); + await consumer.next("bound"); + } + }); + + it("a retry of a live or an unknown hold does nothing", async () => { + const { peer } = await rawLinked((h) => serveFlaky(h, { left: 0, broken: false })); + hold(peer, 0, Flaky); + await peer.next("bound"); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + peer.send({ kind: "retry", hold: 9, hops: 16 }); + await transported(); + expect([peer.kinds(), peer.closed]).toEqual([["hello"], false]); + }); + }); + + describe("holds that bind to nothing yet (dependency model §1, §3)", () => { + const Absent = { name: "app.absent", version: 1, fingerprint: "0123456789abcdef" }; + + it("a refused hold waits failed { waiting }, counted; a retry re-runs its start (loading, failed); its link's close drops it", async () => { + const { server, peer } = await rawLinked(() => {}); + hold(peer, 0, Absent); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(failed.retryIn).toBeUndefined(); + expect(server.activity.value.references).toBe(1); + expect(server.references(Halves).value).toEqual([]); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + expect(await peer.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + peer.close(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("a hold on an object parked on a failed dependency is told each new cause, then loading, then bound", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + let cores = 0; + const { peer } = await rawLinked((h) => { + h.serve(Core, () => { + if (++cores < 3) throw new Error(`bug ${cores}`); + return { n: constant(cores) }; + }); + h.serve(Halves, (_, _principal, self) => { + const core = self.get(Core, {}); + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([core.n], ([c]) => c), + set: async () => null, + wait: async () => null, + }, + }; + }); + }); + hold(peer, 0, Halves); + const waiting = (message: string) => ({ + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable", message } } }, + }); + // The core failed at load (its first outcome): the parked object names it at once. + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 1")); + // A retry passes on through the parked object to its broken dependency (one host's retry operation). + // (Its new attempt failed within the batch of changes the parked object checks once: no loading between.) + peer.send({ kind: "retry", hold: 0, hops: 16 }); + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 2")); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + await peer.next("loading"); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + }); + + describe("a reconnecting consumer", () => { + const Tally = remote("app.tally", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, + }); + const Fresh = remote("app.fresh", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, + }); + + /** A server serving `Tally` (lingering a minute) and `Fresh` (not lingering, its native's close waits for `closing`). */ + async function served(closing: Promise) { + const serve = (self: Self) => { + const doc = self.state(N); + return { + variant: "v" as const, + members: { + n: doc, + bump: async (ctx: Context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }, + }; + }; + return linked((h) => { + h.serve(Tally, (_, _principal, self) => serve(self), { linger: 60_000 }); + h.serve(Fresh, (_, _principal, self) => { + self.open( + async () => null, + () => closing, + ); + return serve(self); + }); + }); + } + + /** Drops the client's link and connects it again. */ + async function reconnect(server: Host, client: Host, link: Link) { + link.close(); + await transported(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await transported(); + } + + it("finds a lingering object as it left it: its in-memory documents kept; one without linger is new (race 7)", async () => { + const closing = Promise.resolve(); + const { server, client, links } = await served(closing); + const tally = await client.get(Tally, {}, context); + const fresh = await client.get(Fresh, {}, context); + await tally.bump(context); + await fresh.bump(context); + await reconnect(server, client, links[0]); + expect([tally.state.value, fresh.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + expect([tally.n.value, fresh.n.value]).toEqual([{ n: 1 }, { n: 0 }]); + }); + + it("re-holds an object whose previous instance still tears down: bound once that ended (race 17)", async () => { + const closing = Promise.withResolvers(); + const { server, client, links } = await served(closing.promise); + const fresh = await client.get(Fresh, {}, context); + await fresh.bump(context); + await reconnect(server, client, links[0]); + expect([fresh.state.value, fresh.n.value]).toEqual([{ kind: "loading" }, { n: 1 }]); + closing.resolve(); + await transported(); + expect([fresh.state.value, fresh.n.value]).toEqual([{ kind: "live" }, { n: 0 }]); + }); + }); +}); + +/** + * A remote type whose document can be retired; `big` derives a flag that a retirement of a small value leaves as is, + * `t` is `n` and throws while it is negative. + */ +const Retiring = remote("app.retiring", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + v: { + state: { doc: obj({ n: int() }), big: bool(), t: int() }, + methods: { set: method(int()), retire: method() }, + }, + }, +}); + +const RETIRING = wireNames(Retiring, "v"); + +/** Every member of `Retiring` retired with its document, sorted. */ +const retiredAll = [RETIRING.state.doc!, RETIRING.state.big!, RETIRING.state.t!].sort(); + +/** A server serving `Retiring` to a raw peer that acks only when told; `call` calls a method of hold 0. */ +async function retiring() { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((h) => + h.serve(Retiring, (_, _principal, self) => { + const doc = self.state(N); + return { + variant: "v", + members: { + doc, + big: self.derive([doc], ([{ n }]) => n > 10), + t: self.derive([doc], ([{ n }]) => { + if (n < 0) throw new Error(`failed ${n}`); + return n; + }), + set: async (n, ctx) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + retire: async (ctx) => { + await doc.retire(ctx); + return null; + }, + }, + }; + }), + ), + }, + ], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + await transported(); + let calls = 0; + const call = async (method: "set" | "retire", args?: number) => { + const id = calls++; + const payload = args === undefined ? {} : { args: jsonPart(args) }; + peer.send({ kind: "call", call: id, hold: 0, method: RETIRING.methods[method]!, ...payload }); + expect((await peer.next("result")).call).toBe(id); + }; + const { name: type, version, fingerprint } = Retiring; + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); + await peer.next("bound"); + peer.ack(); + return { peer, call }; +} + +forEachCodec(() => { + describe('retired members (decision "One retirement mechanism")', () => { + it("are listed in the state that carries the retirement, a derive whose value stayed without ops", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + const set = await peer.next("state"); + expect(set.holds[0]!.retired).toEqual([]); + peer.ack(); + await call("retire"); + const retired = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(retired.members)).toEqual({ doc: [["r", { n: 0 }]], t: [["r", 0]] }); + expect(retired.retired.toSorted()).toEqual(retiredAll); + }); + + it("are kept while the window is full and sent once with the folded ops", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // The window is full: the retirement and the next change wait, folded into one entry. + await call("retire"); + await call("set", 4); + await transported(); + expect(peer.kinds().filter((kind) => kind === "state")).toEqual([]); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(pending.members)).toEqual({ + doc: [ + ["r", { n: 0 }], + ["s", ["n"], 4], + ], + t: [["r", 4]], + }); + expect(pending.retired.toSorted()).toEqual(retiredAll); + peer.ack(); + peer.ack(); + await call("set", 5); + expect((await peer.next("state")).holds[0]!.retired).toEqual([]); + }); + + it("of a member failed again before its entry leaves wait for its next value, never with its failure", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // The window is full: `t` fails, heals with the retirement (n is 0 again), and fails again. + await call("set", -1); + await call("retire"); + await call("set", -3); + peer.ack(); + const failed = (await peer.next("state")).holds[0]!; + expect(failed.failed).toEqual([ + { member: RETIRING.state.t!, error: { kind: "plugin", code: "internal", message: "failed -3" } }, + ]); + expect(Object.keys(RETIRING.named(failed.members))).toEqual(["doc"]); + expect(failed.retired.toSorted()).toEqual([RETIRING.state.doc!, RETIRING.state.big!].sort()); + peer.ack(); + peer.ack(); + await call("set", 5); + const healed = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(healed.members).t).toEqual([["r", 5]]); + expect(healed.retired).toEqual([RETIRING.state.t!]); + }); + }); + + describe("failures while the window is full", () => { + it("fold per member: 1000 failing commits leave one failure entry, the member's ops dropped; bounded", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // An earlier episode, healed: its failure leaves with the heal. + await call("set", -1); + await call("set", 3); + for (let n = 1; n <= 1000; n++) await call("set", -1000 - n); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + // One failure call per episode: the error of the commit that started it. + expect(pending.failed).toEqual([ + { member: RETIRING.state.t!, error: { kind: "plugin", code: "internal", message: "failed -1001" } }, + ]); + expect(Object.keys(RETIRING.named(pending.members))).not.toContain("t"); + expect(peer.codec.encode({ kind: "state", holds: [pending] }).byteLength).toBeLessThan(200); + }); + + it("are replaced by a value that follows them: the value as a root replacement, no failure", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + await call("set", -1); + await call("set", 3); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + expect(pending.failed).toEqual([]); + expect(RETIRING.named(pending.members).t).toEqual([["r", 3]]); + }); + }); +}); + +/** Lets every pending reaction run (a memory link delivers by microtask). */ +async function flush(): Promise { + for (let i = 0; i < 100; i++) await Promise.resolve(); +} + +/** A local type nobody serves. */ +const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + +describe("retries across links: the hop budget (audit A5)", () => { + it.each([1, 9, 50])( + "quiesce after %i concurrent Retries round a waiting ring across hosts: each stops at its budget, no attempt started", + async (operations) => { + type P = { kind: "server" } | { kind: "worker" }; + const Server = hostKind("app.retry-server"); + const Worker = hostKind("app.retry-worker"); + const LocalA = local("app.a", { version: 1, params: {}, members: { n: int() } }); + const LocalB = local("app.b", { version: 1, params: {}, members: { n: int() } }); + const RemoteA = remote("app.remote-a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { reader: { state: { n: int() } } }, + }); + const RemoteB = remote("app.remote-b", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { reader: { state: { n: int() } } }, + }); + const hops: number[] = []; + let factories = 0; + const count = (link: Link): Link => ({ + send: (bytes) => { + const message = binaryCodec.decode(bytes); + if (message.kind === "retry") hops.push(message.hops); + link.send(bytes); + }, + onMessage: (listener) => link.onMessage(listener), + onClose: (listener) => link.onClose(listener), + close: () => link.close(), + }); + const [down, up] = memoryLinkPair(); + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + server.route( + Worker, + { + authorize: () => true, + resolve: async () => ({ link: count(down), principal: { kind: "worker" }, trusted: true }), + }, + context, + ); + worker.connect(count(up), { principal: { kind: "server" }, trusted: true }, context); + try { + const left = server.load( + [ + { + name: "@test/a", + facet: defineFacet

((h) => { + h.serve(LocalA, (_, self) => { + factories++; + return { n: self.get(RemoteB, {}).n }; + }); + h.serve(RemoteA, (_, _principal, self) => { + factories++; + return { variant: "reader", members: { n: self.get(LocalA, {}).n } }; + }); + }), + }, + ], + context, + ); + const right = worker.load( + [ + { + name: "@test/b", + facet: defineFacet

((h) => { + h.serve(LocalB, (_, self) => { + factories++; + self.get(Missing, {}); + return { n: self.get(RemoteA, {}).n }; + }); + h.serve(RemoteB, (_, _principal, self) => { + factories++; + return { variant: "reader", members: { n: self.get(LocalB, {}).n } }; + }); + }), + }, + ], + context, + ); + await Promise.all([left, right]); + await flush(); + const a = server.hold(LocalA, {}, context); + expect(a.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const made = factories; + for (let i = 0; i < operations; i++) a.retry(); + await flush(); + // Each passed round the ring with one hop less per host (16 from the first), never sent at 0. + const lap = Array.from({ length: 16 }, (_, i) => 16 - i); + expect(hops.toSorted((x, y) => y - x)).toEqual(lap.flatMap((hop) => Array(operations).fill(hop))); + await flush(); + expect(hops).toHaveLength(16 * operations); + // Parked objects only passed it on: no attempt started. + expect(factories).toBe(made); + } finally { + await server.close(context); + await worker.close(context); + } + }, + ); +}); + +/** A remote object on the server: a label, a counter and `bump`. */ +const Shown = remote("app.shown", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { label: str(), n: int() }, methods: { bump: method(undefined, int()) } } }, +}); + +/** `Shown` v2: its label became an object; v1 consumers map back through `accepts`. */ +const ShownV2 = evolve(Shown, (e) => e.state("v", "label", obj({ text: str() }))); + +/** Serves `Shown` labelled `label` (with a native when `native`); `bump` adds 1 to its in-memory counter. */ +function serveShown(h: FacetHost, label: string, native = false): void { + h.serve(Shown, (_, _principal, self) => { + const doc = self.state(N); + if (native) + self.open( + async () => label, + () => {}, + ); + return { + variant: "v", + members: { + label: constant(label), + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }, + }; + }); +} + +/** Serves `ShownV2` labelled `text`; with `accepts`, v1 consumers see `label` as its text. */ +function serveShownV2(h: FacetHost, text: string, accept: boolean): void { + h.serve( + ShownV2, + (_, _principal, self) => { + const doc = self.state(N); + return { + variant: "v", + members: { + label: constant({ text }), + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }, + }; + }, + accept ? { accepts: { 1: { state: { v: { label: (current) => current.text } } } } } : {}, + ); +} + +/** Reloads the server's plugin `@test/app` (as `linked` names it) with `setup`. */ +function reloadServer(server: Host, setup: (host: FacetHost) => void) { + return server.reload([{ name: "@test/app", facet: defineFacet(setup) }], context); +} + +describe("holds across a reload (slice 7b §6)", () => { + it("a swapped object keeps its holds: a fresh bound once the new instance is live, never loading (audit A9)", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1")); + const shown = await client.get(Shown, {}, context); + await shown.bump(context); + const states = statesOf(await shown.state.watch(context)); + const label = changesOf(await shown.label.watch(context)); + const n = changesOf(await shown.n.watch(context)); + await reloadServer(server, (h) => serveShown(h, "v2")); + await transported(); + expect(label).toEqual([{ kind: "value", value: "v2", ops: [["r", "v2"]], retired: false }]); + // The fresh bound carries every member: an unchanged one may show one frame (nothing lost or stale). + expect(valuesOf(n)).toEqual([1]); + expect(states).toEqual([]); + expect(await shown.bump(context)).toBe(2); + await server.close(context); + }); + + it("a restarted object's holds see loading, then a fresh bound", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1", true)); + const shown = await client.get(Shown, {}, context); + const states = statesOf(await shown.state.watch(context)); + const label = changesOf(await shown.label.watch(context)); + await reloadServer(server, (h) => serveShown(h, "v2", true)); + await transported(); + expect(states).toEqual(["loading", "live"]); + expect(valuesOf(label)).toEqual(["v2"]); + await server.close(context); + }); + + it("a version the new code no longer accepts unbinds its holds (version); a later reload accepting it binds them", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1")); + const shown = await client.get(Shown, {}, context); + await reloadServer(server, (h) => serveShownV2(h, "v2", false)); + await transported(); + expect(shown.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + expect(await rejection(shown.bump(context))).toMatchObject({ code: "contract_mismatch" }); + await reloadServer(server, (h) => serveShownV2(h, "v3", true)); + await transported(); + expect(shown.state.value).toEqual({ kind: "live" }); + expect(shown.label.value).toBe("v3"); + await server.close(context); + }); + + it("a removed type unbinds its holds (not_served); served again, they bind and a derive over them heals", async () => { + const Panel = local("app.panel", { version: 1, params: {}, members: { text: str() } }); + const { server, client } = await linked((h) => serveShown(h, "v1")); + await client.reload( + [ + { + name: "@test/panel", + facet: defineFacet((h) => + h.serve(Panel, (_, self) => { + const shown = self.get(Shown, {}); + return { text: self.derive([shown.label], ([label]) => `panel ${label}`) }; + }), + ), + }, + ], + context, + ); + const panel = await client.get(Panel, {}, context); + const shown = client.hold(Shown, {}, context); + await server.reload([], context); + await transported(); + expect(shown.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable" } }, + }); + // The dependent stays live (R18) with its last value. + expect(panel.state.value).toEqual({ kind: "live" }); + expect(panel.text.value).toBe("panel v1"); + await server.reload( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v2")) }], + context, + ); + await transported(); + expect(shown.state.value).toEqual({ kind: "live" }); + expect(panel.text.value).toBe("panel v2"); + await server.close(context); + }); + + it("a hold unbound while the window is full drops its pending ops; acks of messages sent stay counted", async () => { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v1")) }], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + const SHOWN = wireNames(Shown, "v"); + await peer.next("hello"); + hold(peer, 0, Shown); + await peer.next("bound"); + for (let call = 0; call < 3; call++) { + peer.send({ kind: "call", call, hold: 0, method: SHOWN.methods.bump! }); + await peer.next("result"); + } + // One state in flight with the bound: the third commit's ops wait in their pending queue. + await peer.next("state"); + await server.reload([], context); + await transported(); + expect((await peer.next("failed")).cause).toMatchObject({ kind: "waiting" }); + peer.ack(); + peer.ack(); + await transported(); + expect(peer.kinds()).toEqual([]); + await server.close(context); + }); + + it("a swap sends each bound hold one fresh bound with the new instance's values, no state (audit A9)", async () => { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v1")) }], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const SHOWN = wireNames(Shown, "v"); + await peer.next("hello"); + hold(peer, 0, Shown); + hold(peer, 1, Shown); + await peer.next("bound"); + await transported(); + peer.received.length = 0; + await reloadServer(server, (h) => serveShown(h, "v2")); + await transported(); + expect(new Set(peer.kinds())).toEqual(new Set(["bound"])); + const entries = peer.received.flatMap((message) => (message.kind === "bound" ? message.holds : [])); + expect(entries.map(({ hold }) => hold).sort()).toEqual([0, 1]); + for (const entry of entries) expect(SHOWN.named(entry.members)).toEqual({ label: "v2", n: 0 }); + await server.close(context); + }); + + it("a swap during a full window: calls are answered by the old names until the fresh bound goes (audit A9)", async () => { + const Two = remote("app.two-variants", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + a: { state: { label: str() }, methods: { bump: method(undefined, int()), ping: method(undefined, int()) } }, + b: { state: { label: str() }, methods: { ping: method(undefined, int()) } }, + }, + }); + const serveTwo = (variant: "a" | "b", label: string) => + defineFacet((h) => + h.serve(Two, (_, _principal, self) => { + const doc = self.state(N); + const ping = async () => 7; + const bump = async (ctx: Context) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }; + return variant === "a" + ? { variant, members: { label: constant(label), bump, ping } } + : { variant, members: { label: constant(label), ping } }; + }), + ); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: serveTwo("a", "v1") }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + const A = wireNames(Two, "a"); + await peer.next("hello"); + hold(peer, 0, Two); + await peer.next("bound"); + // A second message in flight: the window (two) is full. + const changed = defineFacet((h) => + h.serve(Two, (_, _principal, self) => ({ + variant: "a", + members: { + label: self.derive([self.state(N)], ([d]) => `n ${d.n}`), + bump: async () => 0, + ping: async () => 7, + }, + })), + ); + await server.reload([{ name: "@test/app", facet: changed }], context); + await peer.next("bound"); + // Swaps in place while the window is full: the fresh bound waits for a slot, the hold answers by its old names. + await server.reload([{ name: "@test/app", facet: serveTwo("a", "v2") }], context); + await transported(); + peer.send({ kind: "call", call: 0, hold: 0, method: A.methods.bump! }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + // The new variant lacks a method of the old names: the attempt refuses it. + await server.reload([{ name: "@test/app", facet: serveTwo("b", "v3") }], context); + await transported(); + peer.send({ kind: "call", call: 1, hold: 0, method: A.methods.bump! }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "invalid", reason: "member" } } }); + peer.send({ kind: "call", call: 2, hold: 0, method: A.methods.ping! }); + expect(partValue((await peer.next("result")).value!)).toBe(7); + expect(peer.kinds().filter((kind) => kind === "bound")).toEqual([]); + peer.ack(); + const fresh = await peer.next("bound"); + expect(fresh.holds[0]).toMatchObject({ hold: 0, variant: wireNames(Two, "b").variant }); + expect(wireNames(Two, "b").named(fresh.holds[0]!.members)).toEqual({ label: "v3" }); + await server.close(context); + }); + + it("collects the swapped-out instance while the hold's fresh bound waits for a slot (Astra, audit A9 GC)", async () => { + const Type = remote("astra.audit-gc", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { n: int() } } }, + }); + let declared: Self | undefined; + const facet = (n: number) => + defineFacet((h) => + h.serve(Type, (_, _principal, self) => { + declared = self; + return { variant: "main", members: { n: constant(n) } }; + }), + ); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@astra/audit-gc", facet: facet(1) }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + await peer.next("hello"); + hold(peer, 0, Type); + await peer.next("bound"); + // A second bound in flight: the window is full. + await server.reload([{ name: "@astra/audit-gc", facet: facet(2) }], context); + await peer.next("bound"); + const retired = new WeakRef(declared!); + declared = undefined; + await server.reload([{ name: "@astra/audit-gc", facet: facet(3) }], context); + await transported(); + await flushed(); + // Its fresh bound waits for an ack; the hold keeps the old names for calls, not the old instance. + expect(peer.kinds().filter((kind) => kind === "bound")).toEqual([]); + await expectCollected(retired); + peer.ack(); + await peer.next("bound"); + await server.close(context); + }); + + it("through a relay: a worker's reload reaches the tab as it reaches a direct consumer", async () => { + const { tab, workers, server } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const worker = workers.get("session|S1")!; + const states = statesOf(await notes.state.watch(context)); + await worker.reload([], context); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await worker.reload( + [{ name: "@test/worker", facet: defineFacet((h) => void serveNotes(h, PlacedNotes)) }], + context, + ); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(states.at(-1)).toBe("live"); + await server.close(context); + }); +}); + +describe("holds across more than one swap (fix round 1)", () => { + it("keeps a real subscriber when a changed constant later becomes dynamic", async () => { + type P = { kind: "server" } | { kind: "tab" }; + const Server = hostKind("sol-server"); + const Type = remote("sol.wire", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: int() }, methods: { set: method(int()) } } }, + }); + const Doc = defineState("wire", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const facet = (choice: { kind: "constant"; n: number } | { kind: "dynamic" }) => + defineFacet

((h) => + h.serve(Type, (_, _principal, self) => { + const doc = self.state(Doc); + return { + variant: "main", + members: { + x: choice.kind === "constant" ? constant(choice.n) : self.derive([doc], ([value]) => value.n), + set: async (n: number, ctx: Context) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }, + }; + }), + ); + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const tab = createHost

({ onError: () => {} }); + try { + await server.load([{ name: "@sol/wire", facet: facet({ kind: "constant", n: 1 }) }], context); + await tab.load([], context); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "tab" } }, context); + tab.connect(down, { principal: { kind: "server" } }, context); + const held = await tab.get(Type, {}, context); + await server.reload([{ name: "@sol/wire", facet: facet({ kind: "constant", n: 2 }) }], context); + await delivered(); + expect(held.x.value).toBe(2); + await server.reload([{ name: "@sol/wire", facet: facet({ kind: "dynamic" }) }], context); + await delivered(); + expect(held.x.value).toBe(0); + await held.set(7, context); + await delivered(); + expect(held.x.value).toBe(7); + } finally { + await tab.close(context); + await server.close(context); + } + }); +}); + +describe("a hold after its object's live failure (final review)", () => { + const attempted = instancesOf(ChordObject, "attempt"); + + it("follows a member re-exposing another object's source until the admitted commits published (audit A10)", async () => { + const Point = obj({ n: int() }); + const A = local("app.foreign", { version: 1, params: {}, members: { x: Point } }); + const B = remote("app.reexposing", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: Point } } }, + }); + let foreign: Writable<{ n: number }> | undefined; + let self: Self | undefined; + const { server, client } = await linked((h) => { + h.serve(A, (_, s) => { + foreign = s.writable({ n: 0 }); + return { x: s.derive([foreign], ([value]) => value) }; + }); + h.serve(B, (_, _principal, s) => { + self = s; + return { variant: "main", members: { x: s.get(A, {}).x } }; + }); + }); + const ref = await client.get(B, {}, context); + const calls = changesOf(await ref.x.watch(context)); + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + // A's change lands after B failed, in the window before B's admitted commits published: it shows (one more + // change), then the member holds with the failure. + foreign!.set({ n: 1 }); + await delivered(); + expect([ref.x.value, valuesOf(calls)]).toEqual([{ n: 1 }, [{ n: 1 }, { failed: "boom" }]]); + await client.close(context); + await server.close(context); + }); +}); + +describe("a hold across a swap whose mapping fails (final review)", () => { + it("gets the member's failure in its fresh bound and heals with the new source (Sol, audit A9)", async () => { + const W1 = remote("app.swap-failed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: int() } } }, + }); + const W2 = evolve(W1, (e) => e.state("main", "x", obj({ n: int() }))); + let throws = true; + let point: Writable<{ n: number }> | undefined; + const serve = (h: FacetHost, map: (value: { n: number }) => number) => + h.serve( + W2, + (_, _principal, s) => { + point = s.writable({ n: 1 }); + return { variant: "main", members: { x: s.derive([point], ([value]) => value) } }; + }, + { accepts: { 1: { state: { main: { x: map } } } } }, + ); + const { server, client } = await linked((h) => serve(h, (value) => value.n)); + const ref = await client.get(W1, {}, context); + const calls = changesOf(await ref.x.watch(context)); + // The new instance's mapping throws for its first value. + const failing = (value: { n: number }) => { + if (throws) throw new Error("member boom"); + return value.n; + }; + await server.reload( + [{ name: "@test/app", facet: defineFacet((h) => serve(h, failing)) }], + context, + ); + await delivered(); + // Live, the member failed: its watch got the failure, a read throws it (as a failed member does on the host). + expect([ref.state.value.kind, valuesOf(calls)]).toEqual(["live", [{ failed: "member boom" }]]); + expect(() => ref.x.value).toThrow("member boom"); + // It follows the new source: the next value heals it. + throws = false; + point!.set({ n: 3 }); + await delivered(); + expect([ref.x.value, valuesOf(calls)]).toEqual([3, [{ failed: "member boom" }, 3]]); + await client.close(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/remote/host-link.test.ts b/packages/chord/test/next/remote/host-link.test.ts new file mode 100644 index 000000000..874287405 --- /dev/null +++ b/packages/chord/test/next/remote/host-link.test.ts @@ -0,0 +1,509 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type FacetHost, type HostError } from "../../../src/next/host/index.ts"; +import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + delivered, + EDITOR, + forEachCodec, + framesOf, + incarnationOf, + Notes, + PlacedNotes, + partValue, + RAW_PEER, + RawPeer, + rawLinked, + rawWorkers, + rejection, + relayed, + SERVER_ADDRESS, + Server, + Session, + serveNotes, + settledHeap, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +/** A server type that tells who holds it, and echoes as whom it is called. */ +const ServerWho = remote("app.server-who", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { who: str() }, methods: { echo: method(str(), str()) } } }, +}); + +/** A worker type that asks the server, from a handler (`ask`) and from its factory (`who`). */ +const Asker = remote("app.asker", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { who: str() }, methods: { ask: method(str(), str()) } } }, +}); + +/** A worker type that reads another session's notes through the server. */ +const Neighbour = remote("app.neighbour", { + version: 1, + params: { session: str(), other: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { count: int() } } }, +}); + +function serveWho(h: FacetHost): void { + h.serve(ServerWho, (_, principal) => ({ + variant: "v", + members: { who: constant(principal.kind), echo: async (text, ctx) => `${ctx.principal?.kind}:${text}` }, + })); +} + +function serveAsker(h: FacetHost): void { + h.serve(Asker, (_, _principal, self) => { + const server = self.get(ServerWho, {}); + return { + variant: "v", + members: { + who: server.who, + ask: async (text, ctx) => (await ctx.get(ServerWho, {})).echo(text, ctx), + }, + }; + }); + h.serve(Neighbour, ({ other }, _principal, self) => { + const notes = self.get(PlacedNotes, { session: other }); + return { variant: "v", members: { count: notes.count } }; + }); +} + +forEachCodec(() => { + describe("two-sided links", () => { + it("carry the server's holds of a worker type and the worker's holds of a server type over one route link", async () => { + const { tab, routeLinks } = await relayed({ server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + // The worker's factory held `ServerWho` over its parent link, the route link the server made. + expect(asker.who.value).toBe("worker"); + expect(routeLinks).toHaveLength(1); + }); + + it("carry calls both ways with equal ids: each side's ids are its own", async () => { + const { tab } = await relayed({ server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + // Server to worker: call 0 on the route link; inside it, worker to server: call 0 the other way. + expect(await asker.ask("x", context)).toBe("worker:x"); + expect(await Promise.all([asker.ask("a", context), asker.ask("b", context)])).toEqual([ + "worker:a", + "worker:b", + ]); + }); + + it("let a worker reach another worker through the server", async () => { + const { tab, workers, resolves } = await relayed({ worker: serveAsker }); + const neighbour = await tab.get(Neighbour, { session: "S1", other: "S2" }, context); + expect(resolves).toEqual(["session|S1", "session|S2"]); + await addNote(workers.get("session|S2")!, "S2", "a"); + await transported(); + expect(neighbour.count.value).toBe(1); + }); + + it("apply this host's own messages and forward the tab origin's, each acknowledged by its consumer", async () => { + const peers: RawPeer[] = []; + const { tab, server } = await relayed({ resolve: rawWorkers(peers) }); + const own = server.get(PlacedNotes, { session: "S1" }, context); + await transported(); + const [worker] = peers; + const first = await worker!.next("hold"); + bindPlaced(worker!, first); + const relayedHold = tab.get(PlacedNotes, { session: "S1" }, context); + const second = await worker!.next("hold"); + expect([first.origin, second.origin]).toEqual([undefined, 1]); + bindPlaced(worker!, second); + const [serverNotes, tabs] = [await own, await relayedHold]; + await transported(); + worker!.received.length = 0; + const ops = jsonPart([["r", 5]]); + for (const { hold, origin } of [first, second]) { + const from = origin === undefined ? {} : { origin }; + const members = [{ member: EDITOR.state.count!, ops }]; + worker!.send({ kind: "state", ...from, holds: [{ hold, members, failed: [], retired: [] }] }); + } + await transported(); + expect(serverNotes.count.value).toBe(5); + expect(tabs.count.value).toBe(5); + expect(worker!.received).toEqual([{ kind: "ack" }, { kind: "ack", origin: 1 }]); + }); + }); + + describe("principals on holds (section 4's table)", () => { + /** A server serving a guest-facing gateway and a local view, both reaching `PlacedNotes` on a raw worker. */ + async function principals(tab: TestPrincipal = { kind: "guest", session: "S1" }) { + const peers: RawPeer[] = []; + const Gateway = remote("app.gateway", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() }, methods: { open: method(str(), int()) } } }, + }); + // With params: not activated by `load`, which would wait for the raw worker. + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + const relay = await relayed({ + tab, + resolve: rawWorkers(peers), + server: (h) => { + h.serve(Gateway, (_, _principal, self) => { + // `self.get` in a remote object: the object's principal, this host's incarnation. + const own = self.get(PlacedNotes, { session: "own" }); + return { + variant: "v", + members: { + n: own.count, + // `ctx.get` in a handler: the caller's principal and incarnation. + open: async (session, ctx) => (await ctx.get(PlacedNotes, { session })).count.value, + }, + }; + }); + h.serve(View, (_, self) => ({ n: self.get(PlacedNotes, { session: "view" }).count })); + }, + }); + return { ...relay, peers, Gateway, View }; + } + + /** The next hold the raw worker of `peers[index]` receives, bound at once, with whom it acts for. */ + async function nextHold(peers: RawPeer[], index = 0) { + for (let tries = 0; peers[index] === undefined && tries < 10; tries++) await transported(); + const hold = await peers[index]!.next("hold"); + bindPlaced(peers[index]!, hold); + const { session } = partValue(hold.params) as { session: string }; + return { session, principal: hold.principal, incarnation: hold.incarnation }; + } + + it("a relayed hold acts as the link's principal: a forged principal from a tab is ignored (race 9)", async () => { + const { server, peers } = await principals(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const tab = new RawPeer(b); + tab.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "owner" }, + }); + expect(await nextHold(peers)).toEqual({ + session: "S1", + principal: { kind: "guest", session: "S1" }, + incarnation: RAW_PEER, + }); + }); + + it("a hold relayed over a parent path names nobody: a child's principal and incarnation stay behind", async () => { + const worker = createHost({ principal: { kind: "worker", session: "S1" }, onError: () => {} }); + await worker.load([], context); + const [up, parentEnd] = memoryLinkPair(); + worker.connect(up, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(parentEnd); + const [a, b] = memoryLinkPair(); + worker.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const child = new RawPeer(b); + child.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "owner" }, + incarnation: RAW_PEER, + }); + const relayedHold = await parent.next("hold"); + expect(relayedHold).toMatchObject({ hold: 0, origin: 1, type: PlacedNotes.name }); + expect("principal" in relayedHold).toBe(false); + expect("incarnation" in relayedHold).toBe(false); + await worker.close(context); + }); + + it("host.get through a route acts as the host: its principal and incarnation", async () => { + const { server, peers } = await principals(); + void server.get(PlacedNotes, { session: "S1" }, context); + expect(await nextHold(peers)).toEqual({ + session: "S1", + principal: { kind: "server" }, + incarnation: incarnationOf(server), + }); + }); + + it("ctx.get in a guest's call acts as the guest, with the tab's incarnation (race 10)", async () => { + const { tab, peers, Gateway } = await principals(); + const got = tab.get(Gateway, {}, context); + expect(await nextHold(peers)).toMatchObject({ session: "own" }); + const opened = (await got).open("S1", context); + // Another address: another worker. + expect(await nextHold(peers, 1)).toEqual({ + session: "S1", + principal: { kind: "guest", session: "S1" }, + incarnation: incarnationOf(tab), + }); + expect(await opened).toBe(0); + }); + + it("self.get in a remote object acts as that object's principal, with this host's incarnation", async () => { + const { tab, server, peers, Gateway } = await principals(); + tab.hold(Gateway, {}, context); + expect(await nextHold(peers)).toEqual({ + session: "own", + principal: { kind: "guest", session: "S1" }, + incarnation: incarnationOf(server), + }); + }); + + it("self.get in a local object acts as the host", async () => { + const { server, peers, View } = await principals(); + void server.get(View, { id: 1 }, context); + expect(await nextHold(peers)).toEqual({ + session: "view", + principal: { kind: "server" }, + incarnation: incarnationOf(server), + }); + }); + + it("two origins with equal principal keys but other incarnations are two upstream holds", async () => { + const { tab, server, peers } = await principals({ kind: "owner" }); + const other = createHost({ onError: () => {} }); + await other.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + other.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + void tab.get(PlacedNotes, { session: "S1" }, context); + const first = await nextHold(peers); + void other.get(PlacedNotes, { session: "S1" }, context); + const second = await nextHold(peers); + expect([first, second]).toEqual([ + { session: "S1", principal: { kind: "owner" }, incarnation: incarnationOf(tab) }, + { session: "S1", principal: { kind: "owner" }, incarnation: incarnationOf(other) }, + ]); + }); + + it("upward is a boundary: a worker's hold of a server type acts as the worker, whoever it serves (race 11)", async () => { + const { tab } = await relayed({ tab: { kind: "guest", session: "S1" }, server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + expect(await asker.ask("x", context)).toBe("worker:x"); + }); + }); + + describe("trusted links", () => { + it("work as untrusted ones do", async () => { + const { tab } = await relayed({ trusted: true }); + expect((await tab.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("editor"); + }); + }); +}); + +/** Binds a hold a raw worker received (its id and origin): `PlacedNotes`'s editor variant, no notes. */ +function bindPlaced(peer: RawPeer, { hold, origin }: { readonly hold: number; readonly origin?: number }): void { + const members = [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ]; + const from = origin === undefined ? {} : { origin }; + peer.send({ kind: "bound", ...from, holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); +} + +/** A compressor that counts what it does (`nodeCompression` underneath). */ +function counting(): Compression & { deflates: number; inflates: number } { + const counts = { + deflates: 0, + inflates: 0, + deflate: (bytes: Uint8Array) => { + counts.deflates++; + return nodeCompression.deflate(bytes); + }, + inflate: (bytes: Uint8Array, limit: number) => { + counts.inflates++; + return nodeCompression.inflate(bytes, limit); + }, + }; + return counts; +} + +/** A note text large enough that a notes list with it is deflated at its origin. */ +const LARGE = "lorem ipsum ".repeat(2000); + +/** + * A server serving `Notes` with `compression`, and a client with its own `compression` (none: it inflates + * asynchronously), its parent link's limits `limits`. + */ +async function compressing(options: { + readonly server?: Compression; + readonly client?: Compression; + readonly limits?: PartLimits; +}) { + const errors: HostError[] = []; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + compression: options.server, + onError: (error) => errors.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const client = createHost({ compression: options.client, onError: (error) => errors.push(error) }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec(), limits: options.limits }, context); + return { server, client, errors }; +} + +forEachCodec(() => { + describe("parts: compression at the origin and limits", () => { + it("deflate JSON parts above 16 KiB at their origin and inflate them where owned, synchronously; never up a parent link's arguments", async () => { + const [server, client] = [counting(), counting()]; + const hosts = await compressing({ server, client }); + await addNote(hosts.server, "S1", LARGE); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("an owner's notes are the editor's"); + expect(notes.notes.value).toEqual([{ text: LARGE }]); + expect([server.deflates, client.inflates]).toEqual([1, 1]); + // A large argument goes up the client's parent link uncompressed: child links refuse compressed parts. + await notes.add({ text: LARGE }, context); + expect([client.deflates, server.inflates]).toEqual([0, 0]); + await transported(); + expect(notes.notes.value).toEqual([{ text: LARGE }, { text: LARGE }]); + }); + + it("inflate asynchronously without a compressor: later messages of the link wait, in order", async () => { + const hosts = await compressing({ server: nodeCompression }); + await addNote(hosts.server, "S1", LARGE); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + expect(notes.notes.value).toEqual([{ text: LARGE }]); + const counts = framesOf(await notes.count.watch(context)); + // Each commit sends the whole changed list (a derive's diff appends one note): large ones deflated. + for (let i = 0; i < 5; i++) await addNote(hosts.server, "S1", i % 2 === 0 ? LARGE : "small"); + for (let i = 0; i < 100 && notes.count.value < 6; i++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(notes.count.value).toBe(6); + expect(notes.notes.value.map(({ text }) => text.length)).toEqual( + [LARGE, LARGE, "small", LARGE, "small", LARGE].map((text) => text.length), + ); + await delivered(); + const seen = counts.map(([count]) => count); + expect(seen).toEqual([...seen].sort()); + }); + + it("fail only the hold of a compressed bomb, on either inflate path (race 11): the link stays", async () => { + for (const client of [nodeCompression, undefined]) { + const hosts = await compressing({ server: nodeCompression, client, limits: { decoded: 100_000 } }); + await addNote(hosts.server, "S1", "x".repeat(200_000)); + await addNote(hosts.server, "S2", "small"); + const error = await rejection(hosts.client.get(Notes, { session: "S1" }, context)); + expect(error).toMatchObject({ code: "too_large", reason: "part" }); + const other = await hosts.client.get(Notes, { session: "S2" }, context); + expect(other.notes.value).toEqual([{ text: "small" }]); + } + }); + + it("cost nothing for uncompressed messages at a host without a compressor: no inflate, no stream", async () => { + const inflated = countInflated(); + try { + const hosts = await compressing({ server: nodeCompression }); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + for (let i = 0; i < 5; i++) await addNote(hosts.server, "S1", "small"); + await transported(); + expect(notes.count.value).toBe(5); + expect(inflated.bytes).toBe(0); + expect(hosts.errors).toEqual([]); + } finally { + inflated.restore(); + } + }); + + it("send every part uncompressed from a host without a compressor", async () => { + const server = counting(); + const hosts = await compressing({ server }); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("an owner's notes are the editor's"); + await notes.add({ text: LARGE }, context); + expect(server.inflates).toBe(0); + }); + }); +}); + +describe("origins only from relays (decision, slice 10 security audit K1/K2)", () => { + const { name: type, version, fingerprint } = Notes; + const hold = { + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; + const withOrigin: Record = { + hold: { ...hold, origin: 1 } as Message, + call: { kind: "call", origin: 1, call: 0, hold: 0, method: EDITOR.methods.wait }, + release: { kind: "release", origin: 1, hold: 0 }, + abort: { kind: "abort", origin: 1, call: 0 }, + retry: { kind: "retry", origin: 1, hold: 0, hops: 1 } as Message, + ack: { kind: "ack", origin: 1 }, + closed: { kind: "closed", origin: 1 }, + }; + + for (const [kind, message] of Object.entries(withOrigin)) { + it(`closes a child link sending ${kind} with an origin (a protocol error)`, async () => { + const { peer, errors } = await rawLinked((h) => void serveNotes(h)); + peer.send(message); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: `${kind}: an origin on a child link` } }, + ]); + }); + } + + it("honours origins on a parent link (a relay's): one window of calls per origin", async () => { + let probe!: ReturnType; + const { peer } = await rawLinked( + (h) => { + probe = serveNotes(h); + }, + { role: "parent" }, + ); + peer.send({ ...hold, origin: 1 } as Message); + peer.send({ ...hold, origin: 2 } as Message); + await transported(); + for (const origin of [1, 2]) peer.send({ kind: "call", origin, call: 0, hold: 0, method: EDITOR.methods.wait }); + await transported(); + expect([peer.closed, probe.waits.length]).toEqual([false, 2]); + }); + + it("keeps no record of an origin that holds nothing: 20,000 fresh origins held and released", async () => { + const { peer } = await rawLinked((h) => void serveNotes(h), { role: "parent" }); + const churn = async (from: number, to: number) => { + for (let origin = from; origin <= to; origin++) { + peer.send({ ...hold, origin } as Message); + peer.send({ kind: "release", origin, hold: 0 }); + if (origin % 1_000 === 0) { + await transported(); + peer.received.length = 0; + } + } + await transported(); + peer.received.length = 0; + }; + // A warm-up first: what a first run allocates for good (compiled code, caches) is no record. + await churn(1, 2_000); + const before = await settledHeap(); + await churn(2_001, 22_000); + // About 1 KiB per origin before: 20 MiB. The bound leaves room for heap noise (CI measured 0.6 MiB). + expect((await settledHeap()) - before).toBeLessThan(4 * 1024 * 1024); + expect(peer.closed).toBe(false); + }, 60_000); +}); diff --git a/packages/chord/test/next/remote/path.test.ts b/packages/chord/test/next/remote/path.test.ts new file mode 100644 index 000000000..ff58b786c --- /dev/null +++ b/packages/chord/test/next/remote/path.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import { ChordError } from "../../../src/next/data/index.ts"; +import type { HostLink } from "../../../src/next/remote/host-link.ts"; +import { Path, type Upstream } from "../../../src/next/remote/path.ts"; + +/** A holder recording where it was held and what it was told, as `name@link` and `name dropped`. */ +function holder(name: string, log: string[]): Upstream { + // The path calls only `attach` and `dropped`. + return { + attach: (link: { readonly name: string }) => void log.push(`${name}@${link.name}`), + dropped: () => void log.push(`${name} dropped`), + } as unknown as Upstream; +} + +/** A link known by its name; the path only passes it on. */ +function linkOf(name: string): HostLink { + return { name } as unknown as HostLink; +} + +describe("a path", () => { + it("asks for a link when it gets a holder without one; holds every holder on each link it gets", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + path.attach(holder("a", log)); + path.attach(holder("b", log)); + expect(changes).toBe(2); + expect(log).toEqual([]); + const first = linkOf("1"); + path.connected(first); + expect(log).toEqual(["a@1", "b@1"]); + path.attach(holder("c", log)); + expect(log.at(-1)).toBe("c@1"); + expect(changes).toBe(2); + }); + + it("tells every holder when its link drops, then asks for the next; a stale link's drop is ignored", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + const [first, second] = [linkOf("1"), linkOf("2")]; + path.connected(first); + path.attach(holder("a", log)); + path.disconnected(first); + expect(log).toEqual(["a@1", "a dropped"]); + expect(changes).toBe(1); + expect(path.link).toBeUndefined(); + path.connected(second); + path.disconnected(first); + expect(log).toEqual(["a@1", "a dropped", "a@2"]); + expect(changes).toBe(1); + expect(path.link).toBe(second); + }); + + it("reports its last holder's detach: no later link holds it", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + const [a, b] = [holder("a", log), holder("b", log)]; + path.connected(linkOf("1")); + path.attach(a); + path.attach(b); + path.detach(a); + expect(changes).toBe(0); + path.detach(b); + expect(changes).toBe(1); + expect(path.held).toBe(false); + path.connected(linkOf("2")); + expect(log).toEqual(["a@1", "b@1"]); + }); + + it("says what changed: a holder attached without a link, a drop, the last release; fails its holders waiting", () => { + const events: string[] = []; + const failed: string[] = []; + const path = new Path((event) => events.push(event.kind)); + const a = { + attach: () => {}, + dropped: () => {}, + unreachable: (error: { readonly message: string }) => void failed.push(error.message), + } as unknown as Upstream; + path.attach(a); + path.unreachable(new ChordError({ code: "unavailable", message: "no worker" })); + const link = linkOf("1"); + path.connected(link); + path.disconnected(link); + path.detach(a); + expect(events).toEqual(["attached", "dropped", "released"]); + expect(failed).toEqual(["no worker"]); + }); +}); diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts new file mode 100644 index 000000000..5abb539e3 --- /dev/null +++ b/packages/chord/test/next/remote/provider.test.ts @@ -0,0 +1,1023 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { PendingOps } from "../../../src/delta/index.ts"; +import { type Contract, int, local, map, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type FacetHost } from "../../../src/next/host/index.ts"; +import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { DeflatedPart, jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + EDITOR, + expectCollected, + forEachCodec, + metered, + Notes, + type NotesProbe, + partValue, + RAW_PEER, + RawPeer, + rawLinked, + SERVER_ADDRESS, + Server, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Sends a hold of `Type` (its own version and fingerprint unless overridden), of `origin` when given. */ +function hold( + peer: RawPeer, + id: number, + Type: Contract, + params: unknown, + override: Partial = {}, + origin?: number, +) { + const { name: type, version, fingerprint } = { ...Type, ...override }; + const from = origin === undefined ? {} : { origin }; + peer.send({ + kind: "hold", + ...from, + hold: id, + type, + version, + fingerprint, + params: jsonPart(params), + address: SERVER_ADDRESS, + }); +} + +/** Sends a call of the method at index `method` of the bound variant, with `args` (none when left out). */ +function call(peer: RawPeer, id: number, holdId: number, method: number, args?: unknown, origin?: number) { + const from = origin === undefined ? {} : { origin }; + const payload = args === undefined ? {} : { args: jsonPart(args) }; + peer.send({ kind: "call", ...from, call: id, hold: holdId, method, ...payload }); +} + +/** A remote type kept per id. */ +const Kept = remote("app.kept", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); + +/** The wire's indexes of `Notes`'s reader variant. */ +const READER = wireNames(Notes, "reader"); + +const { add, fail, wait } = EDITOR.methods as { add: number; fail: number; wait: number }; + +/** A server serving `Notes` and a raw consumer peer stamped `child`. */ +async function notesServer(child: TestPrincipal = { kind: "owner" }, role: "child" | "parent" = "child") { + let probe: NotesProbe | undefined; + const linked = await rawLinked( + (h) => { + probe = serveNotes(h); + }, + { child, role }, + ); + return { ...linked, probe: probe! }; +} + +forEachCodec(() => { + describe("holds arriving on a link", () => { + it("bind as the link's principal: bound with the variant and the snapshot", async () => { + const { peer, server } = await notesServer({ kind: "guest", session: "S1" }); + await addNote(server, "S1", "a"); + hold(peer, 0, Notes, { session: "S1" }); + const [bound] = (await peer.next("bound")).holds; + expect(bound).toMatchObject({ hold: 0, variant: READER.variant, failed: [] }); + expect(READER.named(bound!.members)).toEqual({ notes: [{ text: "a" }], count: 1 }); + }); + + it.each([ + [ + "a type not served here", + { type: "app.nope" }, + { code: "unavailable", message: "app.nope is not served here" }, + ], + [ + "a local type", + { type: "app.note-store" }, + { code: "unavailable", message: "app.note-store is not served here" }, + ], + [ + "another version", + { version: 2 }, + { code: "contract_mismatch", message: "app.notes v2 does not match the served v1" }, + ], + [ + "another fingerprint", + { fingerprint: "0000000000000000" }, + { code: "contract_mismatch", message: "app.notes v1 does not match the served v1" }, + ], + ])("keep %s refused, failed { waiting }", async (_, override, error) => { + const { peer } = await notesServer(); + const { type, ...rest } = override as { type?: string }; + hold(peer, 0, { ...Notes, name: type ?? Notes.name } as Contract, { session: "S1" }, rest); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting", error: { kind: "chord", error } } }); + // No backoff window: an event (a publication) wakes it. + expect(failed.retryIn).toBeUndefined(); + }); + + it("refuse invalid params with a broken failed", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: 1 }); + expect(await peer.next("failed")).toMatchObject({ + cause: { kind: "broken", error: { kind: "chord", error: { code: "invalid", reason: "params" } } }, + }); + }); + + it("arriving before the publication wait for it, and bind before the start barrier is over", async () => { + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + hold(peer, 0, Notes, { session: "S1" }); + await transported(); + expect(peer.kinds()).toEqual(["hello"]); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], + context, + ); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 0, variant: EDITOR.variant }); + }); + }); + + describe("calls arriving on a link", () => { + it("run as the link's principal with the child's incarnation; results and errors go back", async () => { + const { peer, probe } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x" }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + call(peer, 1, 0, fail, { how: "plugin" }); + expect((await peer.next("error")).error).toEqual({ + kind: "plugin", + code: "quota", + message: "too many", + details: { limit: 1 }, + }); + call(peer, 2, 0, wait); + await transported(); + expect(probe.waits[0]).toMatchObject({ principal: { kind: "owner" }, incarnation: RAW_PEER }); + }); + + it("outside the binding's variant cannot be named: an index past its methods, forged at the wire, closes the link", async () => { + const { peer, errors } = await notesServer({ kind: "guest", session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + // The reader variant has one method, `wait`; the editor's `add` has no index in it. + call(peer, 0, 0, Object.keys(READER.methods).length, { text: "x" }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([{ kind: "protocol_error", error: { message: "hold 0: no method 1" } }]); + }); + + it("check their arguments", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x", extra: 1 }); + expect((await peer.next("error")).error).toMatchObject({ + kind: "chord", + error: { code: "invalid", reason: "args" }, + }); + }); + + it("before bound are refused not_ready (race 3)", async () => { + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const { peer } = await rawLinked((h) => + h.serve(Slow, (_, _principal, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + hold(peer, 0, Slow, {}); + call(peer, 0, 0, 0); + expect((await peer.next("error")).error).toMatchObject({ kind: "chord", error: { code: "not_ready" } }); + }); + + it("cancelled are aborted and get no reply; a result crossing the cancel is never sent (race 1)", async () => { + const { peer, probe } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, wait); + await transported(); + peer.send({ kind: "abort", call: 0 }); + await transported(); + expect(probe.waits[0]!.abortSignal?.aborted).toBe(true); + expect(peer.kinds()).toEqual(["hello"]); + }); + }); + + describe("ids", () => { + it("a link ignores ids it does not know and messages for the other direction", async () => { + const { peer, errors } = await notesServer(); + peer.send({ kind: "release", hold: 9 }); + peer.send({ kind: "abort", call: 9 }); + call(peer, 0, 9, add, { text: "x" }); + peer.send({ kind: "gone", hold: 9, error: { kind: "chord", error: { code: "released", message: "m" } } }); + await transported(); + expect(peer.closed).toBe(false); + expect(peer.kinds()).toEqual(["hello"]); + expect(errors).toEqual([]); + }); + + it("a hold reusing a live id closes the link", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([{ kind: "protocol_error", error: { message: "hold 0: the id is in use" } }]); + }); + + it("a call reusing a live id closes the link", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, wait); + call(peer, 0, 0, wait); + await transported(); + expect(peer.closed).toBe(true); + }); + + it("a message the codec cannot decode closes the link", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + peer.sendBytes(new Uint8Array([0xff, 0xff])); + await transported(); + expect(peer.closed).toBe(true); + }); + }); + + describe("one message per link per commit (W1)", () => { + it("three holds on one link: one state message, one part per hold, one encode each", async () => { + const { peer, server } = await notesServer(); + for (const id of [0, 1, 2]) hold(peer, id, Notes, { session: "S1" }); + for (const _ of [0, 1, 2]) await peer.next("bound"); + const { context: counting, counts } = await metered(); + const store = await server.get(Notes, { session: "S1" }, context); + if (store.variant === "editor") await store.add({ text: "a" }, counting); + await transported(); + const states = peer.received.filter((message) => message.kind === "state"); + expect(states).toHaveLength(1); + expect(states[0]).toMatchObject({ holds: [{ hold: 0 }, { hold: 1 }, { hold: 2 }] }); + expect(counts()["chord.link.encodes"]).toBe(3); + // No validation or walk of state on the provider: only the argument checks of `add` and the store's `add`. + expect(counts()["chord.data.validations"]).toBe(2); + expect(counts()["chord.data.walks"]).toBeUndefined(); + }); + + it("a derive travels with its source in the same part", async () => { + const { peer, server } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await addNote(server, "S1", "a"); + const state = await peer.next("state"); + expect(Object.keys(EDITOR.named(state.holds[0]!.members)).sort()).toEqual(["count", "notes"]); + }); + + it("two links: one message each; when one closes, the other keeps updating (race 8)", async () => { + const { peer, server } = await notesServer(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const other = new RawPeer(b); + hold(peer, 0, Notes, { session: "S1" }); + hold(other, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await other.next("bound"); + await addNote(server, "S1", "a"); + await peer.next("state"); + await other.next("state"); + peer.close(); + await transported(); + await addNote(server, "S1", "b"); + const next = await other.next("state"); + expect(Object.keys(EDITOR.named(next.holds[0]!.members)).sort()).toEqual(["count", "notes"]); + }); + }); + + describe("races at the provider", () => { + /** Serves `Kept`, recording each factory run and each native close. */ + function serveKept(h: Parameters[0]>[0], events: string[]) { + h.serve(Kept, ({ id }, _principal, self) => { + events.push(`factory ${id}`); + self.open( + async () => null, + (_value, reason) => void events.push(`close ${reason.kind}`), + ); + return { variant: "v", members: { n: constant(id) } }; + }); + } + + it("a new hold and the old one's release in one turn hand the object over without ending it (R18)", async () => { + const events: string[] = []; + const { peer } = await rawLinked((h) => serveKept(h, events)); + hold(peer, 0, Kept, { id: 1 }); + await peer.next("bound"); + hold(peer, 1, Kept, { id: 1 }); + peer.send({ kind: "release", hold: 0 }); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 1 }); + expect(events).toEqual(["factory 1"]); + }); + + it("a hold released right after it was sent keeps no object", async () => { + const events: string[] = []; + const { peer } = await rawLinked((h) => serveKept(h, events)); + hold(peer, 0, Kept, { id: 1 }); + await Promise.resolve(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + // The object ended: a new hold makes a new one. + hold(peer, 1, Kept, { id: 1 }); + await peer.next("bound"); + expect(events.filter((event) => event.startsWith("factory"))).toEqual(["factory 1", "factory 1"]); + }); + + it("a hold released while it waits for load takes nothing once load resolves", async () => { + const events: string[] = []; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await transported(); + hold(peer, 0, Kept, { id: 1 }); + await Promise.resolve(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveKept(h, events)) }], + context, + ); + await transported(); + expect(events).toEqual([]); + }); + + it("a release while activating unholds; bound is never sent (race 6)", async () => { + const Slow = remote("app.slow", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const aborted: string[] = []; + const { peer } = await rawLinked((h) => + h.serve(Slow, ({ id }, _principal, self) => { + self.open( + (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => resolve(aborted.push(`${id}`))), + ), + () => {}, + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + hold(peer, 0, Slow, { id: 1 }); + await transported(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + expect(aborted).toEqual(["1"]); + expect(peer.kinds()).toEqual(["hello"]); + }); + + it("a hold while the previous instance of its identity tears down waits for it (race 7)", async () => { + const Closing = remote("app.closing", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const events: string[] = []; + let finish: () => void = () => {}; + const { peer } = await rawLinked((h) => + h.serve(Closing, ({ id }, _principal, self) => { + self.open( + async () => void events.push("open"), + () => + new Promise((resolve) => { + events.push("closing"); + finish = () => { + events.push("closed"); + resolve(); + }; + }), + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + hold(peer, 0, Closing, { id: 1 }); + await peer.next("bound"); + peer.send({ kind: "release", hold: 0 }); + hold(peer, 1, Closing, { id: 1 }); + await transported(); + expect(events).toEqual(["open", "closing"]); + finish(); + await peer.next("bound"); + expect(events).toEqual(["open", "closing", "closed", "open"]); + }); + + it("a commit right after the binding continues exactly from the snapshot (race 2)", async () => { + const { peer, server } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + const adding = addNote(server, "S1", "a"); + const [bound] = (await peer.next("bound")).holds; + await adding; + await transported(); + const snapshot = EDITOR.named(bound!.members) as { notes: unknown[] }; + const states = peer.received.filter((message) => message.kind === "state"); + const added = states.flatMap((message) => (EDITOR.named(message.holds[0]!.members).notes ?? []) as unknown[]); + expect(snapshot.notes.length + added.length).toBe(1); + }); + + it("closing a link during a batch drops its state message and releases its holds (race 9)", async () => { + const Counter = remote("app.counter", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int(), seen: int() }, methods: { bump: method() } } }, + }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + // Called inside the batch, from a derive the hold follows. + let during: () => void = () => {}; + const ends: string[] = []; + const { server, errors } = await rawLinked((h) => + h.serve(Counter, ({ id }, _principal, self) => { + const n = self.state(N); + self.open( + async () => null, + () => void ends.push(`counter ${id} ended`), + ); + return { + variant: "v", + members: { + n: self.derive([n], ([{ n }]) => n), + seen: self.derive([n], ([{ n }]) => { + if (n > 0) during(); + return n; + }), + bump: async (ctx) => { + await n.change((draft) => void draft.n++, ctx); + return null; + }, + }, + }; + }), + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + hold(peer, 0, Counter, { id: 1 }); + await peer.next("bound"); + during = () => a.close(); + call(peer, 0, 0, 0); + await transported(); + expect(peer.kinds()).toEqual(["hello"]); + expect(ends).toEqual(["counter 1 ended"]); + expect(errors).toEqual([]); + }); + }); + + describe("local types over a link", () => { + it("never cross one: a hold naming one is not served (failed { waiting })", async () => { + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + const { peer } = await rawLinked((h) => h.serve(Store, () => ({ n: constant(1) }))); + hold(peer, 0, Store, {}); + expect(await peer.next("failed")).toMatchObject({ + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable" } } }, + }); + }); + }); +}); + +/** A store whose `fill` sets `cells[0..n)` to a value in one commit: `n` leaf ops on distinct paths. */ +const Grid = remote("app.grid", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + v: { + state: { grid: obj({ cells: map(int()) }), one: int() }, + methods: { fill: method(obj({ n: int(), value: int() })) }, + }, + }, +}); + +const GridDoc = defineState("grid", { schema: obj({ cells: map(int()) }), version: 1, initial: () => ({ cells: {} }) }); + +function serveGrid(h: FacetHost): void { + h.serve(Grid, (_, _principal, self) => { + const grid = self.state(GridDoc); + return { + variant: "v", + members: { + grid, + one: constant(1), + fill: async ({ n, value }, ctx) => { + await grid.change((draft) => { + for (let cell = 0; cell < n; cell++) draft.cells[`c${cell}`] = value; + }, ctx); + return null; + }, + }, + }; + }); +} + +/** A server serving `Notes` and `Grid` to a raw peer that acks only when told, the link's context counting. */ +async function slowConsumer(role: "child" | "parent" = "child") { + const counting = await metered(); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((h) => { + serveNotes(h); + serveGrid(h); + }), + }, + ], + context, + ); + const [a, b] = memoryLinkPair(); + const options = { principal: { kind: "owner" }, codec: testCodec() } as const; + if (role === "parent") server.connect(a, options, counting.context); + else server.accept(a, options, counting.context); + const peer = new RawPeer(b, { ack: false }); + await transported(); + return { server, peer, counts: counting.counts }; +} + +/** The `state` messages received and not taken. */ +function statesOf(peer: RawPeer): Extract[] { + return peer.received.filter((message) => message.kind === "state"); +} + +/** The `bound` and `state` messages received and not taken, by kind. */ +function windowed(peer: RawPeer): string[] { + return peer.kinds().filter((kind) => kind === "bound" || kind === "state"); +} + +const GRID = wireNames(Grid, "v"); + +forEachCodec(() => { + describe("the window at the provider", () => { + it("holds at most two unacknowledged bound and state messages; what waits goes out folded, encoded once", async () => { + const { server, peer, counts } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (const text of ["a", "b", "c", "d"]) await addNote(server, "S1", text); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + const before = counts()["chord.link.encodes"] ?? 0; + peer.ack(); + await transported(); + const [, pending] = peer.received.filter(({ kind }) => kind === "state") as Extract< + Message, + { kind: "state" } + >[]; + // Three commits waited: one message, one entry, `count`'s three replacements folded into the last. + expect(EDITOR.named(pending!.holds[0]!.members).count).toEqual([["r", 4]]); + expect(counts()["chord.link.encodes"]! - before).toBe(1); + expect(windowed(peer)).toEqual(["state", "state"]); + await addNote(server, "S1", "e"); + await transported(); + expect(windowed(peer)).toEqual(["state", "state"]); + }); + + it("ignores acks that cross nothing: they never open the window past two (race 18)", async () => { + const { server, peer } = await slowConsumer(); + for (const _ of [0, 1, 2]) peer.ack(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + }); + + it("lets a hold that binds while the window is full wait: the waiting state first, its bound at the next slot", async () => { + const { server, peer } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await addNote(server, "S1", "a"); + await addNote(server, "S1", "b"); + hold(peer, 1, Notes, { session: "S1" }); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + peer.ack(); + await transported(); + expect(windowed(peer)).toEqual(["state", "state"]); + await addNote(server, "S1", "c"); + peer.ack(); + await transported(); + // Hold 1's snapshot is taken now, with every commit so far. + const bound = peer.received.findLast(({ kind }) => kind === "bound") as Extract; + expect(bound.holds.map(({ hold }) => hold)).toEqual([1]); + expect(EDITOR.named(bound.holds[0]!.members).count).toBe(3); + }); + + it("overflows past 10,000 pending ops to a replacement with the current value (race 23)", async () => { + const { peer } = await slowConsumer(); + hold(peer, 0, Grid, {}); + await peer.next("bound"); + const fill = (id: number, value: number) => + peer.send({ + kind: "call", + call: id, + hold: 0, + method: GRID.methods.fill!, + args: jsonPart({ n: 6_000, value }), + }); + fill(0, 1); + fill(1, 2); + fill(2, 3); + for (const _ of [0, 1, 2]) await peer.next("result"); + // The first fill took the second slot; the next two overflowed the queue. + peer.ack(); + await transported(); + const states = peer.received.filter(({ kind }) => kind === "state") as Extract[]; + const last = GRID.named(states.at(-1)!.holds[0]!.members).grid as [string, { cells: object }][]; + expect(last[0]![0]).toBe("r"); + expect(Object.keys(last[0]![1].cells)).toHaveLength(6_000); + }); + }); + + describe("origins at the provider (a relay's: on its parent link)", () => { + it("key holds and calls by (origin, id): the same ids on two origins are two holds; answers carry the origin", async () => { + const { peer } = await slowConsumer("parent"); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S2" }, {}, 3); + const [own, relayed] = [await peer.next("bound"), await peer.next("bound")]; + expect(own.origin).toBeUndefined(); + expect(relayed.origin).toBe(3); + call(peer, 0, 0, add, { text: "a" }); + call(peer, 0, 0, add, { text: "b" }, 3); + const results = [await peer.next("result"), await peer.next("result")]; + expect(results.map(({ origin, call }) => ({ origin, call }))).toEqual([ + { origin: undefined, call: 0 }, + { origin: 3, call: 0 }, + ]); + }); + + it("keep a window per origin: a stalled origin folds its commits, another keeps receiving each", async () => { + const { server, peer } = await slowConsumer("parent"); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + await peer.next("bound"); + for (const text of ["a", "b", "c", "d"]) { + await addNote(server, "S1", text); + // Only origin 0 acknowledges. + peer.ack(); + } + await transported(); + const states = statesOf(peer); + expect(states.filter(({ origin }) => origin === undefined)).toHaveLength(4); + // Origin 1: its bound and one state fill its window; three commits wait, folded into its next state. + expect(states.filter(({ origin }) => origin === 1)).toHaveLength(1); + peer.ack(1); + peer.ack(1); + await transported(); + const folded = statesOf(peer) + .filter(({ origin }) => origin === 1) + .at(-1)!; + expect(EDITOR.named(folded.holds[0]!.members).count).toEqual([["r", 4]]); + }); + + it("count calls per origin against N: 256 on each of two origins run, one more on one closes the link", async () => { + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + await peer.next("bound"); + for (let id = 0; id < 256; id++) { + call(peer, id, 0, wait); + call(peer, id, 0, wait, undefined, 1); + } + await transported(); + expect(peer.closed).toBe(false); + call(peer, 256, 0, wait, undefined, 1); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: "call 256: more than 256 calls" } }, + ]); + }); + + it("ignore a call on a hold they do not know before counting it (Q1)", async () => { + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id < 256; id++) call(peer, id, 0, wait); + call(peer, 256, 9, wait); + call(peer, 257, 0, wait, undefined, 4); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + + it("closed releases the origin's holds and aborts its calls; another origin keeps its own", async () => { + const { peer, probe, server } = await notesServer({ kind: "owner" }, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S2" }, {}, 2); + await peer.next("bound"); + await peer.next("bound"); + call(peer, 0, 0, wait, undefined, 2); + await transported(); + const [waiting] = probe.waits; + peer.send({ kind: "closed", origin: 2 }); + await transported(); + expect(waiting!.abortSignal?.aborted).toBe(true); + // Its ids are free: nothing answers them, a new hold of origin 2 starts over. + call(peer, 1, 0, add, { text: "x" }, 2); + await addNote(server, "S1", "a"); + await transported(); + expect(peer.kinds().filter((kind) => kind === "result" || kind === "error")).toEqual([]); + expect(statesOf(peer).map(({ origin }) => origin)).toEqual([undefined]); + }); + + it("carry constants and failed members, and an empty variant's binding, in a first bound (race 24)", async () => { + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: {} }, + }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const { peer } = await rawLinked((h) => { + h.serve(Empty, () => ({ variant: "v", members: {} })); + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + one: constant(1), + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + }, + }; + }); + }); + hold(peer, 0, Halves, {}); + await peer.next("bound"); + call(peer, 0, 0, HALVES.methods.set!, 2); + await peer.next("result"); + hold(peer, 1, Empty, {}); + hold(peer, 2, Halves, {}); + const bounds = [await peer.next("bound"), await peer.next("bound")]; + const [empty, halves] = [1, 2].map((id) => bounds.find(({ holds }) => holds[0]!.hold === id)!); + expect(empty.holds).toEqual([{ hold: 1, variant: 0, members: [], failed: [] }]); + expect(halves.holds).toEqual([ + { + hold: 2, + variant: 0, + members: [{ member: HALVES.state.one!, value: expect.anything() }], + failed: [ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ], + }, + ]); + expect(HALVES.named(halves.holds[0]!.members)).toEqual({ one: 1 }); + }); + }); + + describe("GC of the window", () => { + it("a closed origin drops its pending queue's ops: nothing keeps them", async () => { + const push = PendingOps.prototype.push; + const pushed: WeakRef[] = []; + // Patched by hand: a spy would keep every argument in its call log. + PendingOps.prototype.push = function (this: PendingOps, ops) { + pushed.push(new WeakRef(ops)); + push.call(this, ops); + }; + try { + const { server, peer } = await slowConsumer("parent"); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); + expect(pushed.length).toBeGreaterThan(0); + peer.send({ kind: "closed", origin: 1 }); + await transported(); + await expectCollected(...pushed); + expect(peer.closed).toBe(false); + } finally { + PendingOps.prototype.push = push; + } + }); + }); + + describe("the call limit", () => { + it("closes a link whose peer sends more calls than this side announced", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id <= 256; id++) call(peer, id, 0, wait); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: "call 256: more than 256 calls" } }, + ]); + }); + + it("frees a slot per reply, cancel and release", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id < 256; id++) call(peer, id, 0, wait); + peer.send({ kind: "abort", call: 0 }); + call(peer, 256, 0, wait); + peer.send({ kind: "release", hold: 0 }); + hold(peer, 1, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 300; id < 556; id++) call(peer, id, 1, wait); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + }); +}); + +/** `Halves`: a constant, and a member that fails once `set(2)`. */ +const Halves = remote("app.halves", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { one: int(), half: int() }, methods: { set: method(int()) } } }, +}); + +const HALVES = wireNames(Halves, "v"); + +forEachCodec(() => { + describe("limits per part at the provider", () => { + /** + * A server serving `Notes` behind a raw consumer whose link has `limits`, compressing with `compression`; the raw + * peer is its child, or its parent (only a parent may send compressed arguments). + */ + async function limited(limits: PartLimits, compression?: Compression, role?: "child" | "parent") { + return rawLinked((h) => void serveNotes(h), { limits, compression, role }); + } + + /** Deflated JSON of `{ text }` with `length` characters. */ + const deflatedText = (length: number) => + new DeflatedPart(nodeCompression.deflate(UTF8.encode(JSON.stringify({ text: "x".repeat(length) })))); + + it("fails a call whose argument part is past the wire limit with too_large/part; the link stays", async () => { + const { peer } = await limited({ part: 100 }); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x".repeat(200) }); + expect(await peer.next("error")).toMatchObject({ + call: 0, + error: { kind: "chord", error: { code: "too_large", reason: "part" } }, + }); + call(peer, 1, 0, add, { text: "fits" }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + expect(peer.closed).toBe(false); + }); + + it("fails a hold whose params are past the limit broken too_large/part", async () => { + const { peer } = await limited({ part: 30 }); + hold(peer, 0, Notes, { session: "s".repeat(40) }); + expect(await peer.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: { code: "too_large", reason: "part" } } }, + }); + }); + + it("fails a call whose deflated argument inflates past the decoded limit, on both inflate paths", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer } = await limited({ decoded: 10_000 }, compression, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: deflatedText(50_000) }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + call(peer, 1, 0, add, { text: "fits" }); + expect((await peer.next("result")).call).toBe(1); + } + }); + + it("inflates a deflated argument within the hold's stamped decoded limit, a bomb included (both paths)", async () => { + const output = countInflated(); + try { + for (const compression of [nodeCompression, undefined]) { + // The stamped limit is smaller than the link's own: it bounds the inflate. + const { peer } = await limited({}, compression, "parent"); + peer.send({ ...notesHold(0), limit: 1000 }); + await peer.next("bound"); + const bomb = new DeflatedPart( + nodeCompression.deflate(UTF8.encode(JSON.stringify("x".repeat(64 << 20)))), + ); + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: bomb }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + peer.send({ kind: "call", call: 1, hold: 0, method: add, args: deflatedText(500) }); + expect((await peer.next("result")).call).toBe(1); + } + // The asynchronous inflate stopped within a chunk of the stamped limit, never near the bomb's 64 MiB. + expect(output.bytes).toBeLessThan(1024 * 1024); + } finally { + output.restore(); + } + }); + + it("checks a deflated argument's wire size before inflating it (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer, errors } = await limited({ part: 18 }, compression, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + // Corrupt data past the wire limit: refused by its size, never inflated (no protocol error). + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: new DeflatedPart(new Uint8Array(20)) }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("inflates nothing for a call it ignores: an unknown hold (Q1), a refused one (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer, errors } = await limited({}, compression, "parent"); + const corrupt = () => new DeflatedPart(new Uint8Array([255, 255, 255])); + peer.send({ kind: "call", call: 0, hold: 9, method: add, args: corrupt() }); + peer.send({ ...notesHold(0), type: "app.kept" }); + await peer.next("failed"); + peer.send({ kind: "call", call: 1, hold: 0, method: 0, args: corrupt() }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("refuses compressed arguments from a child, and compressed params on any link: protocol errors", async () => { + const child = await limited({}); + hold(child.peer, 0, Notes, { session: "S1" }); + await child.peer.next("bound"); + child.peer.send({ kind: "call", call: 0, hold: 0, method: add, args: deflatedText(10) }); + await transported(); + expect(child.peer.closed).toBe(true); + expect(child.errors).toMatchObject([{ kind: "protocol_error" }]); + const parent = await limited({}, undefined, "parent"); + parent.peer.send({ ...notesHold(0), params: new DeflatedPart(nodeCompression.deflate(UTF8.encode("{}"))) }); + await transported(); + expect(parent.peer.closed).toBe(true); + expect(parent.errors).toMatchObject([{ kind: "protocol_error" }]); + }); + + it("honours a stamped decoded limit only on a parent link: a child's is ignored (race 12)", async () => { + const { peer } = await limited({}); + peer.send({ ...notesHold(0), limit: 10 }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "longer than ten bytes" }); + expect((await peer.next("result")).call).toBe(0); + }); + }); +}); + +/** A hold of `Notes` for S1 as a message. */ +function notesHold(id: number): Extract { + const { name: type, version, fingerprint } = Notes; + return { + kind: "hold", + hold: id, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; +} + +const UTF8 = new TextEncoder(); diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts new file mode 100644 index 000000000..6f08a6f14 --- /dev/null +++ b/packages/chord/test/next/remote/relay.test.ts @@ -0,0 +1,807 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, jsonPart, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import type * as Values from "../../../src/next/link/values.ts"; +import { RelayedHold } from "../../../src/next/remote/relay.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + EDITOR, + expectCollected, + fakeTimers, + forEachCodec, + instancesOf, + Notes, + PlacedNotes, + RawPeer, + rawWorkers, + rejection, + relayed, + Server, + Session, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** How often a part was parsed in this file, by anyone. */ +const parses = vi.hoisted(() => ({ count: 0 })); + +vi.mock("../../../src/next/link/values.ts", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + parsePart: (part: string | Uint8Array) => { + parses.count++; + return original.parsePart(part); + }, + parsePayload: (part: string | Uint8Array, segments: readonly Uint8Array[] | undefined) => { + parses.count++; + return original.parsePayload(part, segments); + }, + }; +}); + +/** Every relayed hold started in this file, weakly. */ +const relayedHolds = instancesOf(RelayedHold, "start"); + +const READER = wireNames(PlacedNotes, "reader"); + +const { wait, add } = EDITOR.methods as { wait: number; add: number }; + +/** A server routing to raw workers, and a raw tab on a link it accepted as `principal`. */ +async function rawRelay(options: { readonly principal?: TestPrincipal; readonly calls?: number } = {}) { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { calls: options.calls }) }); + const tab = rawTab(relay.server, options.principal); + return { ...relay, workers, raw: tab }; +} + +/** A raw tab on a new link the server accepts as `principal` (owner by default). */ +function rawTab(server: Awaited>["server"], principal?: TestPrincipal): RawPeer { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: principal ?? { kind: "owner" }, codec: testCodec() }, context); + return new RawPeer(b); +} + +/** Sends a hold of `PlacedNotes` for `session` with id `hold`, naming its address. */ +function holdPlaced(peer: RawPeer, hold: number, session = "S1"): void { + peer.send({ + kind: "hold", + hold, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session }), + address: { kind: "session", params: { id: session } }, + }); +} + +/** The raw worker at `index` once it exists (a resolve runs a turn after the hold arrived). */ +async function workerAt(workers: RawPeer[], index = 0): Promise { + for (let tries = 0; workers[index] === undefined && tries < 10; tries++) await transported(); + return workers[index]!; +} + +/** A hold as a raw worker received it: its id and its origin, which every answer names. */ +type Held = { readonly hold: number; readonly origin?: number }; + +/** The origin field of an answer to `held`. */ +function originOf({ origin }: Held): { origin?: number } { + return origin === undefined ? {} : { origin }; +} + +/** Binds `held` on a raw worker with `PlacedNotes`'s editor (or reader) variant and `count`. */ +function bindPlaced(peer: RawPeer, held: Held, count = 0, variant = EDITOR): void { + const members = [ + { member: variant.state.notes!, value: "[]" }, + { member: variant.state.count!, value: jsonPart(count) }, + ]; + peer.send({ + kind: "bound", + ...originOf(held), + holds: [{ hold: held.hold, variant: variant.variant, members, failed: [] }], + }); +} + +/** A `state` of `count` for `holds` (of one origin) on a raw worker. */ +function countState(holds: readonly Held[], count: number): Extract { + const ops = jsonPart([["r", count]]); + return { + kind: "state", + ...originOf(holds[0]!), + holds: holds.map(({ hold }) => ({ + hold, + members: [{ member: EDITOR.state.count!, ops }], + failed: [], + retired: [], + })), + }; +} + +forEachCodec(() => { + describe("ids and origins", () => { + it("keep the downstream ids upstream, under the downstream link's origin; answers come back by it", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 7); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + expect(held).toMatchObject({ hold: 7, origin: 1 }); + bindPlaced(worker, held, 3); + const bound = await raw.next("bound"); + expect(bound.origin).toBeUndefined(); + expect(bound.holds[0]).toMatchObject({ hold: 7, variant: EDITOR.variant }); + raw.send({ kind: "call", call: 5, hold: 7, method: add, args: jsonPart({ text: "x" }) }); + // Calls are forwarded at once, with the same id. + expect(await worker.next("call")).toMatchObject({ origin: 1, call: 5, hold: 7, method: add }); + worker.send({ kind: "result", origin: 1, call: 5, value: "1" }); + const result = await raw.next("result"); + expect(result).toMatchObject({ kind: "result", call: 5 }); + expect(result.origin).toBeUndefined(); + raw.send({ kind: "retry", hold: 7, hops: 5 }); + // Forwarded with its budget as it is: a relay visits no object. + expect(await worker.next("retry")).toEqual({ kind: "retry", origin: 1, hold: 7, hops: 5 }); + }); + + it("give each downstream link its own origin, never reused", async () => { + const { raw, server, workers } = await rawRelay(); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const one = await worker.next("hold"); + const other = rawTab(server); + holdPlaced(other, 0); + const two = await worker.next("hold"); + expect([one.origin, two.origin]).toEqual([1, 2]); + other.close(); + await transported(); + holdPlaced(rawTab(server), 0); + expect((await worker.next("hold")).origin).toBe(3); + }); + + it("drop entries, notices and replies for holds of the origin relayed over another link", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 0, "S1"); + holdPlaced(raw, 1, "S2"); + const one = await workerAt(workers, 0); + const x = await one.next("hold"); + const two = await workerAt(workers, 1); + const y = await two.next("hold"); + expect([x.origin, y.origin]).toEqual([1, 1]); + bindPlaced(one, x); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await one.next("call"); + // Worker two speaks for worker one's hold of the same tab origin. + two.send(countState([x], 9)); + two.send({ kind: "loading", origin: 1, hold: 0 }); + two.send({ kind: "result", origin: 1, call: 0 }); + two.send({ + kind: "gone", + origin: 1, + hold: 0, + error: { kind: "chord", error: { code: "unavailable", message: "forged" } }, + }); + bindPlaced(two, x); + await transported(); + expect(raw.kinds()).toEqual(["hello"]); + // Its windowed messages, left empty, are acknowledged to it at once. + expect(two.kinds().filter((kind) => kind === "ack")).toEqual(["ack", "ack"]); + // A message with its own hold and the other's: only its own entry is forwarded. + bindPlaced(two, y); + await raw.next("bound"); + two.send({ kind: "state", origin: 1, holds: [...countState([x], 9).holds, ...countState([y], 3).holds] }); + const state = await raw.next("state"); + expect(state.holds.map(({ hold }) => hold)).toEqual([1]); + one.send({ kind: "result", origin: 1, call: 0 }); + expect(await raw.next("result")).toMatchObject({ call: 0 }); + }); + + it("ignore answers for an origin from a link it relayed nothing to", async () => { + const { raw, server, workers } = await rawRelay(); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + // Another tab speaks for the first tab's origin on its own link. + const other = rawTab(server); + await transported(); + bindPlaced(other, held); + await transported(); + expect(raw.kinds()).toEqual(["hello"]); + bindPlaced(worker, held); + await raw.next("bound"); + }); + }); + + describe("parts at a relay", () => { + it("are never parsed: params, snapshots, ops, arguments and results travel as they came", async () => { + const { raw, workers } = await rawRelay(); + await transported(); + const before = parses.count; + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held, 1); + await raw.next("bound"); + worker.send(countState([held], 2)); + await raw.next("state"); + raw.send({ kind: "call", call: 0, hold: 0, method: add, args: jsonPart({ text: "x" }) }); + await worker.next("call"); + worker.send({ kind: "result", origin: held.origin, call: 0, value: "2" }); + await raw.next("result"); + expect(parses.count).toBe(before); + }); + }); + + describe("the window through a relay", () => { + it("forwards every bound and state at once and relays each ack to its worker", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { ack: false }) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b, { ack: false }); + holdPlaced(tab, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + for (const count of [1, 2, 3]) worker.send(countState([held], count)); + await transported(); + // The relay holds nothing back: the worker's window for this origin is the limit. + expect(tab.kinds()).toEqual(["hello", "bound", "state", "state", "state"]); + expect(worker.kinds().filter((kind) => kind === "ack")).toEqual([]); + tab.ack(); + tab.ack(); + await transported(); + expect(worker.received.filter(({ kind }) => kind === "ack")).toEqual([ + { kind: "ack", origin: held.origin }, + { kind: "ack", origin: held.origin }, + ]); + }); + + it("acknowledges to the right worker in order: two workers sharing one tab", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { ack: false }) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b, { ack: false }); + holdPlaced(tab, 0, "S1"); + const one = await workerAt(workers, 0); + const x = await one.next("hold"); + holdPlaced(tab, 1, "S2"); + const two = await workerAt(workers, 1); + const y = await two.next("hold"); + bindPlaced(one, x); + await transported(); + bindPlaced(two, y); + await transported(); + tab.ack(); + await transported(); + expect(one.kinds().filter((kind) => kind === "ack")).toEqual(["ack"]); + expect(two.kinds().filter((kind) => kind === "ack")).toEqual([]); + tab.ack(); + await transported(); + expect(two.kinds().filter((kind) => kind === "ack")).toEqual(["ack"]); + }); + + it("never tears a commit: a slow tab's states fold at the worker, a derive over two holds sees both at one moment (race 21)", async () => { + const Pair = local("app.pair", { version: 1, params: {}, members: { pair: str() } }); + const relay = await relayed(); + const seen: string[] = []; + const tab = createHost({ onError: () => {} }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const slow = pausable(b); + tab.connect(slow.link, { principal: { kind: "server" }, codec: testCodec() }, context); + await tab.load( + [ + { + name: "@test/pair", + facet: defineFacet((h) => + h.serve(Pair, (_, self) => { + const one = self.get(PlacedNotes, { session: "S1" }); + const two = self.get(PlacedNotes, { session: "S1" }); + return { + pair: self.derive([one.count, two.count], ([x, y]) => { + seen.push(`${x}/${y}`); + return `${x}/${y}`; + }), + }; + }), + ), + }, + ], + context, + ); + const view = await tab.get(Pair, {}, context); + const watch = await view.pair.watch(context); + watch.start(async () => {}); + const worker = relay.workers.get("session|S1")!; + slow.pause(); + slow.kinds.length = 0; + for (const text of ["a", "b", "c", "d", "e"]) await addNote(worker, "S1", text); + await transported(); + // Two states in the tab origin's window at the worker, the other commits folded there. + expect(slow.kinds).toEqual(["state", "state"]); + slow.resume(); + await transported(); + await transported(); + expect(view.pair.value).toBe("5/5"); + expect(slow.kinds).toEqual(["state", "state", "state"]); + expect(seen.every((pair) => pair.split("/")[0] === pair.split("/")[1])).toBe(true); + expect(relay.errors.server).toEqual([]); + }); + }); + + describe("an upstream drop", () => { + fakeTimers(); + + it("sends loading downstream, fails relayed calls unavailable, holds again with the same id and forwards the new bound, another variant allowed (race 12)", async () => { + const { tab, workers, errors } = await rawRelay(); + const got = tab.get(PlacedNotes, { session: "S1" }, context); + const first = await workerAt(workers); + const held = await first.next("hold"); + bindPlaced(first, held, 1); + const notes = await got; + expect(notes.variant).toBe("editor"); + const call = notes.wait(context); + await first.next("call"); + first.close(); + expect(await rejection(call)).toMatchObject({ code: "unavailable" }); + await transported(); + // The link had completed its handshake: its address resolves again at once (no timer, slice 10 R2). + expect(notes.state.value).toEqual({ kind: "loading" }); + const second = await workerAt(workers, 1); + const again = await second.next("hold"); + expect(again).toMatchObject({ hold: held.hold, origin: held.origin }); + bindPlaced(second, again, 4, READER); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.variant).toBe("reader"); + expect(notes.count.value).toBe(4); + expect(errors.tab).toEqual([]); + }); + + it("answers a call that crossed the loading notice unavailable (race 13)", async () => { + // The address's next resolve never settles: the relayed hold has no upstream link when the call arrives. + const workers: RawPeer[] = []; + const resolve = rawWorkers(workers); + const relay = await relayed({ + resolve: (address) => (workers.length === 0 ? resolve(address) : new Promise(() => {})), + }); + const raw = rawTab(relay.server); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await raw.next("bound"); + worker.close(); + await transported(); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + expect(await raw.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect((await raw.next("error")).error).toMatchObject({ error: { code: "unavailable" } }); + }); + }); + + describe("a parent link that is down (audit)", () => { + it("holds a hold arriving meanwhile on the parent path: it waits loading and binds on the next parent link", async () => { + // The middle host relays its tab's holds to its parent, which reaches it over a route: origins arrive only on + // parent and route links (decision "Origins only from relays"). + const middle = createHost({ + address: Session({ id: "M" }), + principal: { kind: "worker", session: "M" }, + onError: () => {}, + }); + await middle.load([], context); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + const links: Link[] = []; + let gate = Promise.resolve(); + server.route( + Session, + { + authorize: () => true, + resolve: async () => { + await gate; + const [a, b] = memoryLinkPair(); + middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + links.push(a); + return { link: a, principal: { kind: "worker", session: "M" }, codec: testCodec() }; + }, + }, + context, + ); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], + context, + ); + // The server holds something at the middle's address: its route link exists while it does. + const anchor = server.hold(PlacedNotes, { session: "M" }, context); + await transported(); + // The next resolve waits: the middle's parent path is down meanwhile. + let admit: () => void = () => {}; + gate = new Promise((resolve) => { + admit = resolve; + }); + links[0]!.close(); + await transported(); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + middle.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + tab.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + const notes = tab.hold(Notes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + admit(); + await transported(); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + anchor.release(); + }); + }); + + describe("a downstream release", () => { + it("before the upstream hold was sent drops it; after, releases it upstream; messages crossing it are dropped (race 14)", async () => { + let finish: () => void = () => {}; + const workers: RawPeer[] = []; + const make = rawWorkers(workers); + const { server } = await relayed({ + resolve: async (address) => { + await new Promise((resolve) => { + finish = resolve; + }); + return make(address); + }, + }); + const raw = rawTab(server); + holdPlaced(raw, 0); + await transported(); + raw.send({ kind: "release", hold: 0 }); + holdPlaced(raw, 1); + await transported(); + finish(); + const worker = await workerAt(workers); + const sent = await worker.next("hold"); + expect(sent.hold).toBe(1); + await transported(); + expect(worker.kinds()).toEqual(["hello"]); + raw.send({ kind: "release", hold: 1 }); + expect(await worker.next("release")).toEqual({ kind: "release", origin: sent.origin, hold: 1 }); + bindPlaced(worker, sent); + worker.send(countState([sent], 1)); + await transported(); + // The origin held nothing more: its record went and the worker was told `closed` (decision "Origins only from + // relays"); what crossed it is dropped (Q1). + expect(raw.kinds()).toEqual(["hello"]); + expect(worker.received.filter(({ kind }) => kind === "closed")).toEqual([ + { kind: "closed", origin: sent.origin }, + ]); + }); + }); + + describe("a downstream drop", () => { + it("closes its origin upstream: one closed per worker, no release or cancel (race 15)", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 0, "S1"); + holdPlaced(raw, 1, "S2"); + const one = await workerAt(workers, 0); + const held = await one.next("hold"); + const two = await workerAt(workers, 1); + await two.next("hold"); + bindPlaced(one, held); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await one.next("call"); + raw.close(); + await transported(); + for (const worker of [one, two]) { + const ends = worker.received.filter(({ kind }) => ["release", "abort", "closed"].includes(kind)); + expect(ends).toEqual([{ kind: "closed", origin: held.origin }]); + } + }); + }); + + describe("gone from upstream", () => { + it("is forwarded and the relayed hold forgotten (race 16)", async () => { + const { tab, workers } = await rawRelay(); + const got = tab.get(PlacedNotes, { session: "S1" }, context); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + const notes = await got; + const watch = await notes.count.watch(context); + worker.send({ + kind: "gone", + ...originOf(held), + hold: held.hold, + error: { kind: "chord", error: { code: "unavailable", message: "revoked" } }, + }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => notes.count.value).toThrow("revoked"); + // A state crossing the gone reaches the tab, which ignores it; nothing closes. + worker.send(countState([held], 9)); + await transported(); + expect(worker.closed).toBe(false); + }); + }); + + describe("GC at a relay", () => { + it("a released relayed hold and a closed downstream link keep no relay state", async () => { + const { refs, server } = await (async () => { + const { raw, workers, server } = await rawRelay(); + const made = relayedHolds.length; + holdPlaced(raw, 0); + holdPlaced(raw, 1); + const worker = await workerAt(workers); + const one = await worker.next("hold"); + bindPlaced(worker, one); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await worker.next("call"); + raw.send({ kind: "release", hold: 0 }); + raw.close(); + await transported(); + return { refs: relayedHolds.slice(made), server }; + })(); + expect(refs).toHaveLength(2); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); + +describe("received buffers at a relay", () => { + it("are not kept by the parts of a relayed hold after its message was handled", async () => { + const workers: RawPeer[] = []; + const { server } = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + const buffers: WeakRef[] = []; + const listening: Link = { + send: (message) => a.send(message), + close: () => a.close(), + onClose: (listener) => a.onClose(listener), + onMessage: (listener) => + a.onMessage((message) => { + buffers.push(new WeakRef(message.buffer)); + listener(message); + }), + }; + server.accept(listening, { principal: { kind: "owner" }, codec: binaryCodec }, context); + const tab = (() => { + const peer = new RawPeer(b, { codec: binaryCodec }); + holdPlaced(peer, 0, "S1"); + return peer; + })(); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await tab.next("bound"); + expect(buffers.length).toBeGreaterThan(1); + await expectCollected(...buffers); + expect(relayedHolds.at(-1)?.deref()).toBeDefined(); + }); +}); + +describe("mixed codecs", () => { + it("relay between a binary tab link and a text route link", async () => { + const { tab, workers } = await relayed({ codecs: { tab: binaryCodec, route: textCodec } }); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + await addNote(workers.get("session|S1")!, "S1", "é"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "é" }]); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "b" }, context)).toBe(2); + const { context: cancellable, cancel } = withCancel(context); + const waiting = notes.wait(cancellable); + cancel(new Error("stop")); + await expect(waiting).rejects.toThrow("stop"); + }); +}); + +/** + * A link end whose incoming messages can be held back (a slow tab): paused, they wait; resumed, they arrive in order. + * `kinds` lists what arrived, decoded with the test codec. + */ +function pausable(link: Link): { readonly link: Link; readonly kinds: string[]; pause(): void; resume(): void } { + let paused = false; + const held: Uint8Array[] = []; + const kinds: string[] = []; + let listener: (message: Uint8Array) => void = () => {}; + link.onMessage((message) => { + kinds.push(testCodec().decode(message).kind); + if (paused) held.push(message); + else listener(message); + }); + return { + kinds, + link: { + send: (message) => link.send(message), + close: () => link.close(), + onClose: (close) => link.onClose(close), + onMessage: (next) => { + listener = next; + }, + }, + pause: () => { + paused = true; + }, + resume: () => { + paused = false; + for (const message of held.splice(0)) listener(message); + }, + }; +} + +/** A compressor that counts its inflates (`nodeCompression` underneath). */ +function countingInflates(): Compression & { inflates: number } { + const counts = { + inflates: 0, + deflate: (bytes: Uint8Array) => nodeCompression.deflate(bytes), + inflate: (bytes: Uint8Array, limit: number) => { + counts.inflates++; + return nodeCompression.inflate(bytes, limit); + }, + }; + return counts; +} + +forEachCodec(() => { + describe("limits and compression through a relay", () => { + it("check an argument's size on the admitting link at the relay: too_large/part, nothing forwarded", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec(), limits: { part: 100 } }, context); + const raw = new RawPeer(b); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: add, args: jsonPart({ text: "x".repeat(200) }) }); + expect(await raw.next("error")).toMatchObject({ + call: 0, + error: { error: { code: "too_large", reason: "part" } }, + }); + await transported(); + expect(worker.kinds()).not.toContain("call"); + }); + + it("stamp the admitting link's decoded limit on the upstream hold; a tab's own is ignored (race 12)", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept( + a, + { principal: { kind: "owner" }, codec: testCodec(), limits: { decoded: 4096 } }, + context, + ); + const raw = new RawPeer(b); + raw.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + limit: 1_000_000_000, + }); + const worker = await workerAt(workers); + expect((await worker.next("hold")).limit).toBe(4096); + }); + + it("enforce the stamped limit where the part is decoded: the worker refuses the tab's large argument", async () => { + const relay = await relayed({ trusted: true, limits: { decoded: 1000 } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + // Within the tab link's wire limit, past its decoded limit, which the trusted worker link does not have. + expect(await rejection(notes.add({ text: "x".repeat(2000) }, context))).toMatchObject({ + code: "too_large", + reason: "part", + }); + expect(await notes.add({ text: "small" }, context)).toBe(1); + }); + + it("forward deflated parts as they came: the relay inflates nothing (a counter)", async () => { + const [server, tab] = [countingInflates(), countingInflates()]; + const relay = await relayed({ compression: { server, worker: nodeCompression, tab } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const text = "lorem ipsum ".repeat(3000); + await notes.add({ text }, context); + await transported(); + expect(notes.notes.value).toEqual([{ text }]); + // The worker's state deflated once at its origin, inflated by its owner, the tab. + expect([server.inflates, tab.inflates]).toEqual([0, 1]); + }); + + it("forward deflated parts as they came at a relay without a compressor: nothing inflated there", async () => { + const inflated = countInflated(); + try { + const tab = countingInflates(); + const relay = await relayed({ compression: { worker: nodeCompression, tab } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + await notes.add({ text: "lorem ipsum ".repeat(3000) }, context); + await transported(); + expect(tab.inflates).toBe(1); + expect(inflated.bytes).toBe(0); + } finally { + inflated.restore(); + } + }); + + it("forward answers from upstream unchecked: the end that parses a part checks it against its own link", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { limits: { part: 50 } }) }); + const getting = relay.tab.get(PlacedNotes, { session: "S1" }, context); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + const notes = await getting; + if (notes.variant !== "editor") throw new Error("bound as the editor"); + // A state part and a result past the route link's wire limit reach the tab, whose link has none that small. + const ops = jsonPart([["r", 1234567890], ...Array(10).fill(["r", 7])]); + worker.send({ + kind: "state", + ...originOf(held), + holds: [{ hold: held.hold, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }], + }); + const added = notes.add({ text: "x" }, context); + const call = await worker.next("call"); + worker.send({ kind: "result", ...originOf(held), call: call.call, value: jsonPart("y".repeat(60)) }); + expect(await added).toBe("y".repeat(60)); + await transported(); + expect(notes.count.value).toBe(7); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it.each([0, 1200])( + "detach a reference failed too_large from its route; retry %i ms after the drop places it anew, a new hold", + async (wait) => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { limits: { part: 50 } }) }); + const notes = relay.server.hold(PlacedNotes, { session: "S1" }, context); + const first = await workerAt(workers); + const held = await first.next("hold"); + const members = [ + { member: EDITOR.state.notes!, value: jsonPart([{ text: "z".repeat(60) }]) }, + { member: EDITOR.state.count!, value: jsonPart(1) }, + ]; + first.send({ kind: "bound", holds: [{ hold: held.hold, variant: EDITOR.variant, members, failed: [] }] }); + expect(await first.next("release")).toMatchObject({ hold: held.hold }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + // The worker's link drops: nothing holds the address, so nothing resolves it again for this reference. + first.close(); + await new Promise((resolve) => setTimeout(resolve, wait)); + await transported(); + expect(workers).toHaveLength(1); + // The route forgot the address: the retry places the reference anew, resolves it and holds it. + notes.retry(); + expect(notes.state.value.kind).toBe("loading"); + for (let tries = 0; workers.length < 2 && tries < 30; tries++) await new Promise((r) => setTimeout(r, 100)); + const second = await workerAt(workers, 1); + bindPlaced(second, await second.next("hold"), 2); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.count.value).toBe(2); + }, + ); + + it("let a tab without a compressor inflate the worker's deflated parts asynchronously, in order", async () => { + const relay = await relayed({ compression: { worker: nodeCompression, server: nodeCompression } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const text = "lorem ipsum ".repeat(3000); + for (let i = 0; i < 3; i++) await notes.add({ text }, context); + for (let i = 0; i < 100 && notes.count.value < 3; i++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(notes.notes.value).toEqual([{ text }, { text }, { text }]); + }); + }); +}); diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts new file mode 100644 index 000000000..3141dbc84 --- /dev/null +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -0,0 +1,850 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { PluginError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type Host, + type HostError, + type Reference, + type RefIn, +} from "../../../src/next/host/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart, wireError } from "../../../src/next/link/internal.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + bindHold, + bindNext, + changesOf, + delivered, + EDITOR, + expectCollected, + failures, + fakeTimers, + forEachCodec, + framesOf, + instancesOf, + linked, + Notes, + partValue, + RawPeer, + rawParent, + rawProvider, + rejection, + Server, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** A counter document of the remote types defined in tests. */ +const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +forEachCodec(() => { + describe("a hold over the parent link", () => { + it("reject a broken failure at once", async () => { + const Broken = remote("app.broken", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const { client } = await linked((h) => + h.serve(Broken, () => { + throw new Error("factory threw"); + }), + ); + expect(await rejection(client.get(Broken, {}, context, { readyTimeout: 5_000 }))).toEqual({ + code: "unavailable", + message: "app.broken: factory threw", + }); + }); + + it("gone: the reference ends, its pending calls reject with the error", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + const watch = await notes.notes.watch(context); + const call = notes.wait(context); + await peer.next("call"); + peer.send({ + kind: "gone", + hold, + error: { kind: "chord", error: { code: "released", reason: "ended", message: "app.notes(S1): ended" } }, + }); + expect(await rejection(call)).toEqual({ code: "released", reason: "ended", message: "app.notes(S1): ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => notes.count.value).toThrow("app.notes(S1): ended"); + }); + + it("two references from one host are two holds", async () => { + const { client, peer } = await rawProvider(); + const first = client.get(Notes, { session: "S1" }, context); + const second = client.get(Notes, { session: "S1" }, context); + expect(await bindNext(peer)).toBe(0); + expect(await bindNext(peer)).toBe(1); + await Promise.all([first, second]); + }); + + it("a member failure fails only that replica: reads throw it, its watch gets a failure call and stays open; the others update", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + const count = changesOf(await notes.count.watch(context)); + const list = framesOf(await notes.notes.watch(context)); + const failed = [{ member: EDITOR.state.count!, error: wireError(new PluginError("derive", "count threw")) }]; + const members = [{ member: EDITOR.state.notes!, ops: jsonPart([["p", [], 0, 0, [{ text: "a" }]]]) }]; + peer.send({ kind: "state", holds: [{ hold, members, failed, retired: [] }] }); + await delivered(); + expect(count).toEqual([{ kind: "failed", error: expect.objectContaining({ code: "derive" }) }]); + expect(list).toEqual([[[{ text: "a" }], [["p", [], 0, 0, [{ text: "a" }]]]]]); + expect(() => notes.count.value).toThrow("count threw"); + // The provider's derive healed: its next value is a root replacement, in the same hold. + const healed = [{ member: EDITOR.state.count!, ops: jsonPart([["r", 1]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: healed, failed: [], retired: [] }] }); + await delivered(); + expect(notes.count.value).toBe(1); + expect(count.at(-1)).toEqual({ kind: "value", value: 1, ops: [["r", 1]], retired: false }); + }); + }); + + describe("retired members", () => { + it("reach the replicas' watches with the flag: with the member's ops, or alone", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer, [{ text: "a" }]); + const notes = await got; + const list = changesOf(await notes.notes.watch(context)); + const count = changesOf(await notes.count.watch(context)); + const members = [{ member: EDITOR.state.notes!, ops: jsonPart([["r", []]]) }]; + const retired = [EDITOR.state.notes!, EDITOR.state.count!]; + peer.send({ kind: "state", holds: [{ hold, members, failed: [], retired }] }); + await delivered(); + expect(list).toEqual([{ kind: "value", value: [], ops: [["r", []]], retired: true }]); + expect(count).toEqual([{ kind: "value", value: 1, ops: [], retired: true }]); + }); + + it("of a failed member wait for its next value: never read from the failed replica; the link stays open", async () => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer, [{ text: "a" }]); + const notes = await got; + const count = changesOf(await notes.count.watch(context)); + const member = EDITOR.state.count!; + const failed = [{ member, error: wireError(new PluginError("derive", "count threw")) }]; + peer.send({ kind: "state", holds: [{ hold, members: [], failed, retired: [] }] }); + // A retirement alone (no ops) of the failed member, then its healed value. + peer.send({ kind: "state", holds: [{ hold, members: [], failed: [], retired: [member] }] }); + await delivered(); + const value = [{ member, ops: jsonPart([["r", 5]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: value, failed: [], retired: [] }] }); + await transported(); + expect([peer.closed, notes.state.value, errors]).toEqual([false, { kind: "live" }, []]); + expect(count).toEqual([ + { kind: "failed", error: expect.objectContaining({ message: "count threw" }) }, + { kind: "value", value: 5, ops: [["r", 5]], retired: true }, + ]); + }); + + it.each([ + ["names a member outside the variant", [7]], + ["names a member twice", [EDITOR.state.count!, EDITOR.state.count!]], + ])("that %s closes the link", async (_, retired) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + await got; + peer.send({ kind: "state", holds: [{ hold, members: [], failed: [], retired }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + }); + + describe("a snapshot", () => { + it.each([ + ["misses a member", [{ member: EDITOR.state.notes!, value: "[]" }], EDITOR.variant], + [ + "names a member twice", + [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ], + EDITOR.variant, + ], + ["names a member outside the variant", [{ member: 7, value: "0" }], EDITOR.variant], + ["names a variant outside the version", [], 5], + ])("that %s closes the link; the reference stays loading", async (_, members, variant) => { + const { client, peer, errors } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ kind: "bound", holds: [{ hold, variant, members, failed: [] }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(notes.variant).toBeUndefined(); + }); + }); + + describe("transient failures over a link", () => { + fakeTimers(); + + it("are waited through; the deadline rejects with the last cause and releases the hold", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const opens: string[] = []; + const { client } = await linked((h) => + h.serve(Flaky, ({ id }, _principal, self) => { + self.open( + async (ctx) => { + opens.push(`open ${opens.length + 1}`); + if (opens.length === 1) throw new Error("offline"); + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + opens.push("aborted"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + const got = rejection(client.get(Flaky, { id: 1 }, context, { readyTimeout: 1_500 })); + await transported(); + await vi.advanceTimersByTimeAsync(1_000); + await transported(); + expect(opens).toEqual(["open 1", "open 2"]); + await vi.advanceTimersByTimeAsync(500); + expect(await got).toEqual({ code: "unavailable", message: "app.flaky(1): offline" }); + await transported(); + // The release ended the object: its pending attempt was abandoned. + expect(opens).toEqual(["open 1", "open 2", "aborted"]); + }); + }); + + /** Every remote object bound while this file runs, weakly. */ + const remoteObjects = instancesOf(RemoteObject, "hold"); + + describe("GC of a released remote reference", () => { + it("a released reference retained by the test keeps no hold, replica or object", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const { notes, refs } = await (async () => { + const made = remoteObjects.length; + const notes = await client.get(Notes, { session: "S1" }, context); + const refs: WeakRef[] = [new WeakRef(resolveWatchable(notes.notes)!), ...remoteObjects.slice(made)]; + expect(refs).toHaveLength(2); + notes.release(); + await transported(); + return { notes, refs }; + })(); + await expectCollected(...refs); + expect([notes, server, client]).toBeDefined(); + }); + + it("a failure call's error kept by the app keeps neither the replica nor the remote object", async () => { + const Halves = remote("app.halves", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, + }); + const { server, client } = await linked((h) => + h.serve(Halves, (_, _principal, self) => { + const doc = self.state(Counter); + return { + variant: "v", + members: { + half: self.derive([doc], ([{ n }]) => { + // A fresh error: its stack's frames would keep what threw it. + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await doc.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + }, + }; + }), + ); + const kept: unknown[] = []; + const refs = await (async () => { + const made = remoteObjects.length; + const halves = await client.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + const refs: WeakRef[] = [new WeakRef(resolveWatchable(halves.half)!), ...remoteObjects.slice(made)]; + (await halves.half.watch(context)).start(async (call) => { + if (call.kind === "failed") kept.push(call.error); + }); + await halves.set(2, context); + await transported(); + halves.release(); + await transported(); + return refs; + })(); + expect(kept).toHaveLength(1); + await expectCollected(...refs); + expect([server, client]).toBeDefined(); + }); + }); + + describe("self.get of a type over the link (dependency model §3)", () => { + it("a failed hold parks the dependent failed { waiting }: no new attempt or hold; bound wakes it", async () => { + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + let factories = 0; + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + factories++; + return { count: self.get(Notes, { session: "S1" }).count }; + }), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "warming" } } }, + }); + // Parked on a remote blocker: its first outcome, so the start barrier is over. + await loading; + await transported(); + expect(errors).toEqual([]); + const view = client.hold(View, {}, context); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.notes(S1): warming" } }, + }); + bindHold(peer, hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(view.count.value).toBe(1); + expect([factories, peer.kinds().filter((kind) => kind === "hold" || kind === "release")]).toEqual([1, []]); + }); + }); + + describe("self.get through a relay whose upstream dropped (R18)", () => { + it("a loading notice while loading changes nothing: the dependent goes on waiting and goes live when bound", async () => { + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const { hold } = await peer.next("hold"); + // The relay's upstream link dropped before the hold was bound: a drop is not a dependency failure (audit). + peer.send({ kind: "loading", hold }); + await transported(); + bindHold(peer, hold, [{ text: "a" }]); + await loading; + expect(errors).toEqual([]); + const view = await client.get(View, {}, context); + expect(view.count.value).toBe(1); + }); + }); + + describe("a dependent of an object failed transiently over the link", () => { + // Timers stand still: neither the backoff window nor a readyTimeout ends while the test runs. + fakeTimers(); + + it("parks at once on the real cause when its load holds it (no wait for a timeout)", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); + const { server, client: first } = await linked((h) => + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + // The first client's get makes the object fail; it stays failed for its backoff window. + const failing = first.get(Flaky, {}, context).catch(() => {}); + await transported(); + const errors: HostError[] = []; + const second = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + second.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + const facet = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Flaky, {}).n }))); + await second.load([{ name: "@test/view", facet }], context); + await transported(); + expect(errors).toEqual([]); + expect(second.hold(View, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.flaky: app.flaky: offline" } }, + }); + await first.close(context); + await failing; + }); + }); +}); + +/** A remote type whose variants share `y`: `a` has `x` and a method, `b` has `z`. */ +const Shifting = remote("app.shifting", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + a: { state: { x: int(), y: int() }, methods: { m: method() } }, + b: { state: { y: int(), z: int() } }, + }, +}); + +/** A client whose `View` derives `x`+`y` of a `Shifting` reference, connected to a raw provider; `View` is bound. */ +async function shifting() { + const View = local("app.view", { version: 1, params: {}, members: { sum: int(), label: str() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + let held: Reference | undefined; + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + const ref = self.get(Shifting, {}); + held = ref; + const { x, y } = ref as RefIn; + return { + sum: self.derive([x, y], ([x, y]) => x + y), + // Reads the reference's variant when `y` changes. + label: self.derive([y], ([y]) => `${ref.variant}:${y}`), + }; + }), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const A = wireNames(Shifting, "a"); + const { hold } = await peer.next("hold"); + const members = [ + { member: A.state.x!, value: "1" }, + { member: A.state.y!, value: "2" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: A.variant, members, failed: [] }] }); + await loading; + const view = await client.get(View, {}, context); + return { client, peer, view, ref: held!, errors }; +} + +forEachCodec(() => { + describe("re-holds after a drop", () => { + it("hold every reference not released again on the next link, with that link's fresh ids, in order", async () => { + const { client, peer } = await rawProvider(); + const refs = ["S1", "S2", "S3"].map((session) => client.hold(Notes, { session }, context)); + for (let i = 0; i < 3; i++) await bindNext(peer); + refs[1]!.release(); + peer.close(); + await transported(); + const next = await rawParent(client); + const holds = [await next.next("hold"), await next.next("hold")]; + expect(holds.map(({ hold, params }) => [hold, partValue(params)])).toEqual([ + [0, { session: "S1" }], + [1, { session: "S3" }], + ]); + }); + + it("apply bound in one batch: each member gets one frame (a diff from its last value) and a fresh decoder", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer, [{ text: "a" }]); + await transported(); + const frames = framesOf(await notes.notes.watch(context)); + const counts = framesOf(await notes.count.watch(context)); + peer.close(); + await transported(); + const next = await rawParent(client); + const hold = await bindNext(next, [{ text: "a" }, { text: "b" }]); + await delivered(); + const list = [{ text: "a" }, { text: "b" }]; + // One frame each, a diff from the last value the reference kept while its hold re-bound. + expect(frames).toEqual([[list, [["p", [], 1, 0, [{ text: "b" }]]]]]); + expect(counts).toEqual([[2, [["r", 2]]]]); + // The fresh decoder: the provider's new encoder starts over, interning paths anew. + next.send({ + kind: "state", + holds: [{ hold, members: [{ member: EDITOR.state.count!, ops: '[["r",3]]' }], failed: [], retired: [] }], + }); + await delivered(); + expect(counts.at(-1)).toEqual([3, [["r", 3]]]); + }); + + it("a derive over two members of a re-held reference sees the snapshot whole: one batch", async () => { + const { client, peer, view } = await shifting(); + const sums = framesOf(await view.sum.watch(context)); + peer.close(); + await transported(); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const A = wireNames(Shifting, "a"); + const members = [ + { member: A.state.x!, value: "10" }, + { member: A.state.y!, value: "20" }, + ]; + next.send({ kind: "bound", holds: [{ hold, variant: A.variant, members, failed: [] }] }); + await transported(); + expect(sums.map(([sum]) => sum)).toEqual([30]); + }); + + it("another variant: vanished members fail (watches stay open), derives over them fail, new members get values, parked calls are checked (race 14)", async () => { + const { client, peer, view, ref } = await shifting(); + expect(view.sum.value).toBe(3); + const a = ref as RefIn; + const x = changesOf(await a.x.watch(context)); + const y = framesOf(await a.y.watch(context)); + const sum = changesOf(await view.sum.watch(context)); + const labels = framesOf(await view.label.watch(context)); + peer.close(); + await transported(); + const parked = rejection(a.m(context)); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const B = wireNames(Shifting, "b"); + const members = [ + { member: B.state.y!, value: "20" }, + { member: B.state.z!, value: "30" }, + ]; + next.send({ kind: "bound", holds: [{ hold, variant: B.variant, members, failed: [] }] }); + await transported(); + expect(ref.variant).toBe("b"); + const vanished = { kind: "failed", error: expect.objectContaining({ code: "invalid", reason: "member" }) }; + expect(x).toEqual([vanished]); + expect(sum).toEqual([vanished]); + expect(y).toEqual([[20, [["r", 20]]]]); + // The variant changed in the same batch: the derive recomputed with the new one. + expect(labels.map(([label]) => label)).toEqual(["b:20"]); + expect((ref as RefIn).z.value).toBe(30); + expect(() => a.x.value).toThrow('app.shifting (variant b) has no state member "x"'); + expect(await parked).toEqual({ + code: "invalid", + reason: "member", + message: 'app.shifting (variant b) has no method "m"', + }); + // A later binding of the variant that has `x` brings it back in the same watches; the derive heals. + next.close(); + await transported(); + const third = await rawParent(client); + const again = (await third.next("hold")).hold; + const A = wireNames(Shifting, "a"); + const back = [ + { member: A.state.x!, value: "5" }, + { member: A.state.y!, value: "6" }, + ]; + third.send({ kind: "bound", holds: [{ hold: again, variant: A.variant, members: back, failed: [] }] }); + await transported(); + expect(x.at(-1)).toEqual({ kind: "value", value: 5, ops: [["r", 5]], retired: false }); + expect(view.sum.value).toBe(11); + expect(sum.at(-1)).toEqual({ kind: "value", value: 11, ops: [["r", 11]], retired: false }); + }); + + it("calls sent before a drop fail unavailable, never resent; an old link's leftovers never reach the next (race 1)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + const { context: cancellable, cancel } = withCancel(context); + const sent = notes.wait(cancellable); + await peer.next("call"); + peer.close(); + expect(await rejection(sent)).toEqual({ code: "unavailable", message: "the link closed" }); + const next = await rawParent(client); + const hold = await bindNext(next); + await transported(); + cancel(new Error("late")); + notes.release(); + await transported(); + expect(next.received).toEqual([{ kind: "ack" }, { kind: "release", hold }]); + }); + + it("a drop before bound, or during the next link's handshake, leaves it loading; the next connect holds again (race 5)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await peer.next("hold"); + peer.close(); + await transported(); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + // The peer never answers the handshake; the hold waits in the channel, then the link drops. + b.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await bindNext(await rawParent(client)); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a re-hold refused after a provider upgrade stays failed; the next connect holds it again", async () => { + const NotesV2 = remote("app.notes", { + version: 2, + params: { session: str() }, + hostAddress: () => Server(), + variants: { editor: { state: { count: int() } } }, + }); + const { server, client, links } = await linked((h) => void serveNotes(h)); + const upgraded = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + const facet = defineFacet((h) => + h.serve(NotesV2, () => ({ variant: "editor", members: { count: constant(0) } })), + ); + await upgraded.load([{ name: "@test/app", facet }], context); + const notes = await client.get(Notes, { session: "S1" }, context); + /** Drops the client's link and connects it to `to`. */ + const relink = async (to: Host, from: Link) => { + from.close(); + await transported(); + const [a, b] = memoryLinkPair(); + to.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await transported(); + return a; + }; + const toUpgraded = await relink(upgraded, links[0]); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + retryAt: undefined, + }); + notes.retry(); + await transported(); + expect(notes.state.value.kind).toBe("failed"); + await relink(server, toUpgraded); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("ref.retry() of a broken failure sends retry on the same hold: no release, no new hold (audit)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "forbidden", message: "not authorized" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + notes.retry(); + expect(await peer.next("retry")).toEqual({ kind: "retry", hold, hops: 16 }); + bindHold(peer, hold); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(peer.received.filter(({ kind }) => kind === "hold" || kind === "release")).toEqual([]); + }); + + it("ref.retry() of an object whose factory threw, held twice: failed, loading, live; a call made then succeeds (audit)", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { only: { state: {}, methods: { ping: method() } } }, + }); + let factories = 0; + const { client } = await linked((h) => + h.serve(Flaky, () => { + if (++factories === 1) throw new Error("bad factory"); + return { variant: "only", members: { ping: async () => null } }; + }), + ); + const one = client.hold(Flaky, {}, context); + const two = client.hold(Flaky, {}, context); + await transported(); + const states: string[] = []; + const watch = await one.state.watch(context); + states.push(watch.value.kind); + watch.start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); + }); + one.retry(); + // A call made while it loads again waits for the binding; none sees the old failure. + let called: Promise | undefined; + const unwatch = await one.state.watch(context); + unwatch.start(async (call) => { + if (call.kind === "value" && call.value.kind === "loading" && called === undefined) { + called = one.ping(context).then( + () => "ok", + (error: Error) => `rejected: ${error.message}`, + ); + } + }); + await transported(); + await transported(); + expect(states).toEqual(["failed", "loading", "live"]); + expect(await called).toBe("ok"); + expect(two.state.value).toEqual({ kind: "live" }); + expect(factories).toBe(2); + }); + + it("a drop while failed: loading, held again on the next link (race 19)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "no configuration" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value.kind).toBe("failed"); + peer.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await bindNext(await rawParent(client)); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a member watch before the first binding waits for it through a failure, and rejects when released (race 16)", async () => { + const { client, peer } = await rawProvider(); + const bound = client.hold(Notes, { session: "S1" }, context); + const failing = client.hold(Notes, { session: "S2" }, context); + const released = client.hold(Notes, { session: "S3" }, context); + const watches = [bound.count.watch(context), failing.count.watch(context), released.count.watch(context)]; + await bindNext(peer, [{ text: "a" }]); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + peer.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error } }, + retryIn: 1_000, + }); + released.release(); + expect((await watches[0]!).value).toBe(1); + expect(await rejection(watches[2]!)).toEqual({ code: "released", message: "app.notes: released" }); + // A failure is waited through (Opus G3): the watch rejects only when its reference ends. + let settled = false; + const settle = (): void => { + settled = true; + }; + void watches[1]!.then(settle, settle); + await transported(); + expect([failing.state.value.kind, settled]).toEqual(["failed", false]); + failing.release(); + expect(await rejection(watches[1]!)).toMatchObject({ code: "released" }); + }); + }); + + describe("a get across a drop", () => { + fakeTimers(); + + it("keeps waiting within its one deadline, from the moment it was called", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context, { readyTimeout: 1_000 }); + const late = rejection(client.get(Notes, { session: "S2" }, context, { readyTimeout: 1_000 })); + await peer.next("hold"); + await peer.next("hold"); + await vi.advanceTimersByTimeAsync(400); + peer.close(); + await transported(); + const next = await rawParent(client); + await bindNext(next, [{ text: "a" }]); + expect((await got).count.value).toBe(1); + await vi.advanceTimersByTimeAsync(600); + expect(await late).toEqual({ + code: "unavailable", + message: "app.notes(S2): did not become live within 1000 ms", + }); + }); + }); + + describe("an activating dependent whose remote dependency's link drops (audit; supersedes race 18)", () => { + fakeTimers(); + + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + ); + + it("stays loading through the drop, no failure reported, and goes live when the next link binds it", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const loading = client.load([{ name: "@test/view", facet }], context); + await peer.next("hold"); + peer.close(); + await transported(); + const view = client.hold(View, {}, context); + expect(view.state.value).toEqual({ kind: "loading" }); + const next = await rawParent(client); + expect(await bindNext(next, [{ text: "a" }])).toBe(0); + await loading; + await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + }); + + it("dropped and never bound again, stays parked loading: nothing fails, one parked_long (audit A11)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const loading = client.load([{ name: "@test/view", facet }], context); + await peer.next("hold"); + // A remote blocker ends the start barrier's first pass. + await loading; + peer.close(); + await transported(); + await vi.advanceTimersByTimeAsync(10_000); + // A diagnostic whatever the blocker (audit A11): its path without a link too; nothing failed. + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view", waitingFor: "app.notes(S1)", state: "loading", since: 0 }, + ]); + expect(client.hold(View, {}, context).state.value).toEqual({ kind: "loading" }); + }); + + it("without any link stays loading past its readyTimeout and goes live when the first link binds its one hold", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + await client.load([{ name: "@test/view", facet }], context); + await vi.advanceTimersByTimeAsync(500); + expect(failures(errors)).toEqual([]); + const next = await rawParent(client); + expect(await bindNext(next, [{ text: "a" }])).toBe(0); + expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); + expect(next.kinds().filter((kind) => kind === "hold" || kind === "release")).toEqual([]); + }); + }); + + describe("GC across a drop", () => { + it("a reference that lived through a drop keeps nothing of the closed link", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + const closed = await (async () => { + const [a, b] = memoryLinkPair(); + peer.close(); + await transported(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const second = new RawPeer(b); + await second.next("hello"); + await bindNext(second); + second.close(); + await transported(); + return [new WeakRef(a), new WeakRef(second)]; + })(); + await bindNext(await rawParent(client)); + await expectCollected(...closed); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + }); +}); diff --git a/packages/chord/test/next/remote/replica.test.ts b/packages/chord/test/next/remote/replica.test.ts new file mode 100644 index 000000000..e42bb477b --- /dev/null +++ b/packages/chord/test/next/remote/replica.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { Replica } from "../../../src/next/remote/replica.ts"; +import { Batch, Derive } from "../../../src/next/state/internal.ts"; +import { changesOf, delivered, framesOf } from "../support.ts"; + +/** A replica of `{ n }` on a fresh batch, with a derive over it attached. */ +function replica() { + const batch = new Batch(() => {}, undefined); + const source = new Replica(batch, { kind: "value", value: { n: 1 } }); + const double = new Derive([source], ([value]) => (value as { n: number }).n * 2, batch); + double.attach( + (declared) => declared as Replica, + () => {}, + ); + return { batch, source, double }; +} + +describe("replicas", () => { + it("adopt a value and its ops inside a batch: one frame, derives recomputed", async () => { + const { batch, source, double } = replica(); + const frames = framesOf(source.openWatch(context)); + const doubles = framesOf(double.openWatch(context)); + batch.commit(() => source.adopt({ n: 2 }, [["s", ["n"], 2]], false), context); + await delivered(); + expect(frames).toEqual([[{ n: 2 }, [["s", ["n"], 2]]]]); + expect(doubles).toEqual([[4, [["r", 4]]]]); + }); + + it("fail: reads throw the error, watches and derives over it get one failure call and recover with a root replacement", async () => { + const { batch, source, double } = replica(); + const calls = changesOf(source.openWatch(context)); + const derived = changesOf(double.openWatch(context)); + const error = new Error("member failed"); + batch.commit(() => source.fail(error), context); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error }]); + expect(derived).toEqual([{ kind: "failed", error }]); + expect(() => source.value).toThrow("member failed"); + // A non-root op cannot apply to a failed member; its recovery is a root replacement. + expect(() => source.decode([["s", ["n"], 2]])).toThrow("root replacement"); + const { value, ops } = source.decode([["r", { n: 3 }]]); + batch.commit(() => source.adopt(value, ops, false), context); + await delivered(); + expect(calls.at(-1)).toEqual({ kind: "value", value: { n: 3 }, ops: [["r", { n: 3 }]], retired: false }); + expect(derived.at(-1)).toEqual({ kind: "value", value: 6, ops: [["r", 6]], retired: false }); + }); + + it("fail once per episode: a second failure while failed gives no call; a retirement meanwhile waits for the value", async () => { + const { batch, source } = replica(); + const calls = changesOf(source.openWatch(context)); + const first = new Error("first"); + batch.commit(() => source.fail(first), context); + batch.commit(() => source.fail(new Error("second")), context); + await delivered(); + // A retirement alone while failed: the value is not read, the flag waits. + batch.commit(() => source.retireAlone(), context); + const { value, ops } = source.decode([["r", { n: 5 }]]); + batch.commit(() => source.adopt(value, ops, false), context); + await delivered(); + expect(calls).toEqual([ + { kind: "failed", error: first }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + ]); + }); + + it("open a new watch while failed at the last value, the failure first; with no value yet, reject it", async () => { + const { batch, source } = replica(); + const error = new Error("member failed"); + batch.commit(() => source.fail(error), context); + const watch = source.openWatch(context); + expect(watch.value).toEqual({ n: 1 }); + const calls = changesOf(watch); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error }]); + // Failed since its binding: there is no value to start at. + const unborn = new Replica(batch, { kind: "failed", error }); + expect(() => unborn.openWatch(context)).toThrow(error); + }); + + it("end with their hold: reads throw its error, watches end", async () => { + const { source } = replica(); + const watch = source.openWatch(context); + source.end(new ChordError({ code: "released", message: "app.notes: ended" })); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => source.value).toThrow("app.notes: ended"); + }); +}); diff --git a/packages/chord/test/next/schema/declarations.ts b/packages/chord/test/next/schema/declarations.ts new file mode 100644 index 000000000..cf0a329c0 --- /dev/null +++ b/packages/chord/test/next/schema/declarations.ts @@ -0,0 +1,180 @@ +// The declarations of the schema switch's proofs (TypeBox to pi-schema): one of every schema helper and option in a +// contract, persisted documents, tables with defaults, an optional and a `json(...)` column, an evolved table and a +// persisted job. Their fingerprints were recorded with the TypeBox helpers; the stored data in `stored/` was written +// by that code (`write-stored.ts`). + +import { + arr, + bool, + bytes, + hostKind, + int, + keyed, + lit, + local, + map, + method, + nul, + num, + obj, + oneOf, + opt, + remote, + str, + union, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { defineTable, evolveTable, json } from "../../../src/next/store/index.ts"; + +/** A schema with every helper and option a declaration can use. */ +export const everything = obj({ + name: str({ min: 1, max: 40, pattern: "^[a-z ]+$" }), + nick: str({ default: "x" }), + age: int({ min: 0, max: 150 }), + count: int({ default: 0 }), + score: num({ min: 0, max: 100 }), + ratio: num({ default: 0.5 }), + active: bool(), + verified: bool({ default: false }), + role: oneOf("admin", "user", "guest"), + level: oneOf(1, 2, 3), + kind: lit("person"), + one: lit(1), + yes: lit(true), + tags: arr(str(), { min: 0, max: 5 }), + points: arr(int()), + extra: map(int()), + note: opt(str()), + nothing: nul(), + choice: union([str(), int(), obj({ deep: arr(obj({ x: num() })) })]), + nested: obj({ inner: opt(obj({ v: str() })) }), +}); + +/** A local contract with state, keyed state and methods of every argument shape. */ +export const Everything = local("fixture.switch.everything", { + version: 1, + params: { id: str(), shard: int() }, + members: { + profile: everything, + notes: keyed(obj({ body: str() })), + plain: method(), + args: method(obj({ a: str(), b: opt(int()), any: unknown() })), + result: method(undefined, arr(str())), + both: method(obj({ when: int({ min: 0 }) }), union([lit("ok"), obj({ error: str() })])), + blob: method(obj({ data: bytes({ max: 1024 }) }), bytes()), + }, +}); + +const Server = hostKind("fixture.switch.server", { region: str() }); + +/** A remote contract with variants and a host address. */ +export const Account = remote("fixture.switch.account", { + version: 1, + params: { user: str({ min: 1 }) }, + hostAddress: () => Server({ region: "eu" }), + variants: { + owner: { state: { balance: num() }, methods: { pay: method(obj({ amount: num({ min: 0 }) }), bool()) } }, + guest: { state: { balance: num() } }, + }, +}); + +/** A persisted singleton document of `everything`. */ +export const Profile = defineState("fixture.switch.profile", { + schema: everything, + version: 1, + initial: () => ({ + name: "ada", + nick: "x", + age: 36, + count: 0, + score: 1, + ratio: 0.5, + active: true, + verified: false, + role: "admin" as const, + level: 1 as const, + kind: "person" as const, + one: 1 as const, + yes: true as const, + tags: [], + points: [], + extra: {}, + nothing: null, + choice: "a", + nested: {}, + }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 2, +}); + +/** A persisted keyed document. */ +export const Notes = defineState("fixture.switch.notes", { + schema: obj({ body: str(), tags: opt(arr(str())) }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +/** A table with defaults, an optional column and a `json(...)` column. */ +export const Rows = defineTable("fixture.switch.rows", { + version: 1, + columns: { + id: str(), + n: int({ default: 0 }), + flag: bool({ default: false }), + label: opt(str()), + data: json(obj({ a: int(), b: opt(arr(str())) }), { version: 1 }), + maybe: opt(json(map(num()), { version: 1 })), + }, + key: ["id"], + indexes: { by_n: ["n", "id"] }, +}); + +/** A table evolved over three versions: a rename, an added column with a default, a json column, a drop. */ +export const EditsV1 = defineTable("fixture.switch.edits", { + version: 1, + columns: { id: str(), file: str(), at: int() }, + key: ["id"], +}); + +export const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); + +export const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +/** A persisted job with a state document. */ +export const ImportState = defineState("fixture.switch.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, +}); + +export const ImportJob = defineJob("fixture.switch.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + persist: { resume: "rerun" }, +}); + +/** The object owning the persisted documents and the tables (a local type with empty params, as tables need). */ +export const Owner = local("fixture.switch.owner", { + version: 1, + params: {}, + members: { + profile: everything, + write: method(obj({ id: str(), body: str() })), + read: method(str(), str()), + dump: method(obj({}), unknown()), + }, +}); diff --git a/packages/chord/test/next/schema/owner.ts b/packages/chord/test/next/schema/owner.ts new file mode 100644 index 000000000..88928df5e --- /dev/null +++ b/packages/chord/test/next/schema/owner.ts @@ -0,0 +1,85 @@ +// The object of the stored-data proof: it owns the persisted documents, the tables and the job of `declarations.ts`, +// writes them (`write-stored.ts`, with the TypeBox helpers) and reads them back (the test, after the switch). + +import type { Context } from "../../../src/context/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import type { TableDefinition } from "../../../src/next/store/index.ts"; +import { openStore, query, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { ImportJob, Notes, Owner, Profile, Rows } from "./declarations.ts"; + +/** What the job's `run` does in this process: wait until its context ends, or finish at once. */ +export type RunBehaviour = "wait" | "finish"; + +/** The finished runs' results, in order. */ +export const finished: unknown[] = []; + +/** The owner's facet over `edits` (the table version this process declares). */ +export function ownerFacet(edits: TableDefinition, behaviour: RunBehaviour) { + return defineFacet((host) => { + host.serve(Owner, (_, self) => { + const profile = self.state(Profile); + const notes = self.state(Notes); + const rows = self.table(Rows); + const editsTable = self.table(edits); + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + await ctx.state.change((draft) => { + draft.done += path.length; + }, ctx); + if (behaviour === "wait") { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + } + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + finished.push(outcome.kind === "done" ? outcome.result : outcome.kind); + }, + }); + return { + profile, + write: async ({ id, body }, ctx) => { + await self.tx(async (tx) => { + (await tx.draft(notes, id)).body = body; + const draft = await tx.draft(profile); + draft.count += 1; + draft.tags.push(id); + draft.extra[id] = body.length; + draft.choice = { deep: [{ x: draft.count }] }; + draft.nested = { inner: { v: body } }; + tx.put(rows, { + id, + label: body, + data: { a: body.length, b: [id] }, + ...(id === "a" ? { maybe: { n: 1.5 } } : {}), + }); + tx.put(editsTable, { id, file: `/notes/${id}.md`, at: body.length } as never); + }, ctx); + await imports.start({ path: id }, ctx); + return null; + }, + read: async (id, ctx) => (await notes.get(id, ctx))?.body ?? "", + dump: async (_, ctx) => ({ + profile: profile.value, + notes: { a: (await notes.get("a", ctx)) ?? null, b: (await notes.get("b", ctx)) ?? null }, + rows: (await rows.page(query(Rows), 10, ctx)).items, + edits: (await editsTable.page(query(edits as TableDefinition), 10, ctx)).items, + runs: imports.running.value.length, + }), + }; + }); + }); +} + +/** A host serving the owner on the SQLite store at `path`. */ +export async function ownerHost( + path: string, + edits: TableDefinition, + behaviour: RunBehaviour, + context: Context, +): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: () => {}, closeTimeout: 100 }); + await host.load([{ name: "@fixture/owner", facet: ownerFacet(edits, behaviour) }], context); + return { store, host }; +} diff --git a/packages/chord/test/next/schema/proofs.ts b/packages/chord/test/next/schema/proofs.ts new file mode 100644 index 000000000..6f285111e --- /dev/null +++ b/packages/chord/test/next/schema/proofs.ts @@ -0,0 +1,113 @@ +// The values the schema switch must keep (`recorded.json`, made with the TypeBox helpers): the fingerprints and normal +// forms of `declarations.ts`, and what a host reads back from the stored data in `stored/`. + +import { createReadStream } from "node:fs"; +import { copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import type { Contract } from "../../../src/next/contracts/index.ts"; +import { contractData } from "../../../src/next/contracts/internal.ts"; +import { snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; +import { decodeJsonl, importStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import * as declared from "./declarations.ts"; +import { finished, ownerHost } from "./owner.ts"; + +/** Every fingerprint and normal form of the declarations. */ +export function fingerprints(): JsonValue { + const contract = (C: Contract) => { + const data = contractData(C); + return { + params: data.params.normal, + versions: data.versions.map((version) => ({ + fingerprint: version.fingerprint, + variants: Object.fromEntries( + [...version.variants].map(([name, variant]) => [ + name, + { + state: Object.fromEntries([...variant.state].map(([m, s]) => [m, s.normal])), + keyed: Object.fromEntries([...variant.keyed].map(([m, s]) => [m, s.normal])), + methods: Object.fromEntries( + [...variant.methods].map(([m, { args, result }]) => [ + m, + { args: args.normal, result: result.normal }, + ]), + ), + }, + ]), + ), + })), + }; + }; + const table = (T: { readonly fingerprint: string; readonly record: unknown }) => ({ + fingerprint: T.fingerprint, + record: T.record as unknown as JsonValue, + }); + return { + contracts: { + everything: contract(declared.Everything), + account: contract(declared.Account), + owner: contract(declared.Owner), + }, + documents: { + profile: { + fingerprint: declared.Profile.fingerprint, + normal: snapshotSchema(declared.everything, "p").normal, + }, + notes: declared.Notes.fingerprint, + importState: declared.ImportState.fingerprint, + }, + tables: { + rows: table(declared.Rows), + editsV1: table(declared.EditsV1), + editsV2: table(declared.EditsV2), + editsV3: table(declared.EditsV3), + }, + job: declared.ImportJob.fingerprint, + }; +} + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); + +/** + * What a host of the current code reads from the stored data: from a copy of the SQLite store, and from a store + * imported from the archive (both written at table version 1, read at version 3; the stopped job runs resume). + */ +export async function storedReads(context: Context): Promise { + const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); + try { + const fromStore = join(directory, "store.db"); + await copyFile(join(stored, "store.db"), fromStore); + const store = await read(fromStore, context); + const imported = await importStore( + sqliteBackend(await openNodeSqliteDatabase(join(directory, "imported.db"))), + decodeJsonl(createReadStream(join(stored, "archive.jsonl"))), + context, + ); + await imported.close(context); + const archive = await read(join(directory, "imported.db"), context); + return { store, archive }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** The owner's dump on the store at `path`, after its stopped runs finished. */ +async function read(path: string, context: Context): Promise { + finished.length = 0; + const { store, host } = await ownerHost(path, declared.EditsV3, "finish", context); + const owner = await host.get(declared.Owner, {}, context); + for (let i = 0; i < 100 && finished.length < 3; i++) await new Promise((resolve) => setTimeout(resolve, 10)); + const dump = await owner.dump({}, context); + const result = { + dump: dump as JsonValue, + finished: [...finished] as JsonValue, + note: await owner.read("b", context), + }; + await host.close(context); + await store.close(context); + return result; +} diff --git a/packages/chord/test/next/schema/recorded.json b/packages/chord/test/next/schema/recorded.json new file mode 100644 index 000000000..00c0a3588 --- /dev/null +++ b/packages/chord/test/next/schema/recorded.json @@ -0,0 +1,1143 @@ +{ + "fingerprints": { + "contracts": { + "everything": { + "params": [ + [ + "id", + { + "type": "string" + } + ], + [ + "shard", + { + "type": "integer" + } + ] + ], + "versions": [ + { + "fingerprint": "3861c5a457ef4aa3", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": { + "notes": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + } + }, + "required": [ + "body" + ], + "type": "object" + } + }, + "methods": { + "plain": { + "args": { + "type": "null" + }, + "result": { + "type": "null" + } + }, + "args": { + "args": { + "additionalProperties": false, + "properties": { + "a": { + "type": "string" + }, + "any": {}, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "any" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "result": { + "args": { + "type": "null" + }, + "result": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "both": { + "args": { + "additionalProperties": false, + "properties": { + "when": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "when" + ], + "type": "object" + }, + "result": { + "anyOf": [ + { + "const": "ok", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + ] + } + }, + "blob": { + "args": { + "additionalProperties": false, + "properties": { + "data": { + "maxLength": 1024, + "type": "bytes" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "result": { + "type": "bytes" + } + } + } + } + } + } + ] + }, + "account": { + "params": [ + [ + "user", + { + "minLength": 1, + "type": "string" + } + ] + ], + "versions": [ + { + "fingerprint": "821be6a79fbe350a", + "variants": { + "owner": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": { + "pay": { + "args": { + "additionalProperties": false, + "properties": { + "amount": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "amount" + ], + "type": "object" + }, + "result": { + "type": "boolean" + } + } + } + }, + "guest": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": {} + } + } + } + ] + }, + "owner": { + "params": [], + "versions": [ + { + "fingerprint": "b01eec2d4e2311b0", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": {}, + "methods": { + "write": { + "args": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + }, + "id": { + "type": "string" + } + }, + "required": [ + "body", + "id" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "read": { + "args": { + "type": "string" + }, + "result": { + "type": "string" + } + }, + "dump": { + "args": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "result": {} + } + } + } + } + } + ] + } + }, + "documents": { + "profile": { + "fingerprint": "f345152aff1bb993", + "normal": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "notes": "bd44e8ea54b8f283", + "importState": "53d035820123516d" + }, + "tables": { + "rows": { + "fingerprint": "a8a7a46b1d1b5058", + "record": { + "id": { + "type": "str" + }, + "n": { + "type": "int", + "default": 0 + }, + "flag": { + "type": "bool", + "default": false + }, + "label": { + "type": "str", + "optional": true + }, + "data": { + "type": "json", + "version": 1, + "fingerprint": "595acb546ede10d5" + }, + "maybe": { + "type": "json", + "optional": true, + "version": 1, + "fingerprint": "27fc51c99dad9a88" + } + } + }, + "editsV1": { + "fingerprint": "b6a64f82180e9b7f", + "record": { + "id": { + "type": "str" + }, + "file": { + "type": "str" + }, + "at": { + "type": "int" + } + } + }, + "editsV2": { + "fingerprint": "2fbd0130077d5eed", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + }, + "dir": { + "type": "str", + "default": "" + } + } + }, + "editsV3": { + "fingerprint": "303055ac505946c3", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + } + } + } + }, + "job": "c280624818bc83ab" + }, + "stored": { + "store": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + }, + "archive": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + } + } +} diff --git a/packages/chord/test/next/schema/schema-switch.test.ts b/packages/chord/test/next/schema/schema-switch.test.ts new file mode 100644 index 000000000..830593422 --- /dev/null +++ b/packages/chord/test/next/schema/schema-switch.test.ts @@ -0,0 +1,22 @@ +// Tests of the schema switch from TypeBox to pi-schema (decision "A schema package beneath pi-ai and Chord"): the same +// declarations give byte-identical fingerprints and normal forms, and data stored by the code before the switch loads +// after it (`recorded.json` and `stored/` were made with the TypeBox helpers, see `write-stored.ts`). + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { fingerprints, storedReads } from "./proofs.ts"; + +const recorded = JSON.parse(readFileSync(new URL("./recorded.json", import.meta.url), "utf8")); + +describe("fingerprints", () => { + it("keeps every fingerprint and normal form of contracts, documents, tables, json columns and jobs", () => { + expect(fingerprints()).toEqual(recorded.fingerprints); + }); +}); + +describe("stored data", () => { + it("reads a store and an archive written before the switch: documents, tables, a migrated table, stopped runs", async () => { + expect(await storedReads(context)).toEqual(recorded.stored); + }); +}); diff --git a/packages/chord/test/next/schema/stored/archive.jsonl b/packages/chord/test/next/schema/stored/archive.jsonl new file mode 100644 index 000000000..7fcbd5972 --- /dev/null +++ b/packages/chord/test/next/schema/stored/archive.jsonl @@ -0,0 +1,36 @@ +{"kind":"manifest","format":1,"schema":2,"lastSeq":13,"nextId":12} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.edits","version":1,"state":"ready","columns":{"id":{"type":"str"},"file":{"type":"str"},"at":{"type":"int"}},"key":["id"],"fingerprint":"b6a64f82180e9b7f"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"a","file":"/notes/a.md","at":10},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"b","file":"/notes/b.md","at":6},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"c","file":"/notes/c.md","at":9},"version":1,"columns":{}} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.rows","version":1,"state":"ready","columns":{"id":{"type":"str"},"n":{"type":"int","default":0},"flag":{"type":"bool","default":false},"label":{"type":"str","optional":true},"data":{"type":"json","version":1,"fingerprint":"595acb546ede10d5"},"maybe":{"type":"json","optional":true,"version":1,"fingerprint":"27fc51c99dad9a88"}},"key":["id"],"fingerprint":"a8a7a46b1d1b5058"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"a","n":0,"flag":false,"label":"first note","data":{"a":10,"b":["a"]},"maybe":{"n":1.5}},"version":1,"columns":{"data":1,"maybe":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"b","n":0,"flag":false,"label":"second","data":{"a":6,"b":["b"]}},"version":1,"columns":{"data":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"c","n":0,"flag":false,"label":"third one","data":{"a":9,"b":["c"]}},"version":1,"columns":{"data":1}} +{"kind":"index","table":"fixture.switch.owner:fixture.switch.rows","name":"by_n","columns":["n","id"]} +{"kind":"document.create","record":{"id":2,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"a","history":"none","createdAt":2,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":2,"seq":2,"content":{"kind":"base","version":1,"value":{"body":"first note"}}} +{"kind":"document.create","record":{"id":3,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.profile","history":"none","createdAt":2,"revision":3,"version":1,"fingerprint":"f345152aff1bb993"}} +{"kind":"document.revision","id":3,"seq":2,"content":{"kind":"base","version":1,"value":{"name":"ada","nick":"x","age":36,"count":1,"score":1,"ratio":0.5,"active":true,"verified":false,"role":"admin","level":1,"kind":"person","one":1,"yes":true,"tags":["a"],"points":[],"extra":{"a":10},"nothing":null,"choice":{"deep":[{"x":1}]},"nested":{"inner":{"v":"first note"}}}}} +{"kind":"document.revision","id":3,"seq":5,"content":{"kind":"delta","version":1,"ops":[["s",["count"],2],["s",["choice"],{"deep":[{"x":2}]}],["s",["nested"],{"inner":{"v":"second"}}],["p",["tags"],1,0,["b"]],["s",["extra","b"],6]]}} +{"kind":"document.revision","id":3,"seq":9,"content":{"kind":"delta","version":1,"ops":[["s",["count"],3],["s",["choice"],{"deep":[{"x":3}]}],["s",["nested"],{"inner":{"v":"third one"}}],["p",["tags"],2,0,["c"]],["s",["extra","c"],9]]}} +{"kind":"document.create","record":{"id":4,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":4,"seq":4,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"a"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":5,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":5,"seq":3,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":5,"seq":6,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":6,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"b","history":"none","createdAt":5,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":6,"seq":5,"content":{"kind":"base","version":1,"value":{"body":"second"}}} +{"kind":"document.create","record":{"id":7,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":7,"seq":8,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"b"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":8,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":8,"seq":7,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":8,"seq":10,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":9,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"c","history":"none","createdAt":9,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":9,"seq":9,"content":{"kind":"base","version":1,"value":{"body":"third one"}}} +{"kind":"document.create","record":{"id":10,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":10,"seq":12,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"c"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":11,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":11,"seq":11,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":11,"seq":13,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"end","writes":34} diff --git a/packages/chord/test/next/schema/stored/store.db b/packages/chord/test/next/schema/stored/store.db new file mode 100644 index 000000000..f9b1e7083 Binary files /dev/null and b/packages/chord/test/next/schema/stored/store.db differ diff --git a/packages/chord/test/next/schema/write-stored.ts b/packages/chord/test/next/schema/write-stored.ts new file mode 100644 index 000000000..e40248d32 --- /dev/null +++ b/packages/chord/test/next/schema/write-stored.ts @@ -0,0 +1,40 @@ +// Writes the stored data of the schema switch's proof (`stored/`): a SQLite store and its export archive, made by the +// code before the switch (TypeBox helpers). Kept as it was written; rerun only to make a new fixture on purpose. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/schema/write-stored.ts + +import { createWriteStream } from "node:fs"; +import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { encodeJsonl } from "../../../src/next/store/index.ts"; +import { EditsV1, Owner } from "./declarations.ts"; +import { ownerHost } from "./owner.ts"; + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); +const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); +const path = join(directory, "store.db"); +try { + const { store, host } = await ownerHost(path, EditsV1, "wait", context); + const owner = await host.get(Owner, {}, context); + for (const [id, body] of [ + ["a", "first note"], + ["b", "second"], + ["c", "third one"], + ] as const) { + await owner.write({ id, body }, context); + } + await new Promise((resolve) => setTimeout(resolve, 200)); + await mkdir(stored, { recursive: true }); + await pipeline(encodeJsonl(store.export(context)), createWriteStream(join(stored, "archive.jsonl"))); + // A clean close: the waiting runs are stored `stopped`, resumed by the next host. + await host.close(context); + await store.close(context); + await copyFile(path, join(stored, "store.db")); + console.log("written to", stored); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/state/batch.test.ts b/packages/chord/test/next/state/batch.test.ts new file mode 100644 index 000000000..c1fd58801 --- /dev/null +++ b/packages/chord/test/next/state/batch.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import type { Host, Reference } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { Batch, type BatchError } from "../../../src/next/state/internal.ts"; +import { delivered, framesOf, loaded, rejection } from "../support.ts"; + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Probe = local("app.probe", { + version: 1, + params: {}, + members: { probe: int(), fixed: int(), set: method(int()) }, +}); + +/** Every Chord call a derive compute may not make, each refused or allowed. */ +type Attempts = { [operation: string]: unknown }; + +/** + * A host whose `probe` derive runs `attempt` on every compute with n > 0. `attempt` gets the host and a reference to + * the probe itself; what each operation threw is recorded. + */ +async function probe(attempt: (host: Host, self: Reference) => Attempts) { + const seen: Attempts[] = []; + const holder: { host?: Host; reference?: Reference } = {}; + const { host } = await loaded((h) => + h.serve(Probe, (_, self) => { + const num = self.state(Num); + return { + probe: self.derive([num], ([v]) => { + if (v.n > 0) seen.push(attempt(holder.host!, holder.reference!)); + if (v.n > 1) throw new Error("compute threw"); + return v.n; + }), + fixed: constant(1), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + holder.host = host; + holder.reference = await host.get(Probe, {}, context); + return { host, reference: holder.reference, seen }; +} + +/** What a synchronous throw or a rejection carries, as JSON for ChordErrors. */ +function outcome(run: () => unknown): unknown { + try { + const result = run(); + return result instanceof Promise ? rejection(result) : result; + } catch (error) { + return rejection(Promise.reject(error)); + } +} + +const reentrant = { code: "invalid", reason: "reentrant" }; + +describe("reentrancy", () => { + const attempts = (host: Host, self: Reference): Attempts => ({ + watch: outcome(() => self.probe.watch(context)), + "watch a constant member": outcome(() => self.fixed.watch(context)), + "watch ref.state": outcome(() => self.state.watch(context)), + release: outcome(() => self.release()), + get: outcome(() => host.get(Probe, {}, context)), + load: outcome(() => host.load([], context)), + close: outcome(() => host.close(context)), + set: outcome(() => self.set(5, context)), + }); + + it("refuses watch (ref.state too), release, get, load, close and a transaction in a commit's derive computes", async () => { + const { reference, seen } = await probe(attempts); + await reference.probe.watch(context); + await reference.set(1, context); + for (const result of await Promise.all(Object.values(seen[0]!))) expect(result).toMatchObject(reentrant); + }); + + it("refuses them in a lazy compute outside a commit", async () => { + const { reference, seen } = await probe(attempts); + await reference.set(1, context); + expect(seen).toEqual([]); + expect(reference.probe.value).toBe(1); + for (const result of await Promise.all(Object.values(seen[0]!))) expect(result).toMatchObject(reentrant); + }); + + it("clears the flag after a throwing compute", async () => { + const { host, reference } = await probe(() => ({})); + await reference.set(2, context); + expect(() => reference.probe.value).toThrow("compute threw"); + expect((await host.get(Probe, {}, context)).probe).toBeDefined(); + await expect(reference.set(3, context)).resolves.toBeNull(); + }); +}); + +describe("delivery", () => { + it("reports a delivery that throws and still delivers the later frames of the commit", async () => { + const Other = defineState("other", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Two = local("app.two", { + version: 1, + params: {}, + members: { bad: obj({ a: int() }), other: obj({ n: int() }), set: method(int()) }, + }); + const { host, errors } = await loaded((h) => + h.serve(Two, (_, self) => { + const num = self.state(Num); + const other = self.state(Other); + return { + // `undefined` is no JSON: diffing it throws. + bad: self.derive([num], ([v]) => (v.n === 0 ? { a: 0 } : ({ a: undefined } as never))), + other, + set: async (n, ctx) => { + // `num` is adopted first, so the derive's delivery comes before `other`'s. + await self.tx(async (tx) => { + (await tx.draft(num)).n = n; + (await tx.draft(other)).n = n; + }, ctx); + return null; + }, + }; + }), + ); + const two = await host.get(Two, {}, context); + await two.bad.watch(context); + const frames = framesOf(await two.other.watch(context)); + await two.set(1, context); + await delivered(); + expect(errors).toMatchObject([{ kind: "delivery_failed" }]); + expect(frames.map(([value]) => value)).toEqual([{ n: 1 }]); + }); +}); + +describe("flushes", () => { + it("run once per listing at the end of the commit, after its deliveries; a throwing one is reported", () => { + const reported: BatchError[] = []; + const batch = new Batch((error) => reported.push(error), undefined); + const order: string[] = []; + batch.commit(() => { + batch.flushing(() => order.push("first")); + batch.flushing(() => { + throw new Error("flush threw"); + }); + batch.flushing(() => order.push("third")); + order.push("adopted"); + }, context); + expect(order).toEqual(["adopted", "first", "third"]); + expect(reported).toMatchObject([{ kind: "delivery_failed", error: { message: "flush threw" } }]); + batch.commit(() => {}, context); + expect(order).toEqual(["adopted", "first", "third"]); + }); +}); diff --git a/packages/chord/test/next/state/derive.test.ts b/packages/chord/test/next/state/derive.test.ts new file mode 100644 index 000000000..a6434688f --- /dev/null +++ b/packages/chord/test/next/state/derive.test.ts @@ -0,0 +1,484 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; +import { Derive } from "../../../src/next/state/internal.ts"; +import type { Watchable } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + framesOf, + instancesOf, + loaded, + metered, + restartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Store = local("app.store", { + version: 1, + params: {}, + members: { n: int(), double: int(), set: method(int()) }, +}); + +/** A store with a number, its double, and a setter. */ +function serveStore(self: Self) { + const num = self.state(Num); + return { + n: self.derive([num], ([v]) => v.n), + double: self.derive([num], ([v]) => v.n * 2), + set: async (n: number, ctx: typeof context) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; +} + +/** Counts derive computes on a host's adapter. */ +async function computes() { + const telemetry = await metered(); + return { telemetry: telemetry.telemetry, count: () => telemetry.counts()["chord.derive.computes"] ?? 0 }; +} + +describe("derives across objects", () => { + it("update in the batch of the commit that changed their source's object", async () => { + const Viewer = local("app.viewer", { version: 1, params: {}, members: { label: str() } }); + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Viewer, (_, self) => ({ + label: self.derive([self.get(Store, {}).double], ([d]) => `double ${d}`), + })); + }); + const store = await host.get(Store, {}, context); + const viewer = await host.get(Viewer, {}, context); + const frames = framesOf(await viewer.label.watch(context)); + await store.set(2, context); + await delivered(); + expect(frames).toEqual([["double 4", [["r", "double 4"]]]]); + }); +}); + +describe("the pull model", () => { + it("computes a diamond's derives once each per batch", async () => { + const { telemetry, count } = await computes(); + const Diamond = local("app.diamond", { version: 1, params: {}, members: { d: int(), set: method(int()) } }); + const { host } = await loaded( + (h) => + h.serve(Diamond, (_, self) => { + const num = self.state(Num); + const b = self.derive([num], ([v]) => v.n + 1); + const c = self.derive([num], ([v]) => v.n + 2); + return { + d: self.derive([b, c], ([x, y]) => x + y), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + { telemetry }, + ); + const diamond = await host.get(Diamond, {}, context); + const frames = framesOf(await diamond.d.watch(context)); + const before = count(); + await diamond.set(1, context); + expect(count() - before).toBe(3); + await delivered(); + expect(frames.map(([value]) => value)).toEqual([5]); + }); + + it("leaves unwatched derives dirty until read; a one-off read does not make them eager", async () => { + const { telemetry, count } = await computes(); + const { host } = await loaded((h) => h.serve(Store, (_, self) => serveStore(self)), { telemetry }); + const store = await host.get(Store, {}, context); + const before = count(); + await store.set(1, context); + expect(count()).toBe(before); + expect(store.double.value).toBe(2); + expect(store.double.value).toBe(2); + expect(count()).toBe(before + 1); + await store.set(2, context); + expect(count()).toBe(before + 1); + }); + + it("diffs a watched derive's old and new value (counted), and sends no frame when they are equal", async () => { + const telemetry = await metered(); + const Parity = local("app.parity", { + version: 1, + params: {}, + members: { even: obj({ even: int() }), set: method(int()) }, + }); + const { host } = await loaded( + (h) => + h.serve(Parity, (_, self) => { + const num = self.state(Num); + return { + even: self.derive([num], ([v]) => ({ even: v.n % 2 })), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + { telemetry: telemetry.telemetry }, + ); + const parity = await host.get(Parity, {}, context); + const frames = framesOf(await parity.even.watch(context)); + await parity.set(2, context); + await parity.set(3, context); + await delivered(); + expect(frames).toEqual([[{ even: 1 }, [["s", ["even"], 1]]]]); + expect(telemetry.counts()["chord.derive.diffs"]).toBe(2); + }); +}); + +describe("failures", () => { + it("give the throwing derive's watches and those of the derives over it one failure call; they heal on a later change", async () => { + const { failing, errors } = await failingDerives(); + const [t, u, v] = await Promise.all([ + failing.t.watch(context), + failing.u.watch(context), + failing.v.watch(context), + ]); + const [ts, us] = [changesOf(t), changesOf(u)]; + const frames = framesOf(v); + await failing.set(1, context); + await delivered(); + expect(ts).toEqual([{ kind: "failed", error: THREW }]); + expect(us).toEqual([{ kind: "failed", error: THREW }]); + expect(() => failing.t.value).toThrow(THREW); + expect(() => failing.u.value).toThrow(THREW); + expect(failing.v.value).toBe(1); + await failing.set(2, context); + await delivered(); + // Open all along; the next value replaces the root. + expect(ts.at(-1)).toEqual({ kind: "value", value: 2, ops: [["r", 2]], retired: false }); + expect(us.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + expect(frames.map(([value]) => value)).toEqual([1, 2]); + // The throwing derive's owner reports it, once; the derive over it (its source failed) does not. + expect(errors).toEqual([{ kind: "derive_failed", object: "app.failing", member: "t", error: THREW }]); + }); + + it("give one failure call per episode: a recompute that throws again is quiet; a heal ends the episode", async () => { + const { failing } = await failingDerives(); + const [t, u] = [changesOf(await failing.t.watch(context)), changesOf(await failing.u.watch(context))]; + for (const n of [1, -1, -2, 3, -3]) { + await failing.set(n, context); + await delivered(); + } + const failed = { kind: "failed", error: THREW }; + expect(t).toEqual([failed, { kind: "value", value: 3, ops: [["r", 3]], retired: false }, failed]); + expect(u).toEqual([failed, { kind: "value", value: 4, ops: [["r", 4]], retired: false }, failed]); + }); + + it("open a new watch on a failed derive at its last good value, the failure its first call, then the heal", async () => { + const { failing } = await failingDerives(); + await failing.set(5, context); + // Lazy: the last good value is the last one computed. + expect(failing.u.value).toBe(6); + await failing.set(1, context); + const [t, u] = [await failing.t.watch(context), await failing.u.watch(context)]; + expect([t.value, u.value]).toEqual([5, 6]); + const [ts, us] = [changesOf(t), changesOf(u)]; + await delivered(); + await failing.set(-1, context); + await failing.set(7, context); + await delivered(); + const failed = { kind: "failed", error: THREW }; + expect(ts).toEqual([failed, { kind: "value", value: 7, ops: [["r", 7]], retired: false }]); + expect(us).toEqual([failed, { kind: "value", value: 8, ops: [["r", 8]], retired: false }]); + }); + + it("report once per failure episode, also when nothing watches (a lazy derive fails when read)", async () => { + const { failing, errors } = await failingDerives(); + await failing.set(1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(() => failing.t.value).toThrow(THREW); + await failing.set(-1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(errors).toHaveLength(1); + await failing.set(2, context); + expect(failing.t.value).toBe(2); + await failing.set(1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(errors).toHaveLength(2); + }); + + it("fold a failure and a value for a busy listener: the value replaces the failure, as a root replacement", async () => { + const { failing } = await failingDerives(); + const watch = await failing.t.watch(context); + const calls: unknown[] = []; + const gate = Promise.withResolvers(); + watch.start(async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + await failing.set(5, context); + await delivered(); + await failing.set(1, context); + await failing.set(6, context); + gate.resolve(); + await delivered(); + expect(calls).toEqual([ + { kind: "value", value: 5, ops: [["r", 5]], retired: false }, + { kind: "value", value: 6, ops: [["r", 6]], retired: false }, + ]); + }); +}); + +const THREW = new Error("t threw"); + +/** `app.failing`: `t` throws while `n` is 1 or negative, `u` is `t + 1`, `v` is `n`; the host's reports collected. */ +async function failingDerives() { + const Failing = local("app.failing", { + version: 1, + params: {}, + members: { t: int(), u: int(), v: int(), set: method(int()) }, + }); + const { host, errors } = await loaded((h) => + h.serve(Failing, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + if (v.n === 1 || v.n < 0) throw THREW; + return v.n; + }); + return { + t, + u: self.derive([t], ([x]) => x + 1), + v: self.derive([num], ([x]) => x.n), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const failing = await host.get(Failing, {}, context); + return { host, failing, errors }; +} + +describe("GC", () => { + it("a detached derive is not held by its former sources", async () => { + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { label: int() } }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Viewer, (_, self) => { + const label = self.derive([self.get(Store, {}).n], ([n]) => n); + refs.push(new WeakRef(label)); + return { label }; + }); + }); + const store = await host.get(Store, {}, context); + (await host.get(Viewer, { id: 1 }, context)).release(); + await store.set(1, context); + await expectCollected(...refs); + }); + + it("a watch open on a failed derive keeps nothing more: it ends with its reference, and a retained one holds no source", async () => { + const Paged = local("app.paged", { + version: 1, + params: { id: int() }, + members: { t: int(), set: method(int()) }, + }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Paged, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + if (v.n === 1) throw THREW; + return v.n; + }); + refs.push(new WeakRef(t), new WeakRef(num)); + return { + t, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const paged = await host.get(Paged, { id: 1 }, context); + const watch = await paged.t.watch(context); + const calls = changesOf(watch); + await paged.set(1, context); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error: THREW }]); + paged.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + // `watch` stays referenced here. + await expectCollected(...refs); + void watch; + await host.close(context); + }); +}); + +describe("GC of handed-out errors", () => { + it("a failure call's fresh error kept by its listener keeps nothing of its host once the host closes", async () => { + const Fresh = local("app.fresh", { version: 1, params: {}, members: { t: int(), set: method(int()) } }); + const refs: WeakRef[] = []; + const kept: unknown[] = []; + await (async () => { + const { host } = await loaded((h) => + h.serve(Fresh, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + // A fresh error: its stack's frames hold the derive, the document and the attempt. + if (v.n === 1) throw new Error("fresh"); + return v.n; + }); + refs.push(new WeakRef(t), new WeakRef(num), new WeakRef(self)); + return { + t, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + refs.push(new WeakRef(host)); + const fresh = await host.get(Fresh, {}, context); + (await fresh.t.watch(context)).start(async (call) => { + if (call.kind === "failed") kept.push(call.error); + }); + await fresh.set(1, context); + await delivered(); + await host.close(context); + })(); + expect(kept).toHaveLength(1); + await expectCollected(...refs); + }); +}); + +describe("mapShared", () => { + it("maps an object element once while it stays the same object; primitives every time", () => { + const f = vi.fn((x: { n: number } | number) => (typeof x === "number" ? x : x.n)); + const map = mapShared(f); + const a = { n: 1 }; + const b = { n: 2 }; + expect(map([a, b, 3])).toEqual([1, 2, 3]); + expect(map([a, { n: 2 }, 3])).toEqual([1, 2, 3]); + expect(f.mock.calls.map(([x]) => x)).toEqual([a, b, 3, { n: 2 }, 3]); + }); + + it("keeps no element alive in its cache", async () => { + const map = mapShared((note: { text: string }) => ({ text: note.text })); + const refs = (() => { + const note = { text: "t" }; + const [result] = map([note]); + return [new WeakRef(note), new WeakRef(result!)]; + })(); + await expectCollected(...refs); + expect(map([{ text: "u" }])).toEqual([{ text: "u" }]); + }); + + it("returns the previous result object for an unchanged element", () => { + const strip = mapShared((note: { text: string; at: number }) => ({ text: note.text })); + const note = { text: "t", at: 1 }; + expect(strip([note])[0]).toBe(strip([note, { text: "u", at: 2 }])[0]); + }); +}); + +const ListV1 = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); + +describe("the retired flag", () => { + it("a derive over a retired source publishes retired: true although its value and ops are unchanged (rule 21)", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + let many: Watchable | undefined; + let twice: Watchable | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + many = self.derive([list], ([value]) => value.items.length > 1); + twice = self.derive([many], ([value]) => !value); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const manys = changesOf(await many!.watch(context)); + const twices = changesOf(await twice!.watch(context)); + await list!.retire(context); + await delivered(); + expect(manys).toEqual([{ kind: "value", value: false, ops: [], retired: true }]); + expect(twices).toEqual([{ kind: "value", value: true, ops: [], retired: true }]); + await list!.change((draft) => void draft.items.push("a", "b"), context); + await delivered(); + expect(manys.at(-1)).toEqual({ kind: "value", value: true, ops: [["r", true]], retired: false }); + }); + + it("a retirement while the derive is failed travels with its next good value", async () => { + const Gate = defineState("gate", { schema: obj({ on: int() }), version: 1, initial: () => ({ on: 0 }) }); + let num: SingletonHandle<{ n: number }> | undefined; + let gate: SingletonHandle<{ on: number }> | undefined; + let n: Watchable | undefined; + await restartable(objects, (_, self) => { + num = self.state(Num); + gate = self.state(Gate); + n = self.derive([num, gate], ([v, g]) => { + if (g.on === 1 && v.n === 0) throw THREW; + return v.n; + }); + }); + await num!.change((draft) => { + draft.n = 1; + }, context); + await gate!.change((draft) => { + draft.on = 1; + }, context); + const calls = changesOf(await n!.watch(context)); + // The retirement makes `n` 0 again, which throws. + await num!.retire(context); + await num!.change((draft) => { + draft.n = 2; + }, context); + await delivered(); + expect(calls).toEqual([ + { kind: "failed", error: THREW }, + { kind: "value", value: 2, ops: [["r", 2]], retired: true }, + ]); + }); + + it("a retirement visits each derive of a diamond once, not once per path (40 derives)", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + let bottom: Watchable | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + let previous: Watchable = self.derive([list], ([value]) => value.items.length); + // 13 layers of two derives joined by a third: 2^13 paths from the top to the bottom. + for (let layer = 0; layer < 13; layer++) { + const left = self.derive([previous], ([n]) => n + 1); + const right = self.derive([previous], ([n]) => n + 2); + previous = self.derive([left, right], ([a, b]) => a + b); + } + bottom = previous; + }); + await list!.change((draft) => void draft.items.push("a"), context); + const frames = changesOf(await bottom!.watch(context)); + const visits = vi.spyOn(Derive.prototype, "sourceRetired"); + try { + await list!.retire(context); + await delivered(); + expect(visits.mock.calls.length).toBeLessThan(100); + } finally { + visits.mockRestore(); + } + expect(frames.at(-1)).toMatchObject({ retired: true }); + }); +}); diff --git a/packages/chord/test/next/state/native-source.test.ts b/packages/chord/test/next/state/native-source.test.ts new file mode 100644 index 000000000..cb31e8770 --- /dev/null +++ b/packages/chord/test/next/state/native-source.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { NativeWatchCall, Watchable } from "../../../src/watch.ts"; +import { delivered, failLive, flushed, instancesOf, loaded, rejection } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Count = defineRegistry("app.count", { version: 1 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Count` (a sum that throws when negative) and `Setter`, which contributes a writable number. */ +function serveCount(h: Parameters[0]>[0]): void { + h.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + h.serve(Setter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +describe("native sources as derive sources", () => { + it("a derive reads JSON and native sources alike, marked when a native one changes", async () => { + const Mixed = local("app.mixed", { version: 1, params: {}, members: { label: str(), name: method(str()) } }); + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Mixed, (_, self) => { + const name = self.writable("a"); + return { + label: self.derive([name, self.registry(Count)], ([n, count]) => `${n}:${count}`), + name: async (text: string) => { + name.set(text); + return null; + }, + }; + }); + }); + const mixed = await host.get(Mixed, {}, context); + await (await host.get(Setter, {}, context)).set(2, context); + expect(mixed.label.value).toBe("a:2"); + await mixed.name("b", context); + expect(mixed.label.value).toBe("b:2"); + await host.close(context); + }); + + it("`ref.state` is a derive source: a state member can show a dependency's reachability", async () => { + const Flaky = local("app.flaky", { version: 1, params: { id: int() }, members: {} }); + const Watcher = local("app.watcher", { version: 1, params: {}, members: { flaky: str() } }); + let flakySelf: Self | undefined; + const { host } = await loaded((h) => { + h.serve(Flaky, (_, self) => { + flakySelf = self; + return {}; + }); + h.serve(Watcher, (_, self) => { + const flaky = self.get(Flaky, { id: 1 }); + return { flaky: self.derive([flaky.state], ([state]) => state.kind) }; + }); + }); + const watcher = await host.get(Watcher, {}, context); + const kinds: string[] = []; + (await watcher.flaky.watch(context)).start( + async (call) => void (call.kind === "value" && kinds.push(call.value)), + ); + expect(watcher.flaky.value).toBe("live"); + failLive(objects, flakySelf!, new ChordError({ code: "unavailable", message: "down" })); + expect(watcher.flaky.value).toBe("failed"); + await delivered(); + expect(kinds).toEqual(["failed"]); + await host.close(context); + }); + + it("types a derive's sources: watchables of either kind, never a native's handle (Fable)", () => { + const declare = (self: Self): void => { + const handle = self.open( + async () => 1, + () => {}, + ); + // @ts-expect-error a native handle has a `value` but is no watchable + self.derive([handle], ([n]) => n); + expectTypeOf(self.derive([self.writable(1), constant("a")], ([n, s]) => s.length + n)).toEqualTypeOf< + Watchable + >(); + }; + expect(declare).toBeTypeOf("function"); + }); + + it("refuses a native source as a state member (Q14)", async () => { + const Bad = local("app.bad", { version: 1, params: {}, members: { n: int() } }); + const { host } = await loaded((h) => h.serve(Bad, (_, self) => ({ n: self.writable(1) as never }))); + expect(await rejection(host.get(Bad, {}, context))).toMatchObject({ + message: "app.bad: n: a state member is a JSON watchable", + }); + await host.close(context); + }); +}); + +describe("watching a native source that fails", () => { + it("a watch taken while it is failed opens at its last good value and gets the failure first", async () => { + const { host } = await loaded(serveCount); + const count = await host.registry(Count, context); + await (await host.get(Setter, {}, context)).set(-1, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + expect(watch.value).toBe(1); + watch.start(async (call) => void calls.push(call)); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error: new Error("negative: -1") }]); + await host.close(context); + }); + + it("an `on` routine started while its source is failed waits for its next good value, never calls with the stale one", async () => { + const Follower = local("app.follower", { version: 1, params: { id: int() }, members: {} }); + const seen: number[] = []; + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Follower, (_, self) => { + self.on(self.registry(Count), ({ value }) => void seen.push(value)); + return {}; + }); + }); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // Read: the failure is composed before the follower starts. + void (await host.registry(Count, context)).value; + const follower = await host.get(Follower, { id: 1 }, context); + await flushed(); + expect(seen).toEqual([]); + await setters.set(4, context); + await delivered(); + expect(seen).toEqual([4]); + follower.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/state/watchable.test.ts b/packages/chord/test/next/state/watchable.test.ts new file mode 100644 index 000000000..35184f31f --- /dev/null +++ b/packages/chord/test/next/state/watchable.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Source, Subscriber } from "../../../src/next/state/internal.ts"; +import { delivered, expectCollected, framesOf, loaded } from "../support.ts"; + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("app.counter", { + version: 1, + params: {}, + members: { num: obj({ n: int() }), set: method(int()) }, +}); + +/** A counter exposing its document, with background errors collected. */ +async function counter() { + const { host, errors } = await loaded((h) => + h.serve(Counter, (_, self) => { + const num = self.state(Num); + return { + num, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + return { counter: await host.get(Counter, {}, context), errors }; +} + +describe("watches", () => { + it("start from the value at acquisition; a listener that keeps up gets each commit", async () => { + const { counter: c } = await counter(); + const watch = await c.num.watch(context); + expect(watch.value).toEqual({ n: 0 }); + const frames = framesOf(watch); + await c.set(1, context); + await delivered(); + await c.set(2, context); + await delivered(); + expect(frames).toEqual([ + [{ n: 1 }, [["s", ["n"], 1]]], + [{ n: 2 }, [["s", ["n"], 2]]], + ]); + }); + + it("give a busy listener one call with the newest value and the folded ops", async () => { + const { counter: c } = await counter(); + const watch = await c.num.watch(context); + const frames: [unknown, unknown][] = []; + let release: () => void = () => {}; + watch.start(async (call) => { + if (call.kind !== "value") return; + frames.push([call.value, call.ops]); + // The first call stays busy until the test lets it return. + if (frames.length === 1) + await new Promise((resolve) => { + release = resolve; + }); + }); + await c.set(1, context); + await delivered(); + await c.set(2, context); + await c.set(3, context); + release(); + await delivered(); + expect(frames).toEqual([ + [{ n: 1 }, [["s", ["n"], 1]]], + // Two commits while busy: one call, the second `s` on the same path replaced the first. + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("end cancelled when their context aborts", async () => { + const { counter: c } = await counter(); + const { context: cancellable, cancel } = withCancel(context); + const watch = await c.num.watch(cancellable); + cancel(); + expect(await watch.closed).toEqual({ kind: "aborted" }); + }); + + it("end with listener_error when the listener throws, reported to onError", async () => { + const { counter: c, errors } = await counter(); + const watch = await c.num.watch(context); + const error = new Error("listener threw"); + watch.start(async () => { + throw error; + }); + await c.set(1, context); + expect(await watch.closed).toEqual({ kind: "listener_error", error }); + expect(errors).toEqual([{ kind: "listener_error", error }]); + }); +}); + +describe("constant", () => { + it("never changes; its watches end only by stop or cancellation and hold nothing after", async () => { + const start = constant({ id: "S1" }); + expect(start.value).toEqual({ id: "S1" }); + const refs = await (async () => { + const watch = await start.watch(context); + expect(watch.value).toBe(start.value); + expect(await watch.stop()).toEqual({ kind: "stopped" }); + return [new WeakRef(watch)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("subscribers", () => { + it("get each batch's frame synchronously, inside the commit, and count as watching", async () => { + const { counter: c } = await counter(); + const source = resolveWatchable(c.num) as Source; + const frames: unknown[] = []; + const subscriber: Subscriber = { + advance: (call) => void (call.kind === "value" && frames.push([call.value, call.ops])), + end: () => {}, + endAfterDelivery: () => {}, + }; + source.subscribe(subscriber); + expect(source.watched).toBe(true); + const setting = c.set(1, context); + await setting; + expect(frames).toEqual([[{ n: 1 }, [["s", ["n"], 1]]]]); + source.unsubscribe(subscriber); + expect(source.watched).toBe(false); + await c.set(2, context); + expect(frames).toHaveLength(1); + }); +}); diff --git a/packages/chord/test/next/state/writable.test.ts b/packages/chord/test/next/state/writable.test.ts new file mode 100644 index 000000000..30b27309f --- /dev/null +++ b/packages/chord/test/next/state/writable.test.ts @@ -0,0 +1,210 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; +import type { WritableSource } from "../../../src/next/state/internal.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { delivered, expectCollected, instancesOf, loaded, metered, rejection, restartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Counter = local("app.counter", { version: 1, params: {}, members: { doubled: int(), set: method(int()) } }); + +/** Serves `Counter` with a writable number, its double as a state member; the writable is handed out to the test. */ +async function counter(options: Parameters[1] = {}) { + let writable: Writable | undefined; + let self: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Counter, (_, made) => { + self = made; + const value = made.writable(1); + writable = value; + return { + doubled: made.derive([value], ([n]) => n * 2), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, options); + const reference = await host.get(Counter, {}, context); + return { host, errors, reference, writable: writable!, self: self! }; +} + +describe("writable", () => { + it("a new value (by identity) is a new frame; the same value notifies nobody and costs one comparison", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]; + writable.set(1); + await delivered(); + expect(calls).toEqual([]); + expect(counts()["chord.derive.computes"]).toBe(computes); + writable.set(2); + writable.set(3); + await delivered(); + expect(calls).toEqual([ + { kind: "value", value: 2 }, + { kind: "value", value: 3 }, + ]); + // A JSON watch folds what came while it waited: the derive's last value. + expect(doubled).toEqual([6]); + expect(writable.value).toBe(3); + await host.close(context); + }); + + it("inside a batch, sets mark it there: one frame with the last value, the derives over it computed once", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]!; + (host as unknown as ChordHost).batch.commit(() => { + writable.set(5); + writable.set(6); + }, context); + await delivered(); + expect(calls).toEqual([{ kind: "value", value: 6 }]); + expect(doubled).toEqual([12]); + expect(counts()["chord.derive.computes"]).toBe(computes + 1); + await host.close(context); + }); + + it("refuses a set inside a derive compute or a compose (`invalid/reentrant`)", async () => { + const Echo = defineRegistry("app.echo", { version: 1 }); + let inner: Writable | undefined; + const Bad = local("app.bad", { version: 1, params: {}, members: { bad: int() } }); + const Good = local("app.good", { version: 1, params: {}, members: {} }); + const { host, errors } = await loaded((h) => { + h.serve(Echo, { + compose: (table) => { + inner?.set(table.length + 100); + return table.length; + }, + }); + h.serve(Good, (_, self) => { + inner = self.writable(0); + self.contribute(Echo, inner); + return {}; + }); + h.serve(Bad, (_, self) => { + const own = self.writable(0); + return { + bad: self.derive([own], ([n]) => { + own.set(n + 1); + return n; + }), + }; + }); + }); + expect((await host.registry(Echo, context)).value).toBe(0); + expect(errors.find((error) => error.kind === "registry_failed")).toMatchObject({ + error: { code: "invalid", reason: "reentrant" }, + }); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + object: "app.bad", + failure: { kind: "broken", error: { code: "invalid", reason: "reentrant" } }, + }); + await host.close(context); + }); + + it("is the attempt's: `set` after its end throws `released`, its watches end `ended`, its value stays readable", async () => { + let writable: Writable | undefined; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) writable = self.writable(1); + }); + const watch = await writable!.watch(context); + await restart(); + expect(await watch.closed).toEqual({ kind: "ended" } satisfies WatchEnd); + expect(() => writable!.set(2)).toThrow("app.restartable(1): ended"); + expect(writable!.value).toBe(1); + await host.close(context); + }); + + it("is a contribution, a derive source and an `on` source", async () => { + const Values = defineRegistry("app.values", { version: 1 }); + const Source = local("app.source", { version: 1, params: {}, members: { set: method(int()), seen: int() } }); + const seen: number[] = []; + const { host } = await loaded((h) => { + h.serve(Values, { compose: (table) => table.map(({ value }) => value) }); + h.serve(Source, (_, self) => { + const value = self.writable(1); + self.contribute(Values, value); + self.on(value, ({ value: n }) => void seen.push(n)); + return { + seen: self.derive([value], ([n]) => n), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }); + const source = await host.get(Source, {}, context); + const values = await host.registry(Values, context); + await source.set(2, context); + await delivered(); + expect(values.value).toEqual([2]); + expect(source.seen.value).toBe(2); + expect(seen).toEqual([1, 2]); + await host.close(context); + }); + + it("is collected with its attempt", async () => { + const refs = await (async () => { + const made: WeakRef[] = []; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) made.push(new WeakRef(self.writable({ big: new Array(1000).fill(0) }))); + }); + await restart(); + await host.close(context); + return made; + })(); + await expectCollected(...refs); + }); + + it("a set of the same value after its attempt ended throws `released` too (Sol, Astra)", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(() => writable.set(1)).toThrow("app.counter: ended"); + }); + + it("a writable an app keeps after `host.close` keeps no closed host; its observers' abort listeners go (Sol)", async () => { + const signal = new AbortController(); + const { weak, writable } = await (async () => { + const { host, writable } = await counter(); + (writable as WritableSource).observe( + { advance: () => {}, end: () => {} }, + withAbortSignal(signal.signal, context), + ); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), writable }; + })(); + await expectCollected(weak); + expect(writable.value).toBe(1); + }); + + it("a late callback's set after the host closed throws `released`", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(await rejection(Promise.resolve().then(() => writable.set(9)))).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/store/allocator.test.ts b/packages/chord/test/next/store/allocator.test.ts new file mode 100644 index 000000000..518787f6d --- /dev/null +++ b/packages/chord/test/next/store/allocator.test.ts @@ -0,0 +1,24 @@ +// Tests of the id allocator (slice 6b D3, §1.2): from 2, raised by explicit ids, saturating at the safe range. + +import { describe, expect, it } from "vitest"; +import { FIRST_ID, IdAllocator } from "../../../src/next/store/allocator.ts"; + +describe("the allocator", () => { + it("mints from its value, and an explicit id raises it above itself, never lowering it", () => { + const ids = new IdAllocator(FIRST_ID); + expect([ids.mint(), ids.mint()]).toEqual([2, 3]); + ids.raise(100); + expect(ids.mint()).toBe(101); + ids.raise(50); + expect(ids.value).toBe(102); + }); + + it("saturates at Number.MAX_SAFE_INTEGER: that id may be written, every later mint is refused", () => { + const ids = new IdAllocator(Number.MAX_SAFE_INTEGER - 1); + expect(ids.mint()).toBe(Number.MAX_SAFE_INTEGER - 1); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + ids.raise(Number.MAX_SAFE_INTEGER); + expect(ids.value).toBe(Number.MAX_SAFE_INTEGER); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + }); +}); diff --git a/packages/chord/test/next/store/applier.test.ts b/packages/chord/test/next/store/applier.test.ts new file mode 100644 index 000000000..5cad8bfe8 --- /dev/null +++ b/packages/chord/test/next/store/applier.test.ts @@ -0,0 +1,116 @@ +// Tests of the applier (slice 6a §13): recorded writes applied as recorded, checked against the store's own records: +// table names and columns by the name rules, indexes, rows against their recorded columns (NULL cells, defaults kept +// as stored, booleans, json(...) cells as JSON text) and documents with their ids kept. + +import { describe, expect, it } from "vitest"; +import { IdAllocator } from "../../../src/next/store/allocator.ts"; +import { applyWrite } from "../../../src/next/store/applier.ts"; +import { Step } from "../../../src/next/store/step.ts"; +import type { StoreWrite } from "../../../src/next/store/writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { rejection } from "../support.ts"; + +const TABLE = "app.store:edits"; + +const table: StoreWrite = { + kind: "table", + table: TABLE, + version: 2, + state: "ready", + columns: { + id: { type: "str" }, + done: { type: "bool", default: false }, + note: { type: "str", optional: true }, + meta: { type: "json", optional: true, version: 1, fingerprint: "f" }, + }, + key: ["id"], + fingerprint: "t", +}; + +/** Applies `writes` in one transaction of a fresh database; resolves with the rows of the table. */ +async function applied(...writes: StoreWrite[]): Promise { + const db = await openNodeSqliteDatabase(":memory:"); + try { + return await db.transaction(async (executor) => { + const step = new Step(executor, 1, new IdAllocator(2), new Map()); + for (const write of writes) await applyWrite(step, write, "test"); + return executor.all(`SELECT * FROM "${TABLE}"`); + }); + } finally { + await db.close(); + } +} + +describe("applied writes", () => { + it("a table and its rows as recorded: NULL for absent opt cells, booleans as 0 and 1, json(...) as text", async () => { + const rows = await applied(table, { + kind: "put", + table: TABLE, + row: { id: "a", done: true, meta: { x: [1] } }, + version: 1, + columns: { meta: 1 }, + }); + expect(rows).toEqual([{ id: "a", done: 1, note: null, meta: '{"x":[1]}', meta$v: 1, $v: 1 }]); + }); + + it.each<[string, StoreWrite[], string]>([ + [ + "a put for a table not recorded", + [{ kind: "put", table: TABLE, row: {}, version: 1, columns: {} }], + "is not recorded", + ], + [ + "a key over an opt column", + [{ ...table, key: ["note"] } as StoreWrite], + '"note" is not a non-opt str or int column', + ], + [ + "a json column without a fingerprint", + [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as StoreWrite], + "a json(...) column has a version and a fingerprint", + ], + [ + "a default of another type", + [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as StoreWrite], + "is not a int value", + ], + [ + "an index over an unknown column", + [table, { kind: "index", table: TABLE, name: "by_x", columns: ["x"] }], + '"x" is not a non-opt str or int column', + ], + [ + "an index twice", + [ + table, + { kind: "index", table: TABLE, name: "by_id", columns: ["id"] }, + { kind: "index", table: TABLE, name: "by_id", columns: ["id"] }, + ], + "not a new index name", + ], + [ + "a version for a cell the row lacks", + [table, { kind: "put", table: TABLE, row: { id: "a", done: false }, version: 1, columns: { meta: 1 } }], + "no json(...) cell of the row", + ], + [ + "a json cell that is no JSON", + [ + table, + { + kind: "put", + table: TABLE, + row: { id: "a", done: false, meta: 1n as never }, + version: 1, + columns: { meta: 1 }, + }, + ], + "not JSON", + ], + ])("refuses %s", async (_, writes, message) => { + expect(await rejection(applied(...writes))).toMatchObject({ + code: "invalid", + message: expect.stringContaining(message), + }); + }); +}); diff --git a/packages/chord/test/next/store/apply-batch.test.ts b/packages/chord/test/next/store/apply-batch.test.ts new file mode 100644 index 000000000..762b4f4d6 --- /dev/null +++ b/packages/chord/test/next/store/apply-batch.test.ts @@ -0,0 +1,290 @@ +// Tests of batch apply (slice 6b §1.6, S5, S17): a store's outbox replayed into an empty store gives the same archive, +// manifest included; every write kind (an adapter's create-plus-retire and change-plus-retire too); the refusals (claims held, seqs that do not follow, writes the records refuse) +// and the divergence of a recorded count. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { applyBatch } from "../../../src/next/store/apply-batch.ts"; +import { + changeDocument, + createDocument, + type DocumentRecord, + type DocumentWrite, + insertDocument, + readDocument, + retireDocument, + retireRecord, + writeContent, +} from "../../../src/next/store/engine.ts"; +import { add, type ColumnRef, lt, prefix } from "../../../src/next/store/expression.ts"; +import type { OutboxCommit } from "../../../src/next/store/outbox.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { archiveStore, exported, memoryStore, rejection, tableIn } from "../support.ts"; + +const OPEN = { admit: () => true, adopt: () => {} }; + +/** The column references a table's callbacks get (the changes are untyped below the transaction). */ +type Refs = { readonly [column: string]: ColumnRef | undefined }; + +const CountsV1 = defineTable("counts", { + version: 1, + columns: { id: str(), n: int(), tag: str() }, + key: ["id"], + indexes: { by_tag: ["tag"] }, +}); + +const CountsV2 = evolveTable(CountsV1, (t) => t.add("extra", int({ default: 0 })).dropIndex("by_tag")); + +/** A store with an outbox from its first commit holding every write kind: an archive store, then counted writes. */ +async function everyKind(): Promise { + const store = await archiveStore({ outbox: true }); + const counts = await tableIn(store, CountsV1); + const staged: StagedWrite[] = []; + const changes = new TableChanges(counts, staged); + for (let i = 0; i < 10; i++) changes.put({ id: `c${i}`, n: i, tag: i % 2 === 0 ? "even" : "odd" }); + await store.commit(staged, OPEN, context); + const bulk: StagedWrite[] = []; + const more = new TableChanges(counts, bulk); + more.update({ id: "c1" }, (c: Refs) => ({ n: add(c.n!, 10) })); + more.delete({ id: "c2" }); + more.updateWhere((c: Refs) => prefix(c.tag!, "ev"), { tag: "EVEN" }); + more.deleteWhere((c: Refs) => lt(c.n!, 1)); + await store.commit(bulk, OPEN, context); + await prepareTables([{ definition: CountsV2, table: "app.store:counts" }], store, OPEN, "app.store", context); + const created: number[] = []; + const fence = { admit: () => true, adopt: (_: number, ids: readonly number[]) => void created.push(...ids) }; + const address = { owner: "app.store", namespace: "chord", kind: "replayed", key: "x" }; + const base = { version: 1, fingerprint: "f1", value: { n: 0 } }; + await store.commit([createDocument(address, base, '{"n":0}')], fence, context); + const ops = [["s", ["n"], 1]] as const; + await store.commit( + [changeDocument(created[0]!, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + OPEN, + context, + ); + await store.commit([retireDocument(created[0]!, 2)], OPEN, context); + await durableShaped(store); + return store; +} + +/** + * What only an adapter writes (Durable's commits): a rewindable creation retired in its own commit (an empty + * lifetime), and a change with the retirement of the same record in one commit. + */ +async function durableShaped(store: ChordStore): Promise { + const address = { owner: "app.durable", namespace: "session", kind: "notes" }; + const base = { version: 1, fingerprint: "", value: { n: 0 } }; + /** A record created at `seq`, retired then too when `retired`. */ + const made = (id: number, key: string, seq: number, retired: boolean): DocumentRecord => ({ + id, + ...address, + key, + history: "rewindable", + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: "{}", + revision: 1, + version: 1, + fingerprint: "", + }); + const create = (key: string, retired: boolean): StagedWrite => ({ + order: "write", + apply: async (step) => { + const record = made(step.ids.mint(), key, step.seq, retired); + await insertDocument(step, record, '{"n":0}'); + return [{ kind: "document.create", record, base }]; + }, + }); + await store.commit([create("empty", true), create("kept", false)], OPEN, context); + const kept = (await store.read( + (executor) => executor.get<{ id: number }>("SELECT id FROM chord_documents WHERE key = 'kept'"), + context, + ))!.id; + const ops = [["s", ["n"], 1]] as const; + const changeAndRetire: StagedWrite = { + order: "write", + apply: async (step) => { + const record = (await readDocument(step, kept))!; + const content = { kind: "delta", version: 1, ops } as const; + await writeContent(step, kept, "rewindable", 1, content, JSON.stringify(ops)); + await retireRecord(step, record); + const writes: DocumentWrite[] = [ + { kind: "document.change", id: kept, revision: 1, content }, + { kind: "document.retire", id: kept, revision: 2 }, + ]; + return writes; + }, + }; + await store.commit([changeAndRetire], OPEN, context); +} + +/** Every commit of `store`'s outbox. */ +async function commits(store: ChordStore): Promise { + return store.outbox!.read(store.outbox!.floor, 10_000, context); +} + +describe("replay", () => { + it("a store's outbox replayed into an empty store gives the same archive, manifest included", async () => { + const source = await everyKind(); + const all = await commits(source); + const kinds = new Set(all.flatMap(({ writes }) => writes.map(({ kind }) => kind))); + expect([...kinds].sort()).toEqual( + [ + "alter", + "document.change", + "document.create", + "document.retire", + "dropIndex", + "index", + "put", + "table", + "update", + "updateWhere", + "delete", + "deleteWhere", + ].sort(), + ); + const replica = await memoryStore(); + await applyBatch(replica, all.slice(0, 3), context); + await applyBatch(replica, all.slice(3), context); + expect(await exported(replica)).toEqual(await exported(source)); + expect([replica.lastSeq, replica.nextId]).toEqual([source.lastSeq, source.nextId]); + }); + + it("restores the allocator: a replica mints above every id it holds", async () => { + const source = await everyKind(); + const replica = await memoryStore(); + await applyBatch(replica, await commits(source), context); + expect(replica.mintId()).toBe(source.nextId); + }); + + it("writes the applied commits to the replica's own outbox when it has one", async () => { + const source = await everyKind(); + const all = await commits(source); + const replica = await memoryStore({ outbox: true }); + await applyBatch(replica, all, context); + expect(await commits(replica)).toEqual(all); + }); +}); + +describe("refusals", () => { + it("while an owner holds a claim on the store (its memory would go stale)", async () => { + const replica = await memoryStore(); + replica.claims.documents("app.store", "chord", { label: "an object" }); + expect(await rejection(applyBatch(replica, [], context))).toMatchObject({ code: "invalid", reason: "args" }); + }); + + it("a claim taken between the call and its turn in the queue refuses the batch; a claim is refused while it applies", async () => { + const replica = await memoryStore(); + const held = await replica.take(() => {}); + const replaying = applyBatch(replica, [], context); + const owner = { label: "an object" }; + replica.claims.documents("app.store", "chord", owner); + held(); + expect(await rejection(replaying)).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + replica.claims.release(owner); + const all = await commits(await everyKind()); + let applied = false; + const applying = applyBatch(replica, all, context).then(() => { + applied = true; + }); + // A claim made and released in one turn holds nothing; once the batch applies (across awaits), one is refused. + let refused: unknown; + while (!applied && refused === undefined) { + await Promise.resolve(); + try { + replica.claims.documents("app.store", "chord", owner); + replica.claims.release(owner); + } catch (error) { + refused = error; + } + } + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await applying; + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + replica.claims.documents("app.store", "chord", owner); + }); + + it("a batch queued behind one that diverges is refused at its turn", async () => { + const all = await commits(await everyKind()); + const diverging = structuredClone(all); + for (const commit of diverging) { + for (const write of commit.writes) if (write.kind === "updateWhere") (write as { count: number }).count += 1; + } + const replica = await memoryStore(); + const first = rejection(applyBatch(replica, diverging, context)); + const second = rejection(applyBatch(replica, all, context)); + expect(String(await first)).toContain("recorded"); + expect(await second).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); + + it("a commit whose next id is not above a document it creates (a saturated id aside)", async () => { + const replica = await memoryStore(); + const record = { + id: 100, + owner: "app.store", + namespace: "chord", + kind: "x", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const base = { version: 1, fingerprint: "", value: {} }; + const create = { kind: "document.create", record, base } as const; + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 2, writes: [create] }], context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 100, writes: [create] }], context))).toMatchObject({ + reason: "args", + }); + await applyBatch(replica, [{ seq: 1, nextId: 101, writes: [create] }], context); + const saturated = { ...create, record: { ...record, id: Number.MAX_SAFE_INTEGER, key: "max" } }; + await applyBatch(replica, [{ seq: 2, nextId: Number.MAX_SAFE_INTEGER, writes: [saturated] }], context); + expect(replica.lastSeq).toBe(2); + }); + + it("a first commit that does not follow lastSeq, or one that does not follow the one before; nothing applied", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + expect(await rejection(applyBatch(replica, all.slice(1), context))).toMatchObject({ reason: "args" }); + expect(await rejection(applyBatch(replica, [all[0]!, all[2]!], context))).toMatchObject({ reason: "args" }); + expect(replica.lastSeq).toBe(0); + await applyBatch(replica, all, context); + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + }); + + it("a write the records refuse rolls the batch back, and the store goes on", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + const broken = structuredClone(all); + (broken[0]!.writes[0] as { table: string }).table = "no:such:table"; + expect(await rejection(applyBatch(replica, broken, context))).toMatchObject({ code: "invalid", reason: "args" }); + const forged = structuredClone(all); + (forged[0]!.writes as unknown[]).push({ kind: "put", table: "x", row: {}, version: 0 }); + expect(await rejection(applyBatch(replica, forged, context))).toMatchObject({ code: "invalid", reason: "args" }); + await applyBatch(replica, all, context); + }); + + it("a recorded count the store does not reproduce stops applying and marks the store diverged (S17)", async () => { + const all = await commits(await everyKind()); + const index = all.findIndex(({ writes }) => writes.some(({ kind }) => kind === "updateWhere")); + const diverging = structuredClone(all); + const write = diverging[index]!.writes.find(({ kind }) => kind === "updateWhere") as { count: number }; + write.count += 1; + const replica = await memoryStore(); + expect(String(await rejection(applyBatch(replica, diverging, context)))).toContain("recorded"); + expect(await rejection(applyBatch(replica, all, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); +}); diff --git a/packages/chord/test/next/store/archive.test.ts b/packages/chord/test/next/store/archive.test.ts new file mode 100644 index 000000000..3df415558 --- /dev/null +++ b/packages/chord/test/next/store/archive.test.ts @@ -0,0 +1,42 @@ +// Tests of the archive part schema (slice 6a §13, rule 25): every part an export yields passes; a part with an +// unknown kind, a missing or extra field or a value of another type is refused, naming its position. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { checkPart } from "../../../src/next/store/archive.ts"; +import { archiveStore, exported } from "../support.ts"; + +describe("the archive part schema", () => { + it("takes every part an export yields", async () => { + const store = await archiveStore(); + const parts = await exported(store); + for (const [position, part] of parts.entries()) expect(checkPart(part, position, context)).toBe(part); + await store.close(context); + }); + + it.each<[string, unknown]>([ + ["an unknown kind", { kind: "blob" }], + ["a missing field", { kind: "end" }], + ["an extra field", { kind: "end", writes: 0, more: 1 }], + ["a negative count", { kind: "end", writes: -1 }], + [ + "a column of an unknown type", + { + kind: "table", + table: "a:b:c", + version: 1, + state: "ready", + columns: { a: { type: "date" } }, + key: ["a"], + fingerprint: "f", + }, + ], + ["a put at version 0", { kind: "put", table: "a:b:c", row: {}, version: 0, columns: {} }], + [ + "a record without an id", + { kind: "document.create", record: { namespace: "n" }, base: { version: 1, fingerprint: "f", value: {} } }, + ], + ])("refuses %s", (_, part) => { + expect(() => checkPart(part, 4, context)).toThrow("import: part 4: not an archive part"); + }); +}); diff --git a/packages/chord/test/next/store/catalog.test.ts b/packages/chord/test/next/store/catalog.test.ts new file mode 100644 index 000000000..11761b30a --- /dev/null +++ b/packages/chord/test/next/store/catalog.test.ts @@ -0,0 +1,63 @@ +// The catalog (slice 6a §8): the store's records of its tables, read once at open, changed after COMMIT returned. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { stageTable } from "../../../src/next/store/table-writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { memoryStore, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), at: int() }, + key: ["id"], + indexes: { by_at: ["at"] }, +}); +const TABLE = "app.store:edits"; + +describe("the catalog", () => { + it("is read once at open: a reopened store knows its tables", async () => { + const directory = await mkdtemp(join(tmpdir(), "chord-catalog-")); + const path = join(directory, "store.sqlite"); + const first = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + await tableIn(first, Edits); + await first.close(context); + const second = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(second.table(TABLE)).toEqual({ + table: TABLE, + version: 1, + state: "ready", + columns: Edits.record, + key: ["id"], + fingerprint: Edits.fingerprint, + indexes: { by_at: ["at"] }, + }); + await second.close(context); + await rm(directory, { recursive: true, force: true }); + }); + + it("changes in memory only after COMMIT returned", async () => { + const store = await memoryStore(); + const { version, record: columns, key, fingerprint } = Edits; + const failing: StagedWrite = { + order: "write", + apply: (step) => + step + .run("INSERT INTO no_such_table VALUES (1)") + .then(() => [{ kind: "dropIndex", table: TABLE, name: "x" }]), + }; + const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); + await expect(store.commit([table, failing], { admit: () => true, adopt: () => {} }, context)).rejects.toThrow( + "no such table", + ); + expect(store.table(TABLE)).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/store/claims.test.ts b/packages/chord/test/next/store/claims.test.ts new file mode 100644 index 000000000..3974487e3 --- /dev/null +++ b/packages/chord/test/next/store/claims.test.ts @@ -0,0 +1,72 @@ +// Tests of claims (slice 6b "owner names and claims", "claims per table and per document namespace"): a table by its +// SQL name and documents by owner and namespace, once per store; released by their claimant's end; none taken while a +// batch applies. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { Claims } from "../../../src/next/store/claims.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore } from "../support.ts"; + +describe("claims", () => { + it("refuse a second claimant of a table or of an owner's namespace; one claimant may claim again", () => { + const claims = new Claims(); + const a = { label: "A" }; + const b = { label: "B" }; + claims.table("x:edits", a); + claims.table("x:edits", a); + claims.documents("x", "chord", a); + expect(() => claims.table("x:edits", b)).toThrow("claimed by A"); + expect(() => claims.documents("x", "chord", b)).toThrow("claimed by A"); + // Another namespace of the same owner, another table: no collision. + claims.documents("x", "session", b); + claims.table("x:entries", b); + expect(claims.held).toBe(true); + claims.release(a); + claims.table("x:edits", b); + claims.release(b); + expect(claims.held).toBe(false); + }); + + it("are refused while locked (a batch applies), taken again once unlocked", () => { + const claims = new Claims(); + const owner = { label: "A" }; + const unlock = claims.lock(); + expect(() => claims.table("x:edits", owner)).toThrow("while a batch applies"); + expect(() => claims.documents("x", "chord", owner)).toThrow("while a batch applies"); + expect(claims.held).toBe(false); + unlock(); + claims.table("x:edits", owner); + expect(claims.held).toBe(true); + }); + + it("an object claims its tables and documents when it declares them, and releases them at its end", async () => { + const store = await memoryStore(); + const Note = defineState("note", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const Edits = defineTable("edits", { version: 1, columns: { id: str() }, key: ["id"] }); + const Notes = local("notes.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded( + (h) => + h.serve(Notes, (_, self) => { + self.state(Note); + self.table(Edits); + return {}; + }), + { store }, + ); + await host.get(Notes, {}, context); + expect(() => store.claims.table("notes.store:edits", { label: "other" })).toThrow("claimed by notes.store"); + expect(() => store.claims.documents("notes.store", "chord", { label: "other" })).toThrow("claimed"); + await host.close(context); + expect(store.claims.held).toBe(false); + store.claims.table("notes.store:edits", { label: "other" }); + }); +}); diff --git a/packages/chord/test/next/store/columns.test.ts b/packages/chord/test/next/store/columns.test.ts new file mode 100644 index 000000000..48e11981c --- /dev/null +++ b/packages/chord/test/next/store/columns.test.ts @@ -0,0 +1,73 @@ +// Table columns (decisions "Flat columns", "Nullable columns and the `set` form"): descriptors and row types. + +import { describe, expect, it } from "vitest"; +import { arr, bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { + type Columns, + columnRecord, + describeColumns, + type PutRowOf, + type RowOf, +} from "../../../src/next/store/columns.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { Equal } from "../support.ts"; + +describe("column descriptors", () => { + it("describes the helpers in declaration order", () => { + const columns = describeColumns( + { + id: str(), + n: int({ default: 0 }), + x: num(), + b: bool(), + note: opt(str()), + meta: opt(json(obj({}), { version: 2 })), + }, + "t", + ); + expect(columns.map(({ name, type, optional }) => [name, type, optional])).toEqual([ + ["id", "str", false], + ["n", "int", false], + ["x", "num", false], + ["b", "bool", false], + ["note", "str", true], + ["meta", "json", true], + ]); + expect(columns.map(columnRecord)).toEqual([ + { type: "str" }, + { type: "int", default: 0 }, + { type: "num" }, + { type: "bool" }, + { type: "str", optional: true }, + { type: "json", optional: true, version: 2, fingerprint: expect.any(String) }, + ]); + }); + + it("checks names: the pattern, no two equal ignoring case, no $", () => { + expect(() => describeColumns({ "a-b": str() }, "t")).toThrow("[A-Za-z_][A-Za-z0-9_]*"); + expect(() => describeColumns({ a$v: str() }, "t")).toThrow('column "a$v"'); + expect(() => describeColumns({ path: str(), Path: str() }, "t")).toThrow("only in case"); + expect(describeColumns({ camelCase: str(), _x1: int() }, "t")).toHaveLength(2); + }); + + it("refuses other schemas, bounds, patterns, opt defaults and defaults of another type", () => { + const columns = (value: unknown) => ({ c: value }) as unknown as Columns; + expect(() => describeColumns(columns(arr(str())), "t")).toThrow("expected str()"); + expect(() => describeColumns(columns(obj({})), "t")).toThrow("expected str()"); + expect(() => describeColumns(columns(str({ max: 3 })), "t")).toThrow('"maxLength" is not supported'); + expect(() => describeColumns(columns(str({ pattern: "a" })), "t")).toThrow('"pattern" is not supported'); + expect(() => describeColumns(columns(opt(int({ default: 1 }))), "t")).toThrow("no default"); + expect(() => describeColumns(columns(int({ default: 1.5 })), "t")).toThrow("not a int value"); + expect(() => describeColumns(columns(str({ default: "a\0" })), "t")).toThrow("NUL"); + }); +}); + +describe("row types", () => { + it("types rows with optional opt(...) fields and puts with optional defaulted fields", () => { + const shape = { id: str(), hits: int({ default: 0 }), note: opt(str()) }; + type S = typeof shape; + const row: Equal, { id: string; hits: number } & { note?: string }> = true; + const put: Equal, { id: string } & { hits?: number; note?: string }> = true; + expect([row, put]).toEqual([true, true]); + }); +}); diff --git a/packages/chord/test/next/store/engine.test.ts b/packages/chord/test/next/store/engine.test.ts new file mode 100644 index 000000000..00b28eae4 --- /dev/null +++ b/packages/chord/test/next/store/engine.test.ts @@ -0,0 +1,265 @@ +// The engine (slice 6a §10, §11, slice 6b §1.5): documents as bases and deltas, their keys in code point order, the +// history rules at a new base and at retirement (R1), reads at a point, records inserted as they are. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { + changeDocument, + createDocument, + type DocumentAddress, + type DocumentHistory, + type DocumentRecord, + findDocument, + insertDocument, + listDocumentKeys, + listDocuments, + loadDocument, + readDocument, + retireDocument, + retireRecord, + writeContent, +} from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import { TABLES } from "../../../src/next/store/schema.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { exported, rejection } from "../support.ts"; + +const OWNER = { owner: "app.store", namespace: "chord", kind: "notes" }; + +/** A store on `:memory:` (or `on`) and a commit helper resolving with the created ids. */ +async function engine(on?: ChordStore) { + const store = + on ?? ((await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore); + const commit = async (...writes: StagedWrite[]): Promise => { + let created: readonly number[] = []; + await store.commit( + writes, + { + admit: () => true, + adopt: (_seq, ids) => { + created = ids; + }, + }, + context, + ); + return created; + }; + const at = (key: string): DocumentAddress => ({ ...OWNER, key }); + const base = (value: object, version = 1) => ({ + version, + fingerprint: `f${version}`.padEnd(16, "0"), + value: value as never, + }); + const create = (key: string, value: object) => createDocument(at(key), base(value), JSON.stringify(value)); + const delta = (id: number, revision: number, ops: Op[]) => + changeDocument(id, revision, { kind: "delta", version: 1, ops }, JSON.stringify(ops)); + const read = (fn: Parameters[0]) => store.read(fn, context) as Promise; + return { store, commit, at, base, create, delta, read }; +} + +/** `store`'s archive with `edit` applied to its parts, imported into a fresh `:memory:` store. */ +async function reimported(store: ChordStore, edit: (parts: ArchivePart[]) => unknown[]): Promise { + const parts = edit(await exported(store)); + return (await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context)) as ChordStore; +} + +/** A staged write applying `apply` in the step (an adapter's composite). */ +function staged(apply: StagedWrite["apply"]): StagedWrite { + return { order: "write", apply }; +} + +/** An adapter's record of `history` at `key`, with an explicit id and the step's seq. */ +function record(id: number, key: string, history: DocumentHistory, createdAt: number): DocumentRecord { + return { id, ...OWNER, key, history, createdAt, revision: 1, version: 1, fingerprint: "" }; +} + +const LIMIT = Number.MAX_SAFE_INTEGER; + +describe("documents", () => { + it("load as their base with the deltas after it applied; a new base prunes the older revisions", async () => { + const { commit, at, base, create, delta, read } = await engine(); + const [id] = await commit(create("a", { n: 0 })); + await commit(delta(id!, 1, [["s", ["n"], 1]])); + await commit(delta(id!, 2, [["s", ["n"], 2]])); + const found = (await read((e) => findDocument(e, at("a"))))!; + expect(found).toMatchObject({ id, revision: 3, version: 1, history: "none", createdAt: 1 }); + expect(await read((e) => loadDocument(e, found))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); + const next = base({ m: 9 }, 2); + await commit(changeDocument(id!, 3, { kind: "base", ...next }, JSON.stringify(next.value))); + const rebased = (await read((e) => findDocument(e, at("a"))))!; + expect(rebased).toMatchObject({ revision: 4, version: 2, fingerprint: next.fingerprint }); + expect(await read((e) => loadDocument(e, rebased))).toMatchObject({ value: { m: 9 }, deltasSinceBase: 0 }); + expect(await read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([{ seq: 4 }]); + }); + + it("list live keys in code point order, paging after a key", async () => { + const { commit, create, read } = await engine(); + await commit(create("\u{1F600}", {}), create("\uFF5E", {}), create("b", {}), create("a", {})); + expect(await read((e) => listDocumentKeys(e, OWNER, undefined, 10))).toEqual(["a", "b", "\uFF5E", "\u{1F600}"]); + expect(await read((e) => listDocumentKeys(e, OWNER, "b", 1))).toEqual(["\uFF5E"]); + }); + + it("retire a none record with its row and revisions (R1); the address takes a new incarnation", async () => { + const { commit, at, create, read } = await engine(); + const [id] = await commit(create("a", { n: 0 })); + const [next] = await commit(retireDocument(id!, 1), create("a", { n: 1 })); + expect(next).toBe(3); + expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 3 }]); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 3 }]); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 3 }); + }); + + it("refuse two live records at one address and a delta on another version: the step is refused", async () => { + const first = await engine(); + await first.commit(first.create("a", {})); + expect(String(await rejection(first.commit(first.create("a", {}))))).toContain("UNIQUE"); + const second = await engine(); + const [id] = await second.commit(second.create("a", {})); + const crossing = changeDocument(id!, 1, { kind: "delta", version: 2, ops: [] }, "[]"); + expect(String(await rejection(second.commit(crossing)))).toContain("not live at revision 1"); + expect(await second.read(async () => 1)).toBe(1); + }); +}); + +describe("history rules (R1) and reads at a point", () => { + /** Creates adapter record `id` at "a" (its base `{ n: 0 }`), then changes and retires it in later commits. */ + async function lifetime(history: DocumentHistory) { + const { store, commit, at, read } = await engine(); + await commit( + staged(async (step) => { + const made = record(10, "a", history, step.seq); + await insertDocument(step, made, '{"n":0}'); + return []; + }), + ); + await commit( + staged(async (step) => { + const content = { kind: "base", version: 1, fingerprint: "", value: { n: 1 } } as const; + await writeContent(step, 10, history, 1, content, '{"n":1}'); + return []; + }), + ); + await commit( + staged(async (step) => { + await writeContent( + step, + 10, + history, + 2, + { kind: "delta", version: 1, ops: [["s", ["n"], 2]] }, + '[["s",["n"],2]]', + ); + return []; + }), + ); + // An adapter's retirement (Durable's): the record read in the step, retired by its history's rule. + await commit( + staged(async (step) => { + await retireRecord(step, (await readDocument(step, 10))!); + return []; + }), + ); + return { store, read, at }; + } + + it("keep a retired latest record's row without revisions, and every revision of a rewindable one", async () => { + const latest = await lifetime("latest"); + expect(await latest.read((e) => e.all(`SELECT id, retired_at FROM ${TABLES.documents}`))).toEqual([ + { id: 10, retired_at: 4 }, + ]); + expect(await latest.read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([]); + const rewindable = await lifetime("rewindable"); + expect(await rewindable.read((e) => e.all(`SELECT seq, kind FROM ${TABLES.revisions} ORDER BY seq`))).toEqual([ + { seq: 1, kind: "base" }, + { seq: 2, kind: "base" }, + { seq: 3, kind: "delta" }, + ]); + }); + + it("find a record at a point by its half-open lifetime, and load a rewindable one at any point inside it", async () => { + const { read, at } = await lifetime("rewindable"); + const found = await Promise.all( + [0, 1, 3, 4, "current" as const].map((point) => + read((e) => findDocument(e, at("a"), point)), + ), + ); + expect(found.map((each) => each?.id)).toEqual([undefined, 10, 10, undefined, undefined]); + const record = found[1]!; + expect(record.retiredAt).toBe(4); + for (const [point, n] of [ + [1, 0], + [2, 1], + [3, 2], + ] as const) { + expect(await read((e) => loadDocument(e, record, point))).toMatchObject({ value: { n }, version: 1 }); + } + expect(await read((e) => listDocuments(e, OWNER, 3, 0, 10))).toMatchObject([{ id: 10 }]); + expect(await read((e) => listDocuments(e, OWNER, "current", 0, 10))).toEqual([]); + }); + + it("find a retired latest record at a point, whose content is gone", async () => { + const { read, at } = await lifetime("latest"); + const found = (await read((e) => findDocument(e, at("a"), 2)))!; + expect(found).toMatchObject({ id: 10, retiredAt: 4, history: "latest" }); + // A read returns data (the store stops on a read's throw, slice 6b §1.4): the outcome as a value. + const outcome = read((e) => + loadDocument(e, found, 2).then( + () => "loaded", + (error: unknown) => String(error), + ), + ); + expect(await outcome).toContain("no base"); + }); + + it("insert a create-plus-retire as an empty lifetime that never holds the live address", async () => { + const { commit, read, at, create } = await engine(); + const [live] = await commit(create("a", {})); + await commit( + staged(async (step) => { + await insertDocument(step, { ...record(20, "a", "latest", step.seq), retiredAt: step.seq }, "{}"); + return []; + }), + ); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: live }); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions} WHERE document_id = 20`))).toEqual( + [], + ); + }); +}); + +describe("counters at the safe-integer limit", () => { + it("allocate document ids up to the safe range after an import near it; past it the commit is refused", async () => { + const { store: source } = await engine(); + const near = (next: number) => + reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextId: next }, ...rest]); + const { store, commit, create } = await engine(await near(LIMIT - 1)); + expect(await commit(create("a", { n: 0 }))).toEqual([LIMIT - 1]); + expect(store.nextId).toBe(LIMIT); + expect(String(await rejection(commit(create("b", { n: 0 }))))).toContain("ID space is exhausted"); + // Two creations in one commit where only one id is left: nothing of it is written. + const both = await engine(await near(LIMIT - 1)); + expect(String(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 }))))).toContain( + "ID space is exhausted", + ); + expect([both.store.lastSeq, both.store.nextId]).toEqual([0, LIMIT - 1]); + }); + + it("count revisions up to the safe range after an import near it; a change past it is refused", async () => { + const { store: source, commit: seed, create } = await engine(); + await seed(create("a", { n: 0 })); + const store = await reimported(source, (parts) => + parts.map((part) => + part.kind === "document.create" ? { ...part, record: { ...part.record, revision: LIMIT - 1 } } : part, + ), + ); + const { commit, delta, read, at } = await engine(store); + const found = (await read((e) => findDocument(e, at("a"))))!; + await commit(delta(found.id, LIMIT - 1, [["s", ["n"], 1]])); + expect(String(await rejection(commit(delta(found.id, LIMIT, [["s", ["n"], 2]]))))).toContain("safe range"); + }); +}); diff --git a/packages/chord/test/next/store/evaluate.test.ts b/packages/chord/test/next/store/evaluate.test.ts new file mode 100644 index 000000000..9f94e2b94 --- /dev/null +++ b/packages/chord/test/next/store/evaluate.test.ts @@ -0,0 +1,211 @@ +// Tests of the JavaScript evaluator of the table vocabulary, and that it agrees with SQLite on the same nodes. +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { applySet, type EvaluatedRow, matches } from "../../../src/next/store/evaluate.ts"; +import { + add, + and, + coalesce, + compileSet, + compileWhere, + concat, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + max, + min, + ne, + not, + or, + prefix, + replacePrefix, + sub, + type WhereNode, + when, +} from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { ColumnRefs } from "../../../src/next/store/query.ts"; +import { createTableSql, TableSql, whereSql } from "../../../src/next/store/sql.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import type { SqliteValue } from "../../../src/sqlite/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const T = defineTable("t", { + version: 1, + columns: { + id: str(), + path: str(), + n: int(), + x: num(), + b: bool(), + note: opt(str()), + count: opt(int()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], +}); +type C = typeof T extends { "~columns"?: infer X } ? NonNullable : never; +const refs = T.refs as ColumnRefs; +const args = expect.objectContaining({ code: "invalid", reason: "args" }); +describe("the evaluator", () => { + const row: EvaluatedRow = { id: "r", path: "a/x", n: 2, x: 1, b: true, count: 3 }; + + it("follows SQL's NULL rules: a comparison with NULL never matches, not and ne included", () => { + const where = (condition: unknown) => compileWhere(T, condition, "where"); + expect(matches(where(eq(refs.note, "a")), row)).toBe(false); + expect(matches(where(ne(refs.note, "a")), row)).toBe(false); + expect(matches(where(not(eq(refs.note, "a"))), row)).toBe(false); + expect(matches(where(or(isNull(refs.note), ne(refs.note, "a"))), row)).toBe(true); + expect(matches(where(and()), row)).toBe(true); + expect(matches(where(or()), row)).toBe(false); + expect(matches(where(inArray(refs.n, [])), row)).toBe(false); + }); + + it("evaluates every right side against the row before (a swap)", () => { + const Swap = defineTable("s", { version: 1, columns: { id: str(), a: str(), b: str() }, key: ["id"] }); + const set = compileSet( + Swap, + { a: (Swap.refs as { b: unknown }).b, b: (Swap.refs as { a: unknown }).a }, + "update", + ); + expect(applySet(set, { id: "1", a: "x", b: "y" }, "s")).toEqual({ id: "1", a: "y", b: "x" }); + }); + + it("leaves a value without the prefix unchanged and refuses integers outside the safe range (rule 10)", () => { + expect(applySet(compileSet(T, { path: replacePrefix(refs.path, "b/", "c/") }, "u"), row, "t").path).toBe("a/x"); + expect(applySet(compileSet(T, { path: replacePrefix(refs.path, "a/", "") }, "u"), row, "t").path).toBe("x"); + const big = { ...row, n: Number.MAX_SAFE_INTEGER }; + expect(() => applySet(compileSet(T, { n: add(refs.n, 1) }, "u"), big, "t")).toThrow(args); + }); +}); + +describe("one semantics: the evaluator agrees with SQLite", () => { + const ROWS: EvaluatedRow[] = [ + { id: "1", path: "a/x", n: 1, x: 0.5, b: true, note: "z", count: 1 }, + { id: "2", path: "a/\u{1F600}", n: 2, x: 2, b: false }, + { id: "3", path: "a\uFFFF", n: -3, x: -1, b: true, note: "" }, + { id: "4", path: "b", n: 0, x: 0, b: false, count: 0 }, + { id: "5", path: "", n: 9, x: 9.5, b: true, note: "a/" }, + { id: "6", path: "\u{10FFFF}", n: 5, x: 1, b: false, count: -2 }, + ]; + const CONDITIONS = [ + eq(refs.note, "z"), + ne(refs.note, "z"), + not(ne(refs.count, 0)), + lt(refs.path, "a\uFFFF"), + gte(refs.path, "a/\u{1F600}"), + prefix(refs.path, "a"), + prefix(refs.path, ""), + prefix(refs.path, "\u{10FFFF}"), + inArray(refs.path, ["b", "", "x"]), + inArray(refs.b, [true]), + or(isNull(refs.count), gt(refs.x, 1)), + and(not(isNull(refs.note)), lte(refs.n, 1)), + not(or(eq(refs.b, true), lt(refs.count, 0))), + ]; + + async function table() { + const db = await openNodeSqliteDatabase(":memory:"); + await db.run(createTableSql("t", T.record, T.key)); + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + for (const row of ROWS) await db.run(sql.put, ...T.rows.put(row, 1).params); + return { db, sql }; + } + + it("selects the rows the evaluator matches", async () => { + const { db } = await table(); + for (const condition of CONDITIONS) { + const node = compileWhere(T, condition, "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text} ORDER BY id`, ...params); + const expected = ROWS.filter((row) => matches(node, row)).map(({ id }) => id); + expect([JSON.stringify(node), rows.map(({ id }) => id)]).toEqual([JSON.stringify(node), expected]); + } + await db.close(); + }); + + it("agrees on every operator's NULL truth under not, and, or and when", async () => { + const PREDICATES = [ + eq(refs.note, "z"), + ne(refs.note, "z"), + lt(refs.count, 1), + lte(refs.count, 0), + gt(refs.note, "a"), + gte(refs.count, 0), + inArray(refs.note, ["z", ""]), + inArray(refs.note, []), + prefix(refs.note, "a"), + prefix(refs.note, ""), + isNull(refs.count), + ]; + const WRAPPED = PREDICATES.flatMap((p) => [ + p, + not(p), + and(p, eq(refs.b, true)), + or(p, eq(refs.b, true)), + not(and(p, eq(refs.b, true))), + not(or(p, eq(refs.b, false))), + ]); + const { db } = await table(); + for (const condition of WRAPPED) { + const node = compileWhere(T, condition, "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text} ORDER BY id`, ...params); + const expected = ROWS.filter((row) => matches(node, row)).map(({ id }) => id); + expect([JSON.stringify(node), rows.map(({ id }) => id)]).toEqual([JSON.stringify(node), expected]); + } + await db.close(); + for (const condition of WRAPPED) { + const { db, sql } = await table(); + const node = compileSet(T, { n: when(condition, 1, 2) }, "updateWhere"); + const statement = sql.updateWhere({ and: [] } as WhereNode, node, new Map()); + await db.run(statement.sql, ...statement.params); + const stored = await db.all<{ [column: string]: SqliteValue }>(`SELECT * FROM t ORDER BY id`); + const read = stored.map((row) => T.rows.decode(row, context)); + expect([JSON.stringify(node), read]).toEqual([ + JSON.stringify(node), + ROWS.map((row) => applySet(node, row, "t")), + ]); + await db.close(); + } + }); + + it("binds a long inArray list as one parameter (beyond SQLite's 32,766 variables)", async () => { + const { db } = await table(); + const values = Array.from({ length: 40_000 }, (_, i) => `v${i}`); + values.push("b"); + const node = compileWhere(T, inArray(refs.path, values), "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + expect(params).toHaveLength(1); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text}`, ...params); + expect(rows.map(({ id }) => id)).toEqual(["4"]); + await db.close(); + }); + + it("updates rows as the evaluator does", async () => { + const SETS = [ + { n: add(refs.n, 10), count: sub(coalesce(refs.count, 5), 1) }, + { path: replacePrefix(refs.path, "a/", "\u{1F600}/"), note: concat(refs.path, "!") }, + { x: min(refs.x, 1), n: max(refs.n, 1), note: when(prefix(refs.path, "a"), refs.path, refs.note) }, + { note: refs.path, path: coalesce(refs.note, "none"), count: undefined }, + ]; + for (const set of SETS) { + const { db, sql } = await table(); + const node = compileSet(T, set, "updateWhere"); + const statement = sql.updateWhere({ and: [] } as WhereNode, node, new Map()); + await db.run(statement.sql, ...statement.params); + const stored = await db.all<{ [column: string]: SqliteValue }>(`SELECT * FROM t ORDER BY id`); + const read = stored.map((row) => T.rows.decode(row, context)); + expect(read).toEqual(ROWS.map((row) => applySet(node, row, "t"))); + await db.close(); + } + }); +}); diff --git a/packages/chord/test/next/store/export.test.ts b/packages/chord/test/next/store/export.test.ts new file mode 100644 index 000000000..9a8d70076 --- /dev/null +++ b/packages/chord/test/next/store/export.test.ts @@ -0,0 +1,283 @@ +// Tests of `store.export(ctx)` (slice 6a §13, rule 26): the parts in archive order, lossless; the commit queue held +// from the first `next()` until the export ends, is closed or aborts (also while the consumer is suspended); GC of +// closed and aborted iterators and their abort listeners. + +import { getEventListeners } from "node:events"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { ARCHIVE_FORMAT } from "../../../src/next/store/archive.ts"; +import { insertDocument, insertRevision } from "../../../src/next/store/engine.ts"; +import { EXPORT_PAGE } from "../../../src/next/store/export.ts"; +import { STORE_SCHEMA } from "../../../src/next/store/schema.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { + archiveStore, + expectCollected, + exported, + flushed, + memoryStore, + rejection, + settledHeap, + tableIn, +} from "../support.ts"; + +const OPEN = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +describe("parts", () => { + it("come in archive order: manifest, per table its table write, puts and indexes, each record and its revisions, end", async () => { + store = await archiveStore(); + const parts = await exported(store); + expect(parts[0]).toEqual({ + kind: "manifest", + format: ARCHIVE_FORMAT, + schema: STORE_SCHEMA, + lastSeq: store.lastSeq, + nextId: store.nextId, + }); + const kinds = parts.map((part) => + part.kind === "put" || part.kind === "table" || part.kind === "index" + ? `${part.kind} ${part.table.split(":")[1]}` + : part.kind, + ); + const runs: [string, number][] = []; + for (const kind of kinds) { + const last = runs.at(-1); + if (last?.[0] === kind) last[1]++; + else runs.push([kind, 1]); + } + expect(runs).toEqual([ + ["manifest", 1], + ["table edits", 1], + ["put edits", 150], + ["index edits", 1], + ["table notes", 1], + ["put notes", 150], + ["document.create", 1], + ["document.revision", 1], + ["document.create", 1], + ["document.revision", 3], + ["document.create", 1], + ["document.revision", 2], + ["end", 1], + ]); + expect(parts.at(-1)).toEqual({ kind: "end", writes: parts.length - 2 }); + }); + + it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, revisions as stored", async () => { + store = await archiveStore(); + const parts = await exported(store); + const edits = parts.filter((part) => part.kind === "put" && part.table.endsWith(":edits")); + expect(edits.slice(0, 2)).toEqual([ + { + kind: "put", + table: "app.store:edits", + row: { id: "e000", path: "p/0", at: 0, note: "n0", meta: { a: 0 }, done: false }, + version: 1, + columns: { meta: 1 }, + }, + { + kind: "put", + table: "app.store:edits", + row: { id: "e001", path: "p/1", at: 1, done: false }, + version: 1, + columns: {}, + }, + ]); + expect(parts.find((part) => part.kind === "table" && part.table.endsWith(":notes"))).toMatchObject({ + version: 2, + state: "migrating", + }); + const notes = parts.filter((part) => part.kind === "put" && part.table.endsWith(":notes")); + expect(notes.map((part) => (part.kind === "put" ? part.version : 0)).filter((v) => v === 2)).toHaveLength(100); + const documents = parts.flatMap((part): unknown[][] => + part.kind === "document.create" + ? [[part.record.key, part.record.revision, part.record.history]] + : part.kind === "document.revision" + ? [[part.content.kind, part.content.kind === "base" ? part.content.value : part.content.ops]] + : [], + ); + expect(documents).toEqual([ + // A singleton's record has no key. + [undefined, 1, "none"], + ["base", { title: "" }], + ["a", 3, "none"], + ["base", { title: "a" }], + ["delta", [["s", ["title"], "a2"]]], + ["delta", [["s", ["title"], "a3"]]], + ["b", 2, "none"], + ["base", { title: "b" }], + ["delta", [["s", ["title"], "b2"]]], + ]); + }); +}); + +describe("revisions", () => { + it("a history longer than a page is read page by page and exported whole, in seq order", async () => { + store = await memoryStore(); + const count = EXPORT_PAGE * 2 + 1; + const record = { + id: 2, + owner: "app.store", + namespace: "session", + kind: "long", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const create: StagedWrite = { + order: "write", + apply: async (step) => { + await insertDocument(step, record, "{}"); + return []; + }, + }; + await store.commit([create], OPEN, context); + for (let n = 1; n < count; n++) { + const ops = JSON.stringify([["s", ["n"], n]]); + const delta: StagedWrite = { + order: "write", + apply: async (step) => { + await insertRevision(step, record.id, step.seq, "delta", 1, ops); + return []; + }, + }; + await store.commit([delta], OPEN, context); + } + const revisions = (await exported(store)).flatMap((part) => (part.kind === "document.revision" ? [part] : [])); + expect(revisions.map(({ seq }) => seq)).toEqual(Array.from({ length: count }, (_, index) => index + 1)); + }); +}); + +describe("the commit queue", () => { + it("is held from the first next() until the export ends: a commit waits", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await iterator.next(); + let committed = false; + const commit = store.commit([], OPEN, context).then(() => { + committed = true; + }); + await flushed(); + expect(committed).toBe(false); + while (!(await iterator.next()).done) {} + await commit; + expect(committed).toBe(true); + }); + + it("is given back by return(); the closed iterator keeps nothing", async () => { + store = await archiveStore(); + let iterator: AsyncIterableIterator | undefined = store.export(context); + await iterator.next(); + await iterator.next(); + await iterator.return!(); + await store.commit([], OPEN, context); + const collected = new WeakRef(iterator); + iterator = undefined; + await expectCollected(collected); + }); + + it("is given back at once when ctx aborts while the consumer is suspended; a later next() rejects aborted", async () => { + store = await archiveStore(); + const { context: cancellable, cancel } = withCancel(context); + const iterator = store.export(cancellable); + await iterator.next(); + const signal = cancellable.abortSignal!; + expect(getEventListeners(signal, "abort")).toHaveLength(1); + cancel(new Error("stop")); + await store.commit([], OPEN, context); + expect(getEventListeners(signal, "abort")).toHaveLength(0); + expect(await rejection(iterator.next())).toMatchObject({ code: "aborted" }); + }); + + it("a retained aborted iterator keeps no page", async () => { + store = await memoryStore(); + const Big = defineTable("big", { version: 1, columns: { id: int(), text: str() }, key: ["id"] }); + const staged: StagedWrite[] = []; + const changes = new TableChanges(await tableIn(store, Big), staged); + for (let i = 0; i < EXPORT_PAGE; i++) changes.put({ id: i, text: `${i}`.padEnd(8_000, "x") }); + await store.commit(staged, OPEN, context); + const { context: cancellable, cancel } = withCancel(context); + const before = await settledHeap(); + const iterator = store.export(cancellable); + // The manifest, the table write, and the first put of a page of 100 rows of 8 KB each. + for (let i = 0; i < 3; i++) await iterator.next(); + cancel(new Error("stop")); + const after = await settledHeap(); + expect(after - before).toBeLessThan(200 * 1024); + expect(await rejection(iterator.next())).toMatchObject({ code: "aborted" }); + }); + + it("code that commits while consuming its own export waits until ctx aborts", async () => { + store = await archiveStore(); + const { context: cancellable, cancel } = withCancel(context); + const iterator = store.export(cancellable); + await iterator.next(); + const commit = store.commit([], OPEN, context); + let committed = false; + void commit.then(() => { + committed = true; + }); + await flushed(); + expect(committed).toBe(false); + cancel(new Error("stop")); + await commit; + expect(committed).toBe(true); + }); + + it("closing the store ends a held export: its next() rejects released, also for a part that needs no read", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await iterator.next(); + await store.close(context); + // The table write would need no read. + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + }); + + it("an abort while next() waits for the queue rejects at once; no abort listener is left", async () => { + store = await archiveStore(); + const first = store.export(context); + await first.next(); + const { context: cancellable, cancel } = withCancel(context); + const second = store.export(cancellable); + const waiting = rejection(second.next()); + await flushed(); + cancel(new Error("stop")); + expect(await waiting).toMatchObject({ code: "aborted" }); + expect(getEventListeners(cancellable.abortSignal!, "abort")).toHaveLength(0); + // The first export still holds the queue; once it ends, the aborted one gives the queue back at once. + await first.return!(); + await store.commit([], OPEN, context); + expect(await rejection(second.next())).toMatchObject({ code: "aborted" }); + }); + + it("closing the store before an export's first next() rejects it released: not even a manifest", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await store.close(context); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + }); + + it("closing the store while an export waits for the queue rejects its next() released", async () => { + store = await archiveStore(); + const first = store.export(context); + await first.next(); + const second = store.export(context); + const waiting = rejection(second.next()); + await flushed(); + await store.close(context); + expect(await waiting).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/store/expression.test.ts b/packages/chord/test/next/store/expression.test.ts new file mode 100644 index 000000000..c39c844ec --- /dev/null +++ b/packages/chord/test/next/store/expression.test.ts @@ -0,0 +1,152 @@ +// Tests of the table expression language: the builders and their compilation into the data vocabulary. +import { describe, expect, it } from "vitest"; +import { bool, int, nul, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { + add, + and, + coalesce, + compileSet, + compileWhere, + concat, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + max, + ne, + not, + or, + prefix, + replacePrefix, + sub, + when, +} from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { ColumnRefs } from "../../../src/next/store/query.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("t", { + version: 1, + columns: { + id: str(), + path: str(), + n: int(), + x: num(), + b: bool(), + note: opt(str()), + count: opt(int()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], +}); +type C = typeof T extends { "~columns"?: infer X } ? NonNullable : never; +const refs = T.refs as ColumnRefs; +const args = expect.objectContaining({ code: "invalid", reason: "args" }); +describe("compiling conditions", () => { + it("compiles every operator into data", () => { + const where = and( + eq(refs.path, "a"), + ne(refs.n, 1), + lt(refs.n, 2), + lte(refs.x, 2.5), + gt(refs.n, 0), + gte(refs.path, "a"), + inArray(refs.n, [1, 2]), + isNull(refs.note), + prefix(refs.path, "a/"), + or(not(eq(refs.b, true))), + ); + expect(compileWhere(T, where, "where")).toEqual({ + and: [ + { eq: ["path", "a"] }, + { ne: ["n", 1] }, + { lt: ["n", 2] }, + { lte: ["x", 2.5] }, + { gt: ["n", 0] }, + { gte: ["path", "a"] }, + { inArray: ["n", [1, 2]] }, + { isNull: "note" }, + { prefix: ["path", "a/"] }, + { or: [{ not: { eq: ["b", true] } }] }, + ], + }); + }); + + it("refuses undefined and null operands (rule 16), wrong types, json and unknown columns", () => { + expect(() => compileWhere(T, eq(refs.note, undefined as never), "where")).toThrow("ask with isNull"); + expect(() => compileWhere(T, eq(refs.note, null as never), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.n, "1" as never), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.n, 1.5), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.meta as never, 1), "where")).toThrow("not queryable"); + expect(() => compileWhere(T, eq({ column: "nope" } as never, 1), "where")).toThrow("expected a column reference"); + expect(() => compileWhere(T, "x", "where")).toThrow("expected a condition"); + expect(() => compileWhere(T, prefix(refs.n as never, "a"), "where")).toThrow("str() column"); + }); + + it("makes text operands storable", () => { + expect(compileWhere(T, eq(refs.path, "a\uD800"), "where")).toEqual({ eq: ["path", "a\uFFFD"] }); + expect(() => compileWhere(T, prefix(refs.path, "a\0"), "where")).toThrow("NUL"); + }); +}); + +describe("compiling sets", () => { + it("compiles plain values and expressions into one vocabulary, a clear as null (rule 14)", () => { + expect(compileSet(T, { path: "p", note: undefined, b: true, meta: { a: 1 } }, "update")).toEqual({ + path: "p", + note: null, + b: true, + meta: { json: { a: 1 } }, + }); + const set = compileSet( + T, + { + n: add(refs.n, 1), + count: sub(coalesce(refs.count, 0), refs.n), + x: max(refs.x, 1.5), + path: replacePrefix(concat(refs.path, "/"), "a/", "b/"), + note: when(gt(refs.n, 1), refs.path, refs.note), + }, + "update", + ); + expect(JSON.parse(JSON.stringify(set))).toEqual({ + n: { add: [{ col: "n" }, 1] }, + count: { sub: [{ coalesce: [{ col: "count" }, 0] }, { col: "n" }] }, + x: { max: [{ col: "x" }, 1.5] }, + path: { replacePrefix: [{ concat: [{ col: "path" }, "/"] }, "a/", "b/"] }, + note: { when: [{ gt: ["n", 1] }, { col: "path" }, { col: "note" }] }, + }); + }); + + it("refuses an empty set, key columns (rule 13), unknown columns and clears of non-opt columns", () => { + expect(() => compileSet(T, {}, "update")).toThrow("empty"); + expect(() => compileSet(T, { id: "x" }, "update")).toThrow("cannot change"); + expect(() => compileSet(T, { nope: 1 }, "update")).toThrow('no column "nope"'); + expect(() => compileSet(T, { path: undefined }, "update")).toThrow("cannot be cleared"); + }); + + it("undefined clears an opt column; null is a json(...) cell's JSON null and refused for scalar columns", () => { + expect(compileSet(T, { note: undefined, meta: undefined }, "update")).toEqual({ note: null, meta: null }); + const Nullable = defineTable("n", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }) }, + key: ["id"], + }); + expect(compileSet(Nullable, { data: null }, "update")).toEqual({ data: { json: null } }); + expect(() => compileSet(T, { note: null as never }, "update")).toThrow("null is no str value"); + }); + + it("refuses integer arithmetic outside int columns (rule 10), type mismatches and nullable results into non-opt columns", () => { + expect(() => compileSet(T, { x: add(refs.n, 1) }, "update")).toThrow("add gives int"); + expect(() => compileSet(T, { path: max(refs.n, 1) as never }, "update")).toThrow("gives int or num"); + expect(() => compileSet(T, { path: refs.n as never }, "update")).toThrow('column "n" is int, not str'); + expect(() => compileSet(T, { n: add(refs.count, 1) }, "update")).toThrow("may be NULL"); + expect(() => compileSet(T, { path: refs.note }, "update")).toThrow("may be NULL"); + expect(compileSet(T, { path: coalesce(refs.note, "d") }, "update")).toEqual({ + path: { coalesce: [{ col: "note" }, "d"] }, + }); + expect(() => compileSet(T, { meta: refs.meta as never }, "update")).toThrow("plain values"); + }); +}); diff --git a/packages/chord/test/next/store/import.test.ts b/packages/chord/test/next/store/import.test.ts new file mode 100644 index 000000000..02350a696 --- /dev/null +++ b/packages/chord/test/next/store/import.test.ts @@ -0,0 +1,496 @@ +// Tests of `importStore` (slice 6a §13, rules 12, 24, 25): round trips (old row versions, a migrating table, +// uncheckpointed deltas), the store it returns, every refusal of the archive's order and content, the rollback, and an +// input that stalls cancelled by `ctx`. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { nul, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { findDocument, loadDocument } from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { ArchiveTables, archiveStore, exported, memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; + +const stores: ChordStore[] = []; +afterEach(async () => { + for (const store of stores.splice(0)) await store.close(context); +}); + +/** `parts` imported into a fresh `:memory:` backend. */ +async function imported(parts: Iterable | AsyncIterable, outbox?: boolean): Promise { + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + const store = (await importStore(backend, parts, context, { outbox })) as ChordStore; + stores.push(store); + return store; +} + +/** The archive of `archiveStore`. */ +async function archive(): Promise { + const source = await archiveStore(); + stores.push(source); + return exported(source); +} + +describe("round trips", () => { + it("an imported archive exports the same parts: lossless, at the manifest's lastSeq and next document id", async () => { + const parts = await archive(); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const manifest = parts[0] as Extract; + expect([store.lastSeq, store.nextId]).toEqual([manifest.lastSeq, manifest.nextId]); + }); + + it("the store keeps its ids, seqs and folded documents; the outbox, when asked, starts after lastSeq", async () => { + const parts = await archive(); + const store = await imported(parts, true); + expect(await store.outbox!.read(store.lastSeq, 10, context)).toEqual([]); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid" }); + const record = await store.read( + (executor) => findDocument(executor, { namespace: "chord", owner: "app.store", kind: "memo", key: "a" }), + context, + ); + const original = parts.find((part) => part.kind === "document.create" && part.record.key === "a"); + expect(record).toEqual((original as Extract).record); + expect((await store.read((executor) => loadDocument(executor, record!), context)).value).toEqual({ title: "a3" }); + }); + + it("a migrating table resumes on the imported store; rows written at version 1 read as the table's", async () => { + const store = await imported(await archive()); + const { EditsV2, NotesV2 } = ArchiveTables; + const notes = "app.store:notes"; + const fence = { admit: () => true, adopt: () => {} }; + await expect( + prepareTables([{ definition: NotesV2, table: notes }], store, fence, "app", context), + ).rejects.toThrow("crash"); + const edits = new TableHandle(EditsV2, "app.store:edits", store, tableOwner()); + expect(await edits.get({ id: "e003" }, context)).toEqual({ + id: "e003", + path: "p/3", + at: 3, + meta: { a: 3 }, + done: false, + }); + }); +}); + +describe("json(...) cells holding null", () => { + it("a JSON null round-trips, required and optional, put and updated; an absent optional cell stays absent", async () => { + const source = await memoryStore(); + stores.push(source); + const Cells = defineTable("cells", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }), more: opt(json(nul(), { version: 1 })) }, + key: ["id"], + }); + const handle = await tableIn(source, Cells); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + changes.put({ id: "a", data: null, more: null }); + changes.put({ id: "b", data: null }); + changes.put({ id: "c", data: null, more: null }); + changes.update({ id: "c" }, { more: undefined }); + await source.commit(staged, { admit: () => true, adopt: () => {} }, context); + const rows = [ + { id: "a", data: null, more: null }, + { id: "b", data: null }, + { id: "c", data: null }, + ]; + for (const row of rows) expect(await handle.get({ id: row.id }, context)).toEqual(row); + const parts = await exported(source); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const copy = new TableHandle(Cells, handle.sql.table, store, tableOwner()); + for (const row of rows) expect(await copy.get({ id: row.id }, context)).toEqual(row); + }); + + it("a JSON null at an older cell version round-trips with its version and migrates on access in the copy", async () => { + const source = await memoryStore(); + stores.push(source); + const V1 = defineTable("cells", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }) }, + key: ["id"], + }); + const old = await tableIn(source, V1); + const staged: StagedWrite[] = []; + new TableChanges(old, staged).put({ id: "a", data: null }); + await source.commit(staged, { admit: () => true, adopt: () => {} }, context); + const V2 = evolveTable(V1, (t) => + t.json("data", json(obj({ was: nul() }), { version: 2, migrate: (value) => ({ was: value as null }) })), + ); + const handle = await tableIn(source, V2); + expect(await handle.get({ id: "a" }, context)).toEqual({ id: "a", data: { was: null } }); + const parts = await exported(source); + expect(parts.find((part) => (part as ArchivePart).kind === "put")).toMatchObject({ + row: { id: "a", data: null }, + columns: { data: 1 }, + }); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const copy = new TableHandle(V2, handle.sql.table, store, tableOwner()); + expect(await copy.get({ id: "a" }, context)).toEqual({ id: "a", data: { was: null } }); + }); +}); + +/** The archive with `edit` applied to a copy of its parts. */ +async function edited(edit: (parts: ArchivePart[]) => unknown[]): Promise { + return edit(structuredClone(await archive())); +} + +/** The index of the first part of `kind` (and table suffix). */ +function first(parts: readonly ArchivePart[], kind: ArchivePart["kind"], table?: string): number { + return parts.findIndex( + (part) => part.kind === kind && (table === undefined || ("table" in part && part.table.endsWith(table))), + ); +} + +interface Mutable { + [field: string]: JsonValue | Mutable; +} + +/** Part `index` as a writable object. */ +/** The positions of every part of `kind`. */ +function all(parts: readonly ArchivePart[], kind: ArchivePart["kind"]): number[] { + return parts.flatMap((part, index) => (part.kind === kind ? [index] : [])); +} + +function at(parts: unknown[], index: number): Mutable { + return parts[index] as Mutable; +} + +/** The content of the revision part at `index`, writable. */ +function revisionContent(parts: unknown[], index: number): Mutable { + return at(parts, index).content as Mutable; +} + +describe("refusals", () => { + it.each<[string, (parts: ArchivePart[]) => unknown[], string]>([ + [ + "a part outside the schema", + (parts) => [...parts.slice(0, 1), { kind: "put" }, ...parts.slice(1)], + "not an archive part", + ], + ["no manifest first", (parts) => parts.slice(1), "the manifest comes first"], + ["a second manifest", (parts) => [parts[0], ...parts], "the manifest comes first, once"], + [ + "another format", + (parts) => { + at(parts, 0).format = 2; + return parts; + }, + "format 2 is not 1", + ], + [ + "a newer store schema", + (parts) => { + at(parts, 0).schema = 9; + return parts; + }, + "schema 9 is newer", + ], + [ + "a lastSeq of 2^53 (a later seq would repeat)", + (parts) => { + at(parts, 0).lastSeq = 2 ** 53; + return parts; + }, + "part 0: not an archive part", + ], + [ + "a document revision beyond 2^53 - 1", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.revision = 2 ** 53; + return parts; + }, + "not an archive part", + ], + ["a part after the end", (parts) => [...parts, { kind: "end", writes: 0 }], "after the end"], + ["no end", (parts) => parts.slice(0, -1), "has no end part"], + [ + "a wrong end count", + (parts) => { + at(parts, parts.length - 1).writes = 1; + return parts; + }, + "counts 1 writes", + ], + [ + "a put before its table", + (parts) => { + const put = first(parts, "put", ":notes"); + return [...parts.slice(0, first(parts, "table")), parts[put], ...parts.slice(first(parts, "table"))]; + }, + "a put follows its table's table write", + ], + [ + "a put after its table's indexes", + (parts) => { + const index = first(parts, "index"); + return [...parts.slice(0, index + 1), parts[index - 1], ...parts.slice(index + 1)]; + }, + "a put follows its table's table write, before its indexes", + ], + [ + "a table twice", + (parts) => { + const table = first(parts, "table", ":notes"); + return [...parts.slice(0, table), parts[first(parts, "table")], ...parts.slice(table)]; + }, + "occurs twice", + ], + [ + "a table after the documents", + (parts) => { + const table = first(parts, "table", ":notes"); + const documents = first(parts, "document.create"); + return [ + ...parts.slice(0, table), + ...parts.slice(documents, -1), + ...parts.slice(table, documents), + parts.at(-1), + ]; + }, + "tables come before the documents", + ], + [ + "a table name that is not owner:table", + (parts) => { + at(parts, first(parts, "table")).table = "edits"; + return parts; + }, + "not owner:table", + ], + [ + "a column name breaking the rules", + (parts) => { + const table = at(parts, first(parts, "table")); + table.columns = { ...(table.columns as Mutable), "bad name": { type: "str", optional: true } }; + return parts; + }, + "not a column name", + ], + [ + "a row with an unknown column", + (parts) => { + (at(parts, first(parts, "put")).row as Mutable).extra = 1; + return parts; + }, + 'unknown column "extra"', + ], + [ + "a row without a non-opt column", + (parts) => { + delete (at(parts, first(parts, "put")).row as Mutable).path; + return parts; + }, + 'column "path" is missing', + ], + [ + "a value of another type", + (parts) => { + (at(parts, first(parts, "put")).row as Mutable).at = "x"; + return parts; + }, + "is not a int value", + ], + [ + "a json(...) cell newer than its column", + (parts) => { + (at(parts, first(parts, "put")).columns as Mutable).meta = 5; + return parts; + }, + "no readable version", + ], + [ + "a row newer than its table", + (parts) => { + at(parts, first(parts, "put")).version = 7; + return parts; + }, + "version 7 is newer", + ], + [ + "an id not below the manifest's next id", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).id = 10_000; + return parts; + }, + "is not below", + ], + [ + "an id twice", + (parts) => { + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).id = (at(parts, one!).record as Mutable).id!; + return parts; + }, + "occurs twice", + ], + [ + "a record created after lastSeq", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).createdAt = 10_000; + return parts; + }, + "a lifetime outside the manifest's lastSeq", + ], + [ + "a base at another version than its record's", + (parts) => { + revisionContent(parts, first(parts, "document.revision")).version = 2; + return parts; + }, + "has no base at its version", + ], + [ + "a delta without a base at its version", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + revisionContent(parts, delta).version = 2; + return parts; + }, + "a delta without a base at its version", + ], + [ + "a revision out of seq order", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + at(parts, delta).seq = 1; + return parts; + }, + "out of seq order", + ], + [ + "a revision of another record", + (parts) => { + at(parts, first(parts, "document.revision")).id = 9_999; + return parts; + }, + "follows its record's create", + ], + [ + "a live record without revisions", + (parts) => parts.filter((_, index) => index !== first(parts, "document.revision")), + "has no base at its version", + ], + [ + "revisions of a retired none record", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.retiredAt = record.createdAt!; + return parts; + }, + "keeps no revisions", + ], + [ + "a retired rewindable record without its history", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.history = "rewindable"; + record.retiredAt = record.createdAt!; + const id = record.id; + return parts.filter((part) => part.kind !== "document.revision" || part.id !== id); + }, + "has no base at its version", + ], + [ + "two live records at one address", + (parts) => { + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).key = (at(parts, one!).record as Mutable).key!; + return parts; + }, + "a second record at one address for an overlapping lifetime", + ], + [ + "a record's meta with a NUL", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).meta = "a\u0000b"; + return parts; + }, + "text may not contain NUL", + ], + [ + "a record's meta with a lone surrogate", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).meta = "a\uD800"; + return parts; + }, + "the meta is not well-formed text", + ], + ])("%s", async (_, edit, message) => { + const parts = await edited(edit); + expect(await rejection(imported(parts))).toMatchObject({ + code: "invalid", + reason: "args", + message: expect.stringContaining(message), + }); + }); + + it("a non-empty backend is refused; a refused import closes its backend and leaves it as it was", async () => { + const directory = await mkdtemp(join(tmpdir(), "chord-import-")); + try { + const path = join(directory, "store.db"); + const parts = await archive(); + const broken = [...parts.slice(0, -1), { kind: "end", writes: 0 }]; + expect( + await rejection(importStore(sqliteBackend(await openNodeSqliteDatabase(path)), broken, context)), + ).toMatchObject({ + code: "invalid", + }); + const db = await openNodeSqliteDatabase(path); + expect(await db.all("SELECT name FROM sqlite_master")).toEqual([]); + await db.close(); + const store = await importStore(sqliteBackend(await openNodeSqliteDatabase(path)), parts, context); + await store.close(context); + expect( + await rejection(importStore(sqliteBackend(await openNodeSqliteDatabase(path)), parts, context)), + ).toMatchObject({ + message: "import: the backend holds a store already", + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("an input that stalls is cancelled by ctx: the import rejects aborted and the input is closed", async () => { + const parts = await archive(); + let closed = false; + const stalled: AsyncIterable = { + [Symbol.asyncIterator]: () => { + let index = 0; + return { + next: () => + index < 3 + ? Promise.resolve({ done: false, value: parts[index++] }) + : new Promise>(() => {}), + return: async () => { + closed = true; + return { done: true, value: undefined }; + }, + }; + }, + }; + const { context: cancellable, cancel } = withCancel(context); + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + const importing = importStore(backend, stalled, cancellable); + await new Promise((resolve) => setTimeout(resolve, 10)); + cancel(new Error("stop")); + expect(await rejection(importing)).toMatchObject({ code: "aborted" }); + expect(closed).toBe(true); + }); +}); diff --git a/packages/chord/test/next/store/json-column.test.ts b/packages/chord/test/next/store/json-column.test.ts new file mode 100644 index 000000000..339aaf1f3 --- /dev/null +++ b/packages/chord/test/next/store/json-column.test.ts @@ -0,0 +1,58 @@ +// `json(...)` columns (rule 8): their declaration, and a cell read and migrated on access. + +import { describe, expect, it } from "vitest"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { JsonColumn, json } from "../../../src/next/store/json-column.ts"; +import { counted } from "../support.ts"; + +const MetaV1 = obj({ tag: str() }); +const MetaV2 = obj({ tags: str(), n: int() }); + +describe("json(...)", () => { + it("is a schema carrying its column: version and the fingerprint of the schema's normal form", () => { + const column = (json(MetaV1, { version: 1 }) as unknown as { json: JsonColumn }).json; + expect(column).toBeInstanceOf(JsonColumn); + expect(column.version).toBe(1); + const again = (json(obj({ tag: str({}) }), { version: 1 }) as unknown as { json: JsonColumn }).json; + const other = (json(MetaV2, { version: 1 }) as unknown as { json: JsonColumn }).json; + expect(again.fingerprint).toBe(column.fingerprint); + expect(other.fingerprint).not.toBe(column.fingerprint); + }); + + it("refuses a version that is not a positive integer", () => { + expect(() => json(MetaV1, { version: 0 })).toThrow("version 0"); + }); +}); + +describe("reading a cell", () => { + const v2 = new JsonColumn(MetaV2, 2, (old, from) => ({ tags: (old as { tag: string }).tag, n: from })); + + it("parses a cell of the current version with no check", async () => { + const counts = await counted((context) => { + expect(v2.read('{"tags":"a","n":1}', 2, "t.meta", context)).toEqual({ tags: "a", n: 1 }); + }); + expect(counts.validations).toBe(0); + }); + + it("migrates an older cell and checks the result once (R8)", async () => { + const counts = await counted((context) => { + expect(v2.read('{"tag":"x"}', 1, "t.meta", context)).toEqual({ tags: "x", n: 1 }); + }); + expect(counts.validations).toBe(1); + }); + + it("refuses a newer cell, a missing or throwing migrate and an invalid result", async () => { + await counted((context) => { + expect(() => v2.read("{}", 3, "t.meta", context)).toThrow("newer than 2"); + expect(() => new JsonColumn(MetaV2, 2, undefined).read("{}", 1, "t.meta", context)).toThrow("no migrate"); + const throwing = new JsonColumn(MetaV2, 2, () => { + throw new Error("no"); + }); + expect(() => throwing.read("{}", 1, "t.meta", context)).toThrow("threw"); + const invalid = new JsonColumn(MetaV2, 2, () => ({ tags: 1, n: 1 }) as never); + expect(() => invalid.read("{}", 1, "t.meta", context)).toThrow( + expect.objectContaining({ code: "unavailable" }), + ); + }); + }); +}); diff --git a/packages/chord/test/next/store/jsonl.test.ts b/packages/chord/test/next/store/jsonl.test.ts new file mode 100644 index 000000000..9664737d3 --- /dev/null +++ b/packages/chord/test/next/store/jsonl.test.ts @@ -0,0 +1,56 @@ +// Tests of the JSONL archive encoding (slice 6a §13): parts as lines, chunks split anywhere (inside a line, inside a +// UTF-8 sequence), blank lines, and the refusal of a line that is no JSON. + +import { describe, expect, it } from "vitest"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { decodeJsonl, encodeJsonl } from "../../../src/next/store/jsonl.ts"; +import { rejection } from "../support.ts"; + +async function* from(values: readonly T[]): AsyncIterable { + yield* values; +} + +async function collect(values: AsyncIterable): Promise { + const all: T[] = []; + for await (const value of values) all.push(value); + return all; +} + +const PARTS: ArchivePart[] = [ + { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextId: 2 }, + { kind: "put", table: "app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, + { kind: "end", writes: 1 }, +]; + +describe("jsonl", () => { + it("encodes one part per line and decodes it back, from chunks split anywhere", async () => { + const bytes = Buffer.concat(await collect(encodeJsonl(from(PARTS)))); + expect(bytes.toString("utf8").split("\n")).toHaveLength(PARTS.length + 1); + for (const size of [1, 2, 3, 7, bytes.length]) { + const chunks: Uint8Array[] = []; + for (let at = 0; at < bytes.length; at += size) chunks.push(bytes.subarray(at, at + size)); + expect(await collect(decodeJsonl(from(chunks)))).toEqual(PARTS); + } + }); + + it("skips blank lines and takes a last line without a newline", async () => { + const text = `\n${JSON.stringify(PARTS[0])}\n\n${JSON.stringify(PARTS[2])}`; + expect(await collect(decodeJsonl(from([new TextEncoder().encode(text)])))).toEqual([PARTS[0], PARTS[2]]); + }); + + it("refuses a line that is no JSON, naming it", async () => { + const text = `${JSON.stringify(PARTS[0])}\n{nope\n`; + expect(await rejection(collect(decodeJsonl(from([new TextEncoder().encode(text)]))))).toMatchObject({ + code: "invalid", + reason: "args", + message: "jsonl: line 2 is not JSON", + }); + }); + + it("refuses bytes that are no UTF-8", async () => { + expect(await rejection(collect(decodeJsonl(from([new Uint8Array([0x7b, 0xff, 0x0a])]))))).toMatchObject({ + code: "invalid", + message: "jsonl: line 1 is not UTF-8", + }); + }); +}); diff --git a/packages/chord/test/next/store/outbox.test.ts b/packages/chord/test/next/store/outbox.test.ts new file mode 100644 index 000000000..4b5196524 --- /dev/null +++ b/packages/chord/test/next/store/outbox.test.ts @@ -0,0 +1,166 @@ +// Tests of the outbox (slice 6a §2; decisions "The outbox is optional, internal and versioned", small items 3-4): on, +// off and kept across opens, its commits in their recorded form, `trim` and its floor, and the format check. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument } from "../../../src/next/store/engine.ts"; +import { OUTBOX_FORMAT } from "../../../src/next/store/outbox.ts"; +import { TABLES } from "../../../src/next/store/schema.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { memoryStore, rejection } from "../support.ts"; + +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; +const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; +const OPEN = { admit: () => true, adopt: () => {} }; + +/** Creates the singleton at `ADDRESS` (with `key`) in one commit. */ +async function created(store: ChordStore, key = ""): Promise { + await store.commit([createDocument({ ...ADDRESS, key }, BASE, JSON.stringify(BASE.value))], OPEN, context); +} + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A store on a fresh file, reopened by `open`. */ +async function onFile() { + directory = await mkdtemp(join(tmpdir(), "chord-outbox-")); + const path = join(directory, "store.sqlite"); + return async (outbox?: boolean) => + (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox })) as ChordStore; +} + +describe("the outbox", () => { + it("is off for a new store, on from the seq it is turned on at, kept when omitted, dropped when turned off", async () => { + const open = await onFile(); + let store = await open(); + await created(store); + await store.close(context); + store = await open(true); + expect(store.outbox?.floor).toBe(1); + const id = 2; + const ops = [["s", ["n"], 2]] as const; + await store.commit( + [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + OPEN, + context, + ); + await store.close(context); + store = await open(); + expect(await store.outbox!.read(1, 10, context)).toEqual([ + { + seq: 2, + nextId: 3, + writes: [{ kind: "document.change", id, revision: 1, content: { kind: "delta", version: 1, ops } }], + }, + ]); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(await rejection(store.outbox!.read(3, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + await store.close(context); + store = await open(false); + expect(store.outbox).toBeUndefined(); + await store.close(context); + }); + + it("records creations with their ids and seqs; trim is idempotent and refuses beyond lastSeq", async () => { + const store = await memoryStore({ outbox: true }); + await created(store); + const [commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes[0]).toMatchObject({ + kind: "document.create", + record: { id: 2, createdAt: 1, ...ADDRESS }, + base: BASE, + }); + await store.outbox!.trim(1, context); + await store.outbox!.trim(1, context); + expect(store.outbox!.floor).toBe(1); + expect(await store.outbox!.read(1, 10, context)).toEqual([]); + expect(await rejection(store.outbox!.trim(2, context))).toMatchObject({ code: "invalid", reason: "args" }); + }); + + it("overlapping trims run one at a time: the stored floor never moves back and equals the one in memory", async () => { + const open = await onFile(); + let store = await open(true); + for (let i = 0; i < 11; i++) await created(store, `k${i}`); + await Promise.all([store.outbox!.trim(9, context), store.outbox!.trim(3, context)]); + expect(store.outbox!.floor).toBe(9); + await store.close(context); + store = await open(); + expect(store.outbox!.floor).toBe(9); + expect(await rejection(store.outbox!.read(3, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect((await store.outbox!.read(9, 10, context)).map(({ seq }) => seq)).toEqual([10, 11]); + await store.close(context); + }); + + it("a read waits for a trim whose commit has not returned: the trimmed floor refuses it, never an empty page", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + const transaction = db.transaction.bind(db); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const store = (await openStore(sqliteBackend(db), context, { outbox: true })) as ChordStore; + await created(store); + // The trim's DELETE committed; its transaction has not returned (the floor in memory is still 0). + db.transaction = async (fn) => { + const result = await transaction(fn); + entered.resolve(); + await gate.promise; + return result; + }; + const trim = store.outbox!.trim(1, context); + await entered.promise; + db.transaction = transaction; + const read = rejection(store.outbox!.read(0, 100, context)); + gate.resolve(); + await trim; + expect(await read).toMatchObject({ code: "invalid", reason: "args" }); + await store.close(context); + }); + + it("a trim waits for a read in flight: the read gets the commits it was admitted for", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + const store = (await openStore(sqliteBackend(db), context, { outbox: true })) as ChordStore; + await created(store); + const all = db.all.bind(db); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + db.all = (async (sql: string, ...params: never[]) => { + entered.resolve(); + await gate.promise; + return all(sql, ...params); + }) as typeof db.all; + const read = store.outbox!.read(0, 100, context); + await entered.promise; + db.all = all; + const trim = store.outbox!.trim(1, context); + gate.resolve(); + expect((await read).map(({ seq }) => seq)).toEqual([1]); + await trim; + expect(store.outbox!.floor).toBe(1); + await store.close(context); + }); + + it("a commit stored in a newer format is refused unavailable; the data stays", async () => { + const open = await onFile(); + let store = await open(true); + await created(store); + await store.close(context); + const db = await openNodeSqliteDatabase(join(directory!, "store.sqlite")); + await db.run(`UPDATE ${TABLES.outbox} SET format = ? WHERE seq = 1`, OUTBOX_FORMAT + 1); + await db.close(); + store = await open(); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ + code: "unavailable", + message: expect.stringContaining(`format ${OUTBOX_FORMAT + 1}`), + }); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "unavailable" }); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/store/query.test.ts b/packages/chord/test/next/store/query.test.ts new file mode 100644 index 000000000..2c6875a5f --- /dev/null +++ b/packages/chord/test/next/store/query.test.ts @@ -0,0 +1,82 @@ +// Queries (slice 6a §7): `where`, `orderBy` and limits over a table, and cursors paging through them. + +import { describe, expect, it } from "vitest"; +import { int, opt, str } from "../../../src/next/contracts/index.ts"; +import { eq, gt } from "../../../src/next/store/expression.ts"; +import { decodeCursor, encodeCursor, query } from "../../../src/next/store/query.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("edits", { + version: 1, + columns: { id: str(), entry: int(), at: int(), note: opt(str()) }, + key: ["id", "entry"], +}); +const args = expect.objectContaining({ code: "invalid", reason: "args" }); + +describe("queries", () => { + it("are immutable; several wheres are and-ed", () => { + const base = query(T); + const one = base.where((e) => eq(e.id, "a")); + const two = one.where((e) => gt(e.at, 1)); + expect(base.condition).toBeUndefined(); + expect(one.condition).toEqual({ eq: ["id", "a"] }); + expect(two.condition).toEqual({ and: [{ eq: ["id", "a"] }, { gt: ["at", 1] }] }); + }); + + it("order by the key ascending by default and append the key as the tie-break, not repeated", () => { + expect(query(T).order).toEqual({ columns: ["id", "entry"], direction: "asc" }); + expect(query(T).orderBy((e) => e.at, "desc").order).toEqual({ + columns: ["at", "id", "entry"], + direction: "desc", + }); + expect(query(T).orderBy((e) => e.entry).order).toEqual({ columns: ["entry", "id"], direction: "asc" }); + }); + + it("refuse duplicate orderBy columns, two directions and opt columns (rule 17)", () => { + expect(() => + query(T) + .orderBy((e) => e.at) + .orderBy((e) => e.at), + ).toThrow("twice"); + expect(() => + query(T) + .orderBy((e) => e.at) + .orderBy((e) => e.id, "desc"), + ).toThrow("one direction"); + expect(() => query(T).orderBy((e) => e.note as never)).toThrow(args); + }); +}); + +describe("cursors", () => { + const order = query(T).orderBy((e) => e.at, "desc").order; + + it("round-trip the order values, also across conditions with the same order", () => { + const cursor = encodeCursor("app:edits", order, [5, "a\u{1F600}", 2]); + expect(cursor).toMatch(/^[A-Za-z0-9_-]+$/); + expect(decodeCursor(T, "app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); + const other = query(T) + .where((e) => eq(e.id, "x")) + .orderBy((e) => e.at, "desc").order; + expect(decodeCursor(T, "app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); + }); + + it("are a trust boundary: another table, order or direction, garbage and bad values are invalid/args", () => { + const cursor = encodeCursor("app:edits", order, [5, "a", 2]); + expect(() => decodeCursor(T, "app:edits", query(T).order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); + expect(() => + decodeCursor( + defineTable("other", { version: 1, columns: { id: str(), entry: int(), at: int() }, key: ["id", "entry"] }), + "app:other", + order, + cursor, + ), + ).toThrow(args); + // The same definition in another object's table. + expect(() => decodeCursor(T, "other:edits", order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, "%%%")).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, 42)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, ["x", "a", 2]))).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, [5, "a"]))).toThrow(args); + }); +}); diff --git a/packages/chord/test/next/store/refusal.test.ts b/packages/chord/test/next/store/refusal.test.ts new file mode 100644 index 000000000..760273954 --- /dev/null +++ b/packages/chord/test/next/store/refusal.test.ts @@ -0,0 +1,65 @@ +// Tests of refusals as Chord's owners take them (slice 6b D4, §1.4): an `int` CHECK fails only its transaction, any +// other refusal its object; the store's own errors pass through. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { add } from "../../../src/next/store/expression.ts"; +import { refusalOf } from "../../../src/next/store/refusal.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore, rejection } from "../support.ts"; + +describe("classification", () => { + it("a CHECK failure by SQLite's extended code or by its message; everything else broken; the store's own errors pass", () => { + const node = Object.assign(new Error("CHECK constraint failed: n"), { errcode: 275 }); + const other = Object.assign(new Error("UNIQUE constraint failed"), { errcode: 2067 }); + expect(refusalOf(node, "x")).toMatchObject({ kind: "check", error: { code: "invalid", reason: "args" } }); + expect(refusalOf(new Error("CHECK constraint failed: n"), "x")).toMatchObject({ kind: "check" }); + expect(refusalOf(other, "x")).toMatchObject({ kind: "broken", error: { code: "unavailable" } }); + const stopped = new ChordError({ + code: "unavailable", + reason: "storage_failed", + message: "store: storage failed", + }); + expect(refusalOf(stopped, "x")).toBeUndefined(); + expect(refusalOf(new ChordError({ code: "released", message: "closed" }), "x")).toBeUndefined(); + }); +}); + +describe("Chord's owners", () => { + const Counts = defineTable("counts", { version: 1, columns: { id: str(), n: int() }, key: ["id"] }); + const Counter = local("app.counter", { + version: 1, + params: {}, + members: { bump: method(obj({ by: int() })), reset: method() }, + }); + + it("an int out of range fails only its transaction; the object stays live", async () => { + const store = await memoryStore(); + const { host, errors } = await loaded( + (h) => + h.serve(Counter, (_, self) => { + const counts = self.table(Counts); + return { + bump: async ({ by }, ctx) => { + await self.tx(async (tx) => tx.update(counts, { id: "a" }, (c) => ({ n: add(c.n, by) })), ctx); + return null; + }, + reset: async (ctx) => { + await self.tx(async (tx) => tx.put(counts, { id: "a", n: Number.MAX_SAFE_INTEGER }), ctx); + return null; + }, + }; + }), + { store }, + ); + const counter = await host.get(Counter, {}, context); + await counter.reset(context); + expect(await rejection(counter.bump({ by: 1 }, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(counter.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + await counter.bump({ by: -1 }, context); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/store/rows.test.ts b/packages/chord/test/next/store/rows.test.ts new file mode 100644 index 000000000..9f784ad90 --- /dev/null +++ b/packages/chord/test/next/store/rows.test.ts @@ -0,0 +1,77 @@ +// Rows (slice 6a §5, §9): `put` rows and keys checked and encoded, stored rows decoded. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("t", { + version: 3, + columns: { + id: str(), + n: int({ default: 7 }), + x: num(), + b: bool(), + note: opt(str()), + meta: json(obj({ a: int() }), { version: 2 }), + }, + key: ["id"], +}); +const args = expect.objectContaining({ code: "invalid", reason: "args" }); + +describe("put rows", () => { + it("encodes a row in physical order, defaults filled, booleans as 0 and 1, json serialized with its version", () => { + const encoded = T.rows.put({ id: "a", x: 1.5, b: true, meta: { a: 1 } }, 3); + expect(encoded.params).toEqual(["a", 7, 1.5, 1, null, '{"a":1}', 2, 3]); + expect(encoded.row).toEqual({ id: "a", n: 7, x: 1.5, b: true, meta: { a: 1 } }); + expect(encoded.columns).toEqual({ meta: 2 }); + expect(encoded.id).toBe(T.rows.key({ id: "a" }, "get").id); + }); + + it("is a closed check: missing and unknown columns, wrong types, unsafe integers, non-finite numbers, NUL", () => { + const row = { id: "a", x: 1, b: false, meta: {} }; + expect(() => T.rows.put({ ...row, id: undefined }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, extra: 1 }, 3)).toThrow('unknown column "extra"'); + expect(() => T.rows.put({ ...row, n: 2 ** 53 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, n: 1.5 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, x: Number.POSITIVE_INFINITY }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, b: 1 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, id: "a\0" }, 3)).toThrow("NUL"); + expect(() => T.rows.put({ ...row, note: null }, 3)).toThrow(args); + expect(() => T.rows.put([], 3)).toThrow("expected an object"); + }); + + it("serializes json cells at staging: a cycle or a BigInt fails there (rule 8)", () => { + const cycle: { self?: unknown } = {}; + cycle.self = cycle; + expect(() => T.rows.put({ id: "a", x: 1, b: false, meta: cycle }, 3)).toThrow(args); + expect(() => T.rows.put({ id: "a", x: 1, b: false, meta: { a: 1n } }, 3)).toThrow(args); + }); + + it("replaces lone surrogates in text", () => { + expect(T.rows.put({ id: "a\uD800", x: 1, b: false, meta: {} }, 3).row.id).toBe("a\uFFFD"); + }); +}); + +describe("keys", () => { + it("takes exactly the key columns", () => { + expect(T.rows.key({ id: "a" }, "get").params).toEqual(["a"]); + expect(() => T.rows.key({}, "get")).toThrow("is missing"); + expect(() => T.rows.key({ id: "a", n: 1 }, "get")).toThrow('unknown column "n"'); + expect(() => T.rows.key({ id: 1 }, "get")).toThrow(args); + }); +}); + +describe("decoding", () => { + it("reads booleans, leaves NULL opt columns absent and parses json cells", () => { + const row = T.rows.decode({ id: "a", n: 1, x: 2, b: 0, note: null, meta: '{"a":1}', meta$v: 2 }, context); + expect(row).toEqual({ id: "a", n: 1, x: 2, b: false, meta: { a: 1 } }); + }); + + it("refuses a json cell of a newer version", () => { + expect(() => T.rows.decode({ id: "a", n: 1, x: 2, b: 0, note: null, meta: "{}", meta$v: 3 }, context)).toThrow( + "newer than 2", + ); + }); +}); diff --git a/packages/chord/test/next/store/sql.test.ts b/packages/chord/test/next/store/sql.test.ts new file mode 100644 index 000000000..38b0c8623 --- /dev/null +++ b/packages/chord/test/next/store/sql.test.ts @@ -0,0 +1,106 @@ +// SQL text (slice 6a §8): quoted names, the DDL of tables and indexes, and the statements of a table. + +import { describe, expect, it } from "vitest"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { + createIndexSql, + createTableSql, + dropIndexSql, + quote, + TableSql, + tableSqlName, +} from "../../../src/next/store/sql.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const T = defineTable("edits", { + version: 1, + columns: { + id: str(), + at: int({ default: 0 }), + x: num(), + b: bool(), + note: opt(str()), + meta: json(obj({}), { version: 2 }), + }, + key: ["id"], +}); + +describe("names", () => { + it("renders ::, quoted", () => { + expect(tableSqlName("@pi/file-history.store", "edits")).toBe("@pi/file-history.store:edits"); + expect(quote("session:a:b")).toBe('"session:a:b"'); + }); +}); + +describe("DDL", () => { + it("renders a STRICT WITHOUT ROWID table with checks, defaults, hidden version columns and the key", () => { + expect(createTableSql("s:o:edits", T.record, T.key)).toBe( + 'CREATE TABLE "s:o:edits" ("id" TEXT NOT NULL, "at" INTEGER NOT NULL DEFAULT 0 CHECK ("at" BETWEEN -9007199254740991 AND 9007199254740991), ' + + '"x" REAL NOT NULL, "b" INTEGER NOT NULL CHECK ("b" IN (0, 1)), "note" TEXT, "meta" TEXT NOT NULL, "meta$v" INTEGER NOT NULL, ' + + '"$v" INTEGER NOT NULL, PRIMARY KEY ("id")) STRICT, WITHOUT ROWID', + ); + expect(createIndexSql("s:o:edits", "by_at", ["at"], ["id"])).toBe( + 'CREATE INDEX "s:o:edits:by_at" ON "s:o:edits" ("at", "id")', + ); + expect(createIndexSql("s:o:edits", "by_id", ["id", "at"], ["id"])).toBe( + 'CREATE INDEX "s:o:edits:by_id" ON "s:o:edits" ("id", "at")', + ); + expect(dropIndexSql("s:o:edits", "by_at")).toBe('DROP INDEX "s:o:edits:by_at"'); + }); + + it("keeps int columns within ±(2^53 - 1) and bool columns within 0 and 1 (rule 10)", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + await db.run(createTableSql("t", T.record, T.key)); + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + const params = T.rows.put({ id: "a", x: 1, b: true, meta: {} }, 1).params; + await db.run(sql.put, ...params); + await expect(db.run(`UPDATE t SET at = ?`, 2 ** 53)).rejects.toThrow("CHECK"); + await expect(db.run(`UPDATE t SET b = 2`)).rejects.toThrow("CHECK"); + await expect(db.run(`UPDATE t SET at = 'x'`)).rejects.toThrow(); + await db.run(`INSERT INTO t (id, x, b, meta, meta$v, "$v") VALUES ('d', 1, 0, '{}', 2, 1)`); + expect(await db.get(`SELECT at FROM t WHERE id = 'd'`)).toEqual({ at: 0 }); + await db.close(); + }); +}); + +describe("statements", () => { + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + + it("renders the fixed statements once", () => { + expect(sql.put).toBe( + 'INSERT OR REPLACE INTO "t" ("id", "at", "x", "b", "note", "meta", "meta$v", "$v") VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + ); + expect(sql.get).toBe('SELECT "id", "at", "x", "b", "note", "meta", "meta$v" FROM "t" WHERE "id" = ?'); + expect(sql.delete).toBe('DELETE FROM "t" WHERE "id" = ?'); + }); + + it("stamps a json column's version with its value, NULL with a clear", () => { + const versions = new Map([["meta", 2]]); + expect(sql.update({ meta: { json: { a: 1 } } }, ["k"], versions)).toEqual({ + sql: 'UPDATE "t" SET "meta" = ?, "meta$v" = ? WHERE "id" = ?', + params: ['{"a":1}', 2, "k"], + }); + expect(sql.update({ meta: null }, ["k"], versions).params).toEqual([null, "k"]); + }); + + it("binds every value: one text per query shape, whatever the values (rule 18)", () => { + const order = { columns: ["at", "id"], direction: "desc" as const }; + const a = sql.select({ inArray: ["id", ["a", "b"]] }, order, [1, "x"], 10); + const b = sql.select({ inArray: ["id", Array.from({ length: 40_000 }, (_, i) => `k${i}`)] }, order, [2, "y"], 99); + expect(a.sql).toBe(b.sql); + expect(a.sql).toBe( + 'SELECT "id", "at", "x", "b", "note", "meta", "meta$v" FROM "t" WHERE "id" IN (SELECT value FROM json_each(?)) AND ("at", "id") < (?, ?) ORDER BY "at" DESC, "id" DESC LIMIT ?', + ); + expect(a.params).toEqual(['["a","b"]', 1, "x", 10]); + expect(sql.count({ prefix: ["id", "\u{10FFFF}"] })).toEqual({ + sql: 'SELECT count(*) AS count FROM "t" WHERE "id" >= ?', + params: ["\u{10FFFF}"], + }); + expect(sql.deleteWhere({ prefix: ["id", "a"] })).toEqual({ + sql: 'DELETE FROM "t" WHERE ("id" >= ? AND "id" < ?)', + params: ["a", "b"], + }); + }); +}); diff --git a/packages/chord/test/next/store/store.test.ts b/packages/chord/test/next/store/store.test.ts new file mode 100644 index 000000000..335cbfd91 --- /dev/null +++ b/packages/chord/test/next/store/store.test.ts @@ -0,0 +1,579 @@ +// The store (slice 6a §2, §3, slice 6b §1): opening, the commit step, refusals and unknown outcomes (D4), the id +// allocator (D3), one host per store (rule 7), close. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument, findDocument, retireDocument } from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import type { StagedWrite, Step } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore, type Store } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { stageTable } from "../../../src/next/store/table-writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { exported, faultyStore, holdNextCommit, metered, rejection } from "../support.ts"; + +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; +const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; + +/** A table a series creates. */ +const Edits = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); +const TABLE = "app.store:edits"; + +/** A store on a fresh `:memory:` database, as Chord's layers see it. */ +async function memoryStore(options?: { readonly outbox?: boolean }): Promise { + return (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, options)) as ChordStore; +} + +/** A fence that admits and records what it adopted. */ +function recording(adopted: [number, readonly number[]][] = []) { + return { + adopted, + fence: { admit: () => true, adopt: (seq: number, ids: readonly number[]) => void adopted.push([seq, ids]) }, + }; +} + +/** Creates the singleton at `ADDRESS` and resolves with its id. */ +async function created(store: ChordStore): Promise { + const { adopted, fence } = recording(); + await store.commit([createDocument(ADDRESS, BASE, JSON.stringify(BASE.value))], fence, context); + return adopted[0]![1][0]!; +} + +/** A staged write whose step fails in the backend: its own code throws, a refusal. */ +const BROKEN: StagedWrite = { + order: "write", + apply: (step) => + step.run("INSERT INTO no_such_table VALUES (1)").then(() => [{ kind: "document.retire", id: 0, revision: 0 }]), +}; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +describe("opening", () => { + it("creates a new store at seq 0 without an outbox, and reopens a file at its last seq and ids", async () => { + directory = await mkdtemp(join(tmpdir(), "chord-store-")); + const path = join(directory, "store.sqlite"); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect([store.lastSeq, store.outbox]).toEqual([0, undefined]); + expect(await created(store)).toBe(2); + await store.close(context); + const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(reopened.lastSeq).toBe(1); + const { adopted, fence } = recording(); + await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [3]]]); + await reopened.close(context); + }); + + it("refuses a database that is not UTF-8, closing the backend (rule 12)", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + await db.exec('PRAGMA encoding = "UTF-16le"'); + await expect(openStore(sqliteBackend(db), context)).rejects.toThrow("not UTF-8"); + await expect(db.get("SELECT 1")).rejects.toThrow(); + }); +}); + +describe("the commit step", () => { + it("assigns one seq per commit, applies retirements before creations, and adopts before the next step", async () => { + const store = await memoryStore(); + const id = await created(store); + const events: string[] = []; + const fence = (name: string) => ({ + admit: () => true, + adopt: (seq: number, ids: readonly number[]) => void events.push(`${name} ${seq} ${ids}`), + }); + const first = store.commit( + [createDocument(ADDRESS, BASE, "{}"), retireDocument(id, 1)], + fence("recreate"), + context, + ); + const second = store.commit( + [changeDocument(3, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], + fence("change"), + context, + ); + expect(await Promise.all([first, second])).toEqual([2, 3]); + expect(events).toEqual(["recreate 2 3", "change 3 "]); + expect(store.lastSeq).toBe(3); + }); + + it("rejects a commit its fence refuses at the entry released, writing nothing; an empty commit takes no seq", async () => { + const store = await memoryStore(); + const refused = store.commit( + [createDocument(ADDRESS, BASE, "{}")], + { admit: () => false, adopt: () => {} }, + context, + ); + expect(await rejection(refused)).toMatchObject({ code: "released" }); + const { adopted, fence } = recording(); + expect(await store.commit([], fence, context)).toBe(0); + expect(adopted).toEqual([[0, []]]); + expect(await created(store)).toBe(2); + }); + + it("numbers up to Number.MAX_SAFE_INTEGER after an import near it, then fails storage_failed: no seq repeats", async () => { + const [manifest, ...rest] = await exported(await memoryStore()); + const parts = [{ ...manifest!, lastSeq: Number.MAX_SAFE_INTEGER - 1 }, ...rest]; + const store = (await importStore( + sqliteBackend(await openNodeSqliteDatabase(":memory:")), + parts, + context, + )) as ChordStore; + expect(await created(store)).toBe(2); + expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); + const refused = store.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], recording().fence, context); + expect(await rejection(refused)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); + expect(await rejection(store.read(async () => null, context))).toMatchObject({ reason: "storage_failed" }); + }); + + it("counts one commit and three statements for a delta, four with an outbox (cost guard)", async () => { + for (const [outbox, statements] of [ + [false, 3], + [true, 4], + ] as const) { + const store = await memoryStore({ outbox }); + const id = await created(store); + const telemetry = await metered(); + const ops = [["s", ["n"], 1]] as const; + await store.commit( + [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + recording().fence, + telemetry.context, + ); + expect(telemetry.counts()).toEqual({ "chord.store.commits": 1, "chord.store.statements": statements }); + } + }); +}); + +describe("refusals and unknown outcomes (D4)", () => { + it("a step whose own code throws is refused: the commit rejects with that error, nothing is adopted, the store goes on", async () => { + const store = await memoryStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + const { adopted, fence } = recording(); + const error = await rejection(store.commit([BROKEN], fence, context)); + expect(String(error)).toContain("no such table"); + expect(isChordError(error)).toBe(false); + expect([adopted, told, store.lastSeq]).toEqual([[], [], 0]); + expect(await created(store)).toBe(2); + expect(store.lastSeq).toBe(1); + }); + + it.each([ + ["BEGIN IMMEDIATE", undefined], + ["COMMIT", undefined], + ["ROLLBACK", BROKEN], + ] as const)( + "a failed %s is an unknown outcome: the store stops storage_failed, its host told once", + async (at, write) => { + const { store, fail } = await faultyStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + fail(at); + const writes = write === undefined ? [createDocument(ADDRESS, BASE, "{}")] : [write]; + expect(await rejection(store.commit(writes, recording().fence, context))).toMatchObject({ + code: "unavailable", + reason: "storage_failed", + }); + expect( + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)), + ).toMatchObject({ + reason: "storage_failed", + }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + expect(told.length).toBe(1); + await store.close(context); + await store.close(context); + }, + ); + + it("an adopt that throws stops the store, an empty commit's too; a conditional update on another revision is refused", async () => { + const store = await memoryStore(); + const id = await created(store); + const throwing = { + admit: () => true, + adopt: () => { + throw new Error("adopt threw"); + }, + }; + expect(await rejection(store.commit([retireDocument(id, 1)], throwing, context))).toMatchObject({ + reason: "storage_failed", + }); + const empty = await memoryStore(); + expect(await rejection(empty.commit([], throwing, context))).toMatchObject({ reason: "storage_failed" }); + expect(await rejection(empty.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + const other = await memoryStore(); + const otherId = await created(other); + const stale = changeDocument(otherId, 7, { kind: "delta", version: 1, ops: [] }, "[]"); + expect(String(await rejection(other.commit([stale], recording().fence, context)))).toContain( + "not live at revision 7", + ); + expect(other.lastSeq).toBe(1); + expect(await other.read(async () => 1, context)).toBe(1); + }); + + it("spare a read whose context was aborted (aborted), and reject a read that settles after the stop", async () => { + const { store, fail } = await faultyStore(); + const cancel = withCancel(context); + cancel.cancel(); + const failing = store.read(() => Promise.reject(new Error("backend")), cancel.context); + expect(await rejection(failing)).toMatchObject({ code: "aborted" }); + const gate = Promise.withResolvers(); + const late = store.read(() => gate.promise, context); + fail("COMMIT"); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)); + gate.resolve(1); + expect(await rejection(late)).toMatchObject({ reason: "storage_failed" }); + }); + + it("runs a read transaction's queries on one state, guarded as a read", async () => { + const store = await memoryStore(); + await created(store); + const counts = await store.readTransaction( + async (executor) => [ + await executor.get("SELECT count(*) AS n FROM chord_documents"), + await executor.get("SELECT 1 AS one"), + ], + context, + ); + expect(counts).toEqual([{ n: 1 }, { one: 1 }]); + expect(await rejection(store.readTransaction(() => Promise.reject(new Error("backend")), context))).toMatchObject( + { + reason: "storage_failed", + }, + ); + }); +}); + +describe("the id allocator (D3)", () => { + /** A staged write raising the allocator above `id`, as an adapter's explicit id does. */ + const raising = (id: number): StagedWrite => ({ + order: "write", + apply: async (step) => { + step.ids.raise(id); + return []; + }, + }); + + it("starts at 2, takes a creation's id as it applies, and gives a mint during a running step the next one", async () => { + const store = await memoryStore({ outbox: true }); + const held = holdNextCommit(store); + const commit = store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + await held.entered; + expect(store.mintId()).toBe(3); + held.release(); + await commit; + expect(store.mintId()).toBe(4); + const [first] = await store.outbox!.read(0, 10, context); + expect([first!.nextId, (first!.writes[0] as { record: { id: number } }).record.id]).toEqual([4, 2]); + expect(store.nextId).toBe(4); + }); + + it("leaves a refused step's ids unused and keeps the raised value", async () => { + const store = await memoryStore(); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}"), BROKEN], recording().fence, context)); + expect(await created(store)).toBe(3); + }); + + it("saturates at Number.MAX_SAFE_INTEGER across reopen: mints and Chord creations are refused, the store goes on", async () => { + directory = await mkdtemp(join(tmpdir(), "chord-store-")); + const path = join(directory, "store.sqlite"); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + await store.commit([raising(Number.MAX_SAFE_INTEGER)], recording().fence, context); + expect(store.nextId).toBe(Number.MAX_SAFE_INTEGER); + await store.close(context); + const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(reopened.nextId).toBe(Number.MAX_SAFE_INTEGER); + expect(() => reopened.mintId()).toThrow("ID space is exhausted"); + const refused = reopened.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + expect(String(await rejection(refused))).toContain("ID space is exhausted"); + expect(await reopened.read(async () => 1, context)).toBe(1); + await reopened.close(context); + }); +}); + +describe("one host per store (rule 7)", () => { + it("refuses a second attachment until the first detaches", async () => { + const store = await memoryStore(); + const attachment = { failed: () => {}, batch: new Batch(() => {}, undefined) }; + store.attach(attachment); + expect(store.batch).toBe(attachment.batch); + expect(() => store.attach(attachment)).toThrow("already attached"); + store.detach(); + expect(store.batch).toBeUndefined(); + store.attach(attachment); + }); +}); + +describe("close", () => { + it("waits for the running step, rejects queued commits and later reads released, and closes the backend", async () => { + const store = await memoryStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const create = createDocument(ADDRESS, BASE, "{}"); + const held: StagedWrite = { + order: "create", + apply: async (step) => { + entered.resolve(); + await gate.promise; + return create.apply(step); + }, + }; + const running = store.commit([held], recording().fence, context); + await entered.promise; + const queued = store.commit([createDocument({ ...ADDRESS, key: "q" }, BASE, "{}")], recording().fence, context); + const closed = (store as Store).close(context); + gate.resolve(); + expect(await running).toBe(1); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + await closed; + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ code: "released" }); + }); +}); + +describe("a commit series", () => { + /** A series commit with no check of its own. */ + const unchecked = (writes: readonly StagedWrite[]) => ({ check: async () => {}, writes }); + + it("stores its commits in one transaction, one seq each, each checked against the ones before it", async () => { + const store = await memoryStore({ outbox: true }); + const seen: (number | undefined)[] = []; + const receipts: [number, number, readonly number[]][] = []; + const telemetry = await metered(); + const result = await store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + // No writes: no seq, it stands at the commit before it. + unchecked([]), + { + // The check reads through its step: it sees the first commit's creation. + check: async (step) => void seen.push((await findDocument(step, ADDRESS))?.id), + writes: [createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], + }, + ], + (index, seq, ids) => receipts.push([index, seq, ids]), + telemetry.context, + ); + expect(result).toEqual({ stored: 3 }); + expect(seen).toEqual([2]); + expect(receipts).toEqual([ + [0, 1, [2]], + [1, 1, []], + [2, 2, [3]], + ]); + expect(store.lastSeq).toBe(2); + expect(telemetry.counts()["chord.store.commits"]).toBe(2); + const outbox = await store.read( + (executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox"), + context, + ); + expect(outbox.map((row) => row.seq)).toEqual([1, 2]); + }); + + it("stops at a check that refuses: the commits before it are stored, it and the ones after are not", async () => { + const store = await memoryStore(); + const refusal = new Error("refused"); + const receipts: number[] = []; + const result = await store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + { + check: () => Promise.reject(refusal), + writes: [createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], + }, + unchecked([createDocument({ ...ADDRESS, key: "c" }, BASE, "{}")]), + ], + (index) => receipts.push(index), + context, + ); + expect(result).toEqual({ stored: 1, refused: refusal }); + expect(receipts).toEqual([0]); + expect(store.lastSeq).toBe(1); + const keys = await store.read( + (executor) => executor.all<{ key: string }>("SELECT key FROM chord_documents ORDER BY id"), + context, + ); + expect(keys.map((row) => row.key)).toEqual([""]); + // The store goes on at the next seq. + const { adopted, fence } = recording(); + await store.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [3]]]); + }); + + it("rolls the whole series back when a check's statement fails, even if the check throws its own error", async () => { + const store = await memoryStore(); + const own = new Error("own"); + const series = store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + { + check: async (step) => { + await step.get("SELECT * FROM no_such_table").catch(() => {}); + throw own; + }, + writes: [], + }, + ], + () => {}, + context, + ); + expect(await rejection(series)).toBe(own); + expect(store.lastSeq).toBe(0); + // A rollback, not a stop: the store goes on; the id the rolled back creation took is not given back (D3). + expect(await created(store)).toBe(3); + }); + + it("rolls the whole series back when a commit's writes throw, the store going on", async () => { + const store = await memoryStore(); + const receipts: number[] = []; + const error = await rejection( + store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")]), unchecked([BROKEN])], + (index) => receipts.push(index), + context, + ), + ); + expect(String(error)).toContain("no such table"); + expect([receipts, store.lastSeq]).toEqual([[], 0]); + expect(await created(store)).toBe(3); + }); + + it("writes the metadata once per series: three statements per delta with its outbox commit, plus one (cost guard)", async () => { + const store = await memoryStore({ outbox: true }); + const id = await created(store); + const telemetry = await metered(); + const delta = (n: number) => { + const ops = [["s", ["n"], n]] as const; + return unchecked([changeDocument(id, n, { kind: "delta", version: 1, ops }, JSON.stringify(ops))]); + }; + await store.commitSeries([delta(1), delta(2), delta(3)], () => {}, telemetry.context); + // A delta committed alone is four statements with an outbox, the metadata among them; a series writes it once. + expect(telemetry.counts()).toEqual({ "chord.store.commits": 3, "chord.store.statements": 3 * 3 + 1 }); + const meta = await store.read( + (executor) => executor.get<{ last_seq: number }>("SELECT last_seq FROM chord_meta"), + context, + ); + expect(meta?.last_seq).toBe(4); + }); + + it.each([ + ["BEGIN IMMEDIATE", undefined], + ["COMMIT", undefined], + ["ROLLBACK", BROKEN], + ] as const)( + "a failed %s is an unknown outcome: nothing is received, the store stops storage_failed", + async (at, write) => { + const { store, fail } = await faultyStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + fail(at); + const receipts: number[] = []; + const second = write ?? createDocument({ ...ADDRESS, key: "b" }, BASE, "{}"); + const series = store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")]), unchecked([second])], + (index) => receipts.push(index), + context, + ); + expect(await rejection(series)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect([receipts, store.lastSeq, told.length]).toEqual([[], 0, 1]); + const after = store.commitSeries([unchecked([createDocument(ADDRESS, BASE, "{}")])], () => {}, context); + expect(await rejection(after)).toMatchObject({ reason: "storage_failed" }); + }, + ); + + it("lets a later commit see the table records an earlier one changed; a rollback leaves the catalog as it was", async () => { + const store = await memoryStore(); + const { version, record: columns, key, fingerprint } = Edits; + const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); + await rejection(store.commitSeries([unchecked([table]), unchecked([BROKEN])], () => {}, context)); + expect(store.table(TABLE)).toBeUndefined(); + const seen: (number | undefined)[] = []; + const sees: StagedWrite = { + order: "write", + apply: async (step) => { + seen.push(step.table(TABLE)?.version); + return []; + }, + }; + const later = { check: async (step: Step) => void seen.push(step.table(TABLE)?.version), writes: [sees] }; + expect(await store.commitSeries([unchecked([table]), later], () => {}, context)).toEqual({ stored: 2 }); + expect(seen).toEqual([1, 1]); + expect(store.table(TABLE)?.version).toBe(1); + }); + + it("runs a commit queued behind a series after the series' receipts", async () => { + const store = await memoryStore(); + const events: string[] = []; + const series = store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + unchecked([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")]), + ], + (index, seq) => events.push(`receive ${index} ${seq}`), + context, + ); + const fence = { + admit: () => { + events.push("admit"); + return true; + }, + adopt: (seq: number) => void events.push(`adopt ${seq}`), + }; + const queued = store.commit([createDocument({ ...ADDRESS, key: "c" }, BASE, "{}")], fence, context); + await Promise.all([series, queued]); + expect(events).toEqual(["receive 0 1", "receive 1 2", "admit", "adopt 3"]); + }); + + it("counts only commits with writes against the seq bound: empty ones at Number.MAX_SAFE_INTEGER still stand", async () => { + const [manifest, ...rest] = await exported(await memoryStore()); + const parts = [{ ...manifest!, lastSeq: Number.MAX_SAFE_INTEGER - 1 }, ...rest]; + const store = (await importStore( + sqliteBackend(await openNodeSqliteDatabase(":memory:")), + parts, + context, + )) as ChordStore; + await created(store); + const receipts: [number, number][] = []; + const empty = await store.commitSeries( + [unchecked([]), unchecked([])], + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(empty).toEqual({ stored: 2 }); + expect(receipts).toEqual([ + [0, Number.MAX_SAFE_INTEGER], + [1, Number.MAX_SAFE_INTEGER], + ]); + const writing = store.commitSeries( + [unchecked([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")])], + () => {}, + context, + ); + expect(await rejection(writing)).toMatchObject({ reason: "storage_failed" }); + }); + + it("stops the store when a receipt throws", async () => { + const store = await memoryStore(); + const series = store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")])], + () => { + throw new Error("receipt threw"); + }, + context, + ); + expect(await rejection(series)).toMatchObject({ reason: "storage_failed" }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + }); +}); diff --git a/packages/chord/test/next/store/table-changes.test.ts b/packages/chord/test/next/store/table-changes.test.ts new file mode 100644 index 000000000..8f3e75047 --- /dev/null +++ b/packages/chord/test/next/store/table-changes.test.ts @@ -0,0 +1,84 @@ +// A transaction's table writes (slice 6a §6, §7): staging, and the `tx.get` overlay over committed rows. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, opt, str } from "../../../src/next/contracts/index.ts"; +import { add, eq, prefix } from "../../../src/next/store/expression.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { memoryStore, rejection, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), path: str(), hits: int({ default: 0 }), note: opt(str()) }, + key: ["id"], +}); +const FENCE = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** Changes to a table holding `e1` (committed); the staged list they append to. */ +async function seeded(): Promise<{ readonly changes: TableChanges; readonly staged: StagedWrite[] }> { + store = await memoryStore(); + const edits = await tableIn(store, Edits); + const seed: StagedWrite[] = []; + new TableChanges(edits, seed).put({ id: "e1", path: "a/1", hits: 1 }); + await store.commit(seed, FENCE, context); + const staged: StagedWrite[] = []; + return { changes: new TableChanges(edits, staged), staged }; +} + +describe("staging", () => { + it("checks at the call, appends in order, and touches no backend before the commit", async () => { + const { changes, staged } = await seeded(); + changes.put({ id: "e2", path: "a/2" }); + expect(() => changes.put({ id: "e3" })).toThrow('column "path" is missing'); + expect(() => changes.update({ id: "e1" }, { id: "x" })).toThrow("cannot change"); + changes.update({ id: "e1" }, { note: "n" }); + changes.delete({ id: "e9" }); + expect(staged).toHaveLength(3); + expect( + await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "app.store:edits"`), context), + ).toEqual({ n: 1 }); + }); +}); + +describe("the tx.get overlay", () => { + it("replays the transaction's own put, update and delete over the committed row", async () => { + const { changes } = await seeded(); + changes.update({ id: "e1" }, (e: { hits: never }) => ({ hits: add(e.hits, 2) })); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "a/1", hits: 3 }); + changes.update({ id: "e1" }, { note: "n" }); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "a/1", hits: 3, note: "n" }); + changes.delete({ id: "e1" }); + expect(await changes.get({ id: "e1" }, context)).toBeUndefined(); + changes.put({ id: "e1", path: "b" }); + changes.update({ id: "e1" }, (e: { hits: never }) => ({ hits: add(e.hits, 1) })); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "b", hits: 1 }); + }); + + it("an update of an absent key changes nothing", async () => { + const { changes } = await seeded(); + changes.update({ id: "e9" }, { note: "n" }); + expect(await changes.get({ id: "e9" }, context)).toBeUndefined(); + }); + + it("is refused on a table after an updateWhere or deleteWhere (W10); other tables are unaffected", async () => { + const { changes } = await seeded(); + expect(await changes.get({ id: "e1" }, context)).toBeDefined(); + changes.updateWhere((e: { path: never }) => prefix(e.path, "a/"), { note: "bulk" }); + expect(await rejection(changes.get({ id: "e1" }, context))).toMatchObject({ + code: "invalid", + reason: "read_after_bulk", + }); + const other = new TableChanges(changes.handle, []); + other.deleteWhere((e: { id: never }) => eq(e.id, "x")); + expect(await rejection(other.get({ id: "e1" }, context))).toMatchObject({ reason: "read_after_bulk" }); + expect(await new TableChanges(changes.handle, []).get({ id: "e1" }, context)).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/store/table-evolve.test.ts b/packages/chord/test/next/store/table-evolve.test.ts new file mode 100644 index 000000000..8118de4bd --- /dev/null +++ b/packages/chord/test/next/store/table-evolve.test.ts @@ -0,0 +1,225 @@ +// Tests of `evolveTable` (slice 6a §9): versions appended and materialized at declaration, the edits recorded before +// and after the data step, their checks, the editor rule shared with contracts, and the types along the chain. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import type { RowOf } from "../../../src/next/store/columns.ts"; +import { add, eq } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; + +const EditsV1 = defineTable("edits", { + version: 1, + columns: { id: str(), file: str(), at: int(), meta: opt(json(obj({ a: int() }), { version: 1 })) }, + key: ["id"], + indexes: { by_file: ["file", "at"] }, +}); + +describe("versions", () => { + it("each evolveTable appends version + 1, materialized and fingerprinted, linked to the one before", () => { + const V2 = evolveTable(EditsV1, (t) => t.add("dir", str({ default: "" }))); + const V3 = evolveTable(V2, (t) => t.index("by_dir", ["dir"])); + expect([V2.version, V3.version]).toEqual([2, 3]); + expect(V3.evolution?.previous).toBe(V2); + expect(V2.evolution?.previous).toBe(EditsV1); + expect(new Set([EditsV1.fingerprint, V2.fingerprint, V3.fingerprint]).size).toBe(3); + expect(V3.indexes).toEqual( + new Map([ + ["by_file", ["file", "at"]], + ["by_dir", ["dir"]], + ]), + ); + expectTypeOf(V3.version).toEqualTypeOf<3>(); + }); + + it("the edit must return the editor it was given (contracts' rule)", () => { + expect(() => evolveTable(EditsV1, () => ({}) as never)).toThrow( + "edits: evolveTable must return the editor it was given", + ); + }); +}); + +describe("edits", () => { + it("are recorded before and after the data step; the step reads the columns as they stand there", () => { + const V2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path })) + .drop("at") + .index("by_dir", ["dir"]), + ); + const migration = V2.evolution!.migration; + expect(migration.before).toEqual([ + { kind: "rename", from: "file", to: "path" }, + { kind: "add", name: "dir", column: { type: "str", default: "" } }, + ]); + expect(migration.after).toEqual([ + { kind: "dropIndex", name: "by_file" }, + { kind: "drop", name: "at" }, + { kind: "index", name: "by_dir", columns: ["dir"] }, + ]); + const step = migration.step!; + expect(step.kind).toBe("rows"); + expect(step.table.columns.map(({ name }) => name)).toEqual(["id", "path", "at", "meta", "dir"]); + expect(V2.columns.map(({ name }) => name)).toEqual(["id", "path", "meta", "dir"]); + expect(V2.indexes).toEqual(new Map([["by_dir", ["dir"]]])); + }); + + it("a rename carries the key and the indexes", () => { + const V2 = evolveTable(EditsV1, (t) => t.rename("id", "uid").rename("file", "path")); + expect(V2.key).toEqual(["uid"]); + expect(V2.indexes).toEqual(new Map([["by_file", ["path", "at"]]])); + expectTypeOf ? C : never>>().toEqualTypeOf< + { uid: string; path: string; at: number } & { meta?: { a: number } } + >(); + }); + + it("an update step is compiled at declaration against the columns there", () => { + const V2 = evolveTable(EditsV1, (t) => + t.update({ where: (e) => eq(e.file, "x"), set: (e) => ({ at: add(e.at, 1) }) }), + ); + expect(V2.evolution!.migration.step).toMatchObject({ + kind: "update", + where: { eq: ["file", "x"] }, + set: { at: { add: [{ col: "at" }, 1] } }, + }); + expect(() => + evolveTable(EditsV1, (t) => + // @ts-expect-error: key columns cannot change (rule 13) + t.update({ set: { id: "x" } }), + ), + ).toThrow('key column "id" cannot change'); + }); + + it("a json edit changes the column's version and keeps its nullability", () => { + const V2 = evolveTable(EditsV1, (t) => + t.json( + "meta", + json(obj({ a: int(), b: int() }), { version: 2, migrate: (old) => ({ ...(old as { a: number }), b: 0 }) }), + ), + ); + expect(V2.byName.get("meta")).toMatchObject({ type: "json", optional: true }); + expect(V2.evolution!.migration.before).toEqual([ + { kind: "json", name: "meta", column: expect.objectContaining({ type: "json", optional: true, version: 2 }) }, + ]); + expectTypeOf ? C : never>>().toEqualTypeOf< + { id: string; file: string; at: number } & { meta?: { a: number; b: number } } + >(); + }); +}); + +describe("checks", () => { + it("a rename may change only the case of its own column", () => { + expect(evolveTable(EditsV1, (t) => t.rename("at", "AT")).byName.has("AT")).toBe(true); + }); + + it.each<[string, () => unknown, string]>([ + ["rename to an existing column", () => evolveTable(EditsV1, (t) => t.rename("file", "at")), 'column "at" exists'], + [ + "rename an unknown column", + () => evolveTable(EditsV1, (t) => t.rename("nope" as "file", "x")), + 'no column "nope"', + ], + [ + "add a column without opt(...) or a default", + // @ts-expect-error: an added column is opt(...) or has a default + () => evolveTable(EditsV1, (t) => t.add("dir", str())), + "needs opt(...) or a default", + ], + [ + "drop a key column", + // @ts-expect-error: a key column cannot be dropped + () => evolveTable(EditsV1, (t) => t.drop("id")), + '"id" is a key column', + ], + [ + "rename after a drop", + // @ts-expect-error: rename comes before drop, index and dropIndex + () => evolveTable(EditsV1, (t) => t.drop("at").rename("file", "path")), + "rename comes before the data step", + ], + [ + "a second data step", + // @ts-expect-error: at most one data step + () => evolveTable(EditsV1, (t) => t.rows(() => ({})).rows(() => ({}))), + "rows comes before the data step", + ], + ["an existing index", () => evolveTable(EditsV1, (t) => t.index("by_file", ["at"])), 'index "by_file" exists'], + ["dropIndex of an unknown index", () => evolveTable(EditsV1, (t) => t.dropIndex("nope")), 'no index "nope"'], + [ + "an index over an opt(...) column", + // @ts-expect-error: index columns are non-opt str() or int() columns + () => evolveTable(EditsV1, (t) => t.index("by_meta", ["meta"])), + "is not a non-opt str() or int() column", + ], + [ + "a json edit of a scalar column", + // @ts-expect-error: only json(...) columns take a json edit + () => evolveTable(EditsV1, (t) => t.json("file", json(obj({}), { version: 2 }))), + "is not a json(...) column", + ], + [ + "a json edit changing the schema at the cell's version (D9)", + () => evolveTable(EditsV1, (t) => t.json("meta", json(obj({ b: str() }), { version: 1 }))), + "changes its schema at version 1", + ], + [ + "a json edit going back a version", + () => + evolveTable( + evolveTable(EditsV1, (t) => t.json("meta", json(obj({ a: int() }), { version: 2 }))), + (t) => t.json("meta", json(obj({ a: int() }), { version: 1 })), + ), + "goes back from version 2", + ], + [ + "an add colliding by case with a column a later drop removes", + () => evolveTable(EditsV1, (t) => t.add("File", str({ default: "" })).drop("file")), + 'column "file" exists', + ], + [ + "a rename colliding by case with another column", + () => evolveTable(EditsV1, (t) => t.rename("at", "FILE")), + 'column "file" exists', + ], + [ + "a name folding to a column's only beyond ASCII (SQLite folds ASCII): the name rule, not a collision", + () => { + const Kinds = defineTable("kinds", { version: 1, columns: { id: str(), kind: str() }, key: ["id"] }); + return evolveTable(Kinds, (t) => t.add("\u212Aind", opt(str()))); + }, + "expected [A-Za-z_][A-Za-z0-9_]*", + ], + [ + "an index on a column missing at that edit, though a later dropIndex removes the index", + // @ts-expect-error: index columns are the table's columns at that edit + () => evolveTable(EditsV1, (t) => t.index("bad", ["missing"]).dropIndex("bad")), + 'index "bad": no column "missing"', + ], + [ + "an index on a column dropped before it", + // @ts-expect-error: the dropped column is gone at this edit + () => evolveTable(EditsV1, (t) => t.drop("at").index("by_at", ["at"])), + 'index "by_at": no column "at"', + ], + [ + "a rename to a name breaking the column name rule, though a later rename fixes it", + () => evolveTable(EditsV1, (t) => t.rename("at", "bad\u0000name").rename("bad\u0000name", "when")), + 'rename: column "bad\u0000name": expected [A-Za-z_][A-Za-z0-9_]*', + ], + [ + "a rename to a name breaking the column name rule, though a later drop removes it", + () => evolveTable(EditsV1, (t) => t.rename("at", "1st").drop("1st")), + 'rename: column "1st": expected', + ], + [ + "an index name outside [a-z0-9_]+", + () => evolveTable(EditsV1, (t) => t.index("By", ["file"])), + "expected [a-z0-9_]+", + ], + ])("refuses %s at declaration", (_, declare, message) => { + expect(declare).toThrow(message); + }); +}); diff --git a/packages/chord/test/next/store/table-handle.test.ts b/packages/chord/test/next/store/table-handle.test.ts new file mode 100644 index 000000000..9d147ad83 --- /dev/null +++ b/packages/chord/test/next/store/table-handle.test.ts @@ -0,0 +1,167 @@ +// Table handles (slice 6a §7): reads outside a transaction, and cells the code cannot read. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { eq, gte, isNull, prefix } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import type { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { holdNextCommit, memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + done: bool({ default: false }), + note: opt(str()), + meta: json(obj({ a: int() }), { version: 1 }), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** A table with `count` rows e0..e(count-1): path `p/`, at `i`. */ +async function filled(count: number, owner = tableOwner()): Promise { + store = await memoryStore(); + const edits = await tableIn(store, Edits, owner); + const staged: StagedWrite[] = []; + const changes = new TableChanges(edits, staged); + for (let i = 0; i < count; i++) + changes.put({ id: `e${String(i).padStart(3, "0")}`, path: `p/${i % 3}`, at: i, meta: { a: i } }); + await store.commit(staged, { admit: () => true, adopt: () => {} }, context); + return edits; +} + +describe("reads outside a transaction", () => { + it("get takes a key object and reads committed rows as stored", async () => { + const edits = await filled(3); + expect(await edits.get({ id: "e001" }, context)).toEqual({ + id: "e001", + path: "p/1", + at: 1, + done: false, + meta: { a: 1 }, + }); + expect(await edits.get({ id: "nope" }, context)).toBeUndefined(); + expect(await rejection(edits.get({ id: "e001", path: "x" }, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("page fetches limit + 1 for next, and the cursor continues in order with the key tie-break", async () => { + const edits = await filled(10); + const q = query(Edits) + .where((e) => prefix(e.path, "p/")) + .orderBy((e) => e.path, "desc"); + const seen: string[] = []; + let after: string | undefined; + do { + const page = await edits.page(q, 3, context, after === undefined ? {} : { after }); + expect(page.items.length).toBeLessThanOrEqual(3); + seen.push(...page.items.map((row) => `${row.path}:${row.id}`)); + after = page.next; + } while (after !== undefined); + const all = (await edits.page(q, 100, context)).items.map((row) => `${row.path}:${row.id}`); + expect(seen).toEqual(all); + expect(all.slice(0, 4)).toEqual(["p/2:e008", "p/2:e005", "p/2:e002", "p/1:e007"]); + expect((await edits.page(q, 4, context)).next).toBeDefined(); + expect((await edits.page(query(Edits), 10, context)).next).toBeUndefined(); + }); + + it("rows reads every page; count counts", async () => { + const edits = await filled(250); + const ids: string[] = []; + for await (const row of edits.rows( + query(Edits).where((e) => gte(e.at, 10)), + context, + )) + ids.push(row.id as string); + expect(ids).toHaveLength(240); + expect(ids[0]).toBe("e010"); + expect( + await edits.count( + query(Edits).where((e) => eq(e.path, "p/0")), + context, + ), + ).toBe(84); + expect( + await edits.count( + query(Edits).where((e) => isNull(e.note)), + context, + ), + ).toBe(250); + }); + + it("refuses a query of another table, a bad limit and a foreign cursor before the backend", async () => { + const edits = await filled(1); + const Other = defineTable("other", { version: 1, columns: { id: str() }, key: ["id"] }); + expect(await rejection(edits.page(query(Other) as never, 1, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 0, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 1.5, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 1, context, { after: "x" }))).toMatchObject({ reason: "args" }); + }); + + it("reads go through the store's guard: refused once the store closed", async () => { + const edits = await filled(1); + await store.close(context); + expect(await rejection(edits.get({ id: "e000" }, context))).toMatchObject({ code: "released" }); + }); +}); + +describe("unreadable cells", () => { + it("a json cell of a newer version fails the owner and the read, the data intact", async () => { + const owner = tableOwner(); + const edits = await filled(1, owner); + await store.read((executor) => executor.run(`UPDATE "app.store:edits" SET "meta$v" = 2`), context); + const error = await rejection(edits.get({ id: "e000" }, context)); + expect(error).toMatchObject({ code: "unavailable" }); + expect(owner.failures).toHaveLength(1); + expect(await store.read((executor) => executor.get(`SELECT meta FROM "app.store:edits"`), context)).toEqual({ + meta: '{"a":0}', + }); + }); +}); + +describe("the version fence of old handles (R15, slice 7b §5)", () => { + it("refuses reads through a handle older than the stored table: released/version", async () => { + store = await memoryStore(); + const old = await tableIn(store, Edits); + const V2 = evolveTable(Edits, (t) => t.rename("note", "comment")); + await tableIn(store, V2); + const refused = { code: "released", reason: "version" }; + expect(await rejection(old.get({ id: "e0" }, context))).toMatchObject(refused); + expect(await rejection(old.count(query(Edits), context))).toMatchObject(refused); + expect(() => old.assertVersion()).toThrow("the store has version 2"); + }); +}); + +describe("the version fence against a migration in flight (fix round 1, Sol 8)", () => { + it("refuses an old handle's read made while the migration's commit runs, rather than issuing obsolete SQL", async () => { + const V1 = defineTable("race", { version: 1, columns: { id: str(), old: str() }, key: ["id"] }); + const V2 = evolveTable(V1, (t) => t.rename("old", "new")); + store = await memoryStore(); + const old = await tableIn(store, V1); + const held = holdNextCommit(store); + const changing = tableIn(store, V2); + await held.entered; + const reading = rejection(old.get({ id: "x" }, context)); + held.release(); + await changing; + expect(await reading).toMatchObject({ code: "released", reason: "version" }); + expect(() => store.assertUsable()).not.toThrow(); + }); +}); diff --git a/packages/chord/test/next/store/table-history.test.ts b/packages/chord/test/next/store/table-history.test.ts new file mode 100644 index 000000000..957e679f9 --- /dev/null +++ b/packages/chord/test/next/store/table-history.test.ts @@ -0,0 +1,73 @@ +// Tests of a table's history (slice 6a §9): what an activation does with the recorded version: create, use, migrate +// from there (resuming a `migrating` one), or refuse a version the code cannot reach. + +import { describe, expect, it } from "vitest"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import type { TableRecord } from "../../../src/next/store/catalog.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { planTable } from "../../../src/next/store/table-history.ts"; + +const V1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); +const V2 = evolveTable(V1, (t) => t.add("dir", str({ default: "" })).rows(() => ({}))); +const V3 = evolveTable(V2, (t) => t.index("by_at", ["at"])); + +/** The record of `definition` in `state`. */ +function recorded(definition: TableDefinition, state: "ready" | "migrating" = "ready"): TableRecord { + const { version, record: columns, key, fingerprint } = definition; + return { table: "t", version, state, columns, key, fingerprint, indexes: {} }; +} + +describe("plans", () => { + it("create a table the store does not have; use one recorded at the newest version", () => { + expect(planTable(V3, undefined, "app.store")).toEqual({ kind: "create" }); + expect(planTable(V3, recorded(V3), "app.store")).toEqual({ kind: "ready" }); + }); + + it("migrate through every later version, oldest first; resume a version recorded migrating", () => { + expect(planTable(V3, recorded(V1), "app.store")).toEqual({ kind: "migrate", versions: [V2, V3], first: "begin" }); + expect(planTable(V3, recorded(V2, "migrating"), "app.store")).toEqual({ + kind: "migrate", + versions: [V2, V3], + first: "resume", + }); + }); +}); + +describe("refusals", () => { + it.each<[string, TableDefinition, TableRecord, string]>([ + ["a newer version (old code)", V2, recorded(V3), "version 3, newer than the code's 2"], + [ + "a version older than a new base", + defineTable("edits", { version: 7, columns: { id: str() }, key: ["id"] }), + recorded(V3), + "older than the history's base 7", + ], + [ + "a history edited in place", + evolveTable(V1, (t) => t.add("dir", str({ default: "x" }))), + recorded(V2), + "recorded with another definition", + ], + [ + "a json(...) column with another schema at the same version (D9)", + defineTable("notes", { + version: 1, + columns: { id: str(), meta: json(obj({ b: int() }), { version: 1 }) }, + key: ["id"], + }), + recorded( + defineTable("notes", { + version: 1, + columns: { id: str(), meta: json(obj({ a: int() }), { version: 1 }) }, + key: ["id"], + }), + ), + "recorded with another definition", + ], + ["a migrating version without a data step", V3, recorded(V3, "migrating"), "has no data step to resume"], + ])("%s", (_, definition, record, message) => { + expect(() => planTable(definition, record, "app.store")).toThrow(message); + }); +}); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts new file mode 100644 index 000000000..e5938015b --- /dev/null +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -0,0 +1,344 @@ +// Tests of tables brought to their version at activation (slice 6a §9): fresh tables created at the newest version, +// eager migrations one version at a time in their commits ((A), chunks of 100, (Z)), `rows` and `update` steps, +// index edits, resumption at `migrating`, and the refusals that leave the data as it was. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, nul, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { lt } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { MIGRATION_CHUNK, prepareTables } from "../../../src/next/store/table-migration.ts"; +import { memoryStore, rejection, tableOwner } from "../support.ts"; + +const TABLE = "app.store:edits"; +const OPEN = { admit: () => true, adopt: () => {} }; + +const EditsV1 = defineTable("edits", { + version: 1, + columns: { id: str(), file: str(), at: int() }, + key: ["id"], + indexes: { by_file: ["file"] }, +}); +const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); +const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** Prepares `definition` as the edits table of `store`; a handle to it. */ +async function prepared(definition: TableDefinition): Promise { + await prepareTables([{ definition, table: TABLE }], store, OPEN, "app.store", context); + return new TableHandle(definition, TABLE, store, tableOwner()); +} + +/** A store whose edits table is at version 1 with `count` rows e000.. (file `d/f`, at `i`). */ +async function filled(count: number): Promise { + store = await memoryStore({ outbox: true }); + const edits = await prepared(EditsV1); + const staged: StagedWrite[] = []; + const changes = new TableChanges(edits, staged); + for (let i = 0; i < count; i++) changes.put({ id: id(i), file: `d${i % 3}/f${i}`, at: i }); + await store.commit(staged, OPEN, context); +} + +function id(i: number): string { + return `e${String(i).padStart(3, "0")}`; +} + +/** The kinds of the writes of every commit after `seq`, each kind with its count. */ +async function commitsAfter(seq: number): Promise { + const commits = await store.outbox!.read(seq, 100, context); + return commits.map(({ writes }) => { + const counts = new Map(); + for (const { kind } of writes) counts.set(kind, (counts.get(kind) ?? 0) + 1); + return [...counts].map(([kind, count]) => (count === 1 ? kind : `${kind} x${count}`)); + }); +} + +/** Every row of `handle`, in key order. */ +async function all(handle: TableHandle, definition: TableDefinition): Promise { + const rows: unknown[] = []; + for await (const row of handle.rows(query(definition), context)) rows.push(row); + return rows; +} + +describe("fresh tables", () => { + it("are created at the newest version, ready, with their indexes, in one commit (no replay)", async () => { + store = await memoryStore({ outbox: true }); + await prepared(EditsV3); + expect(store.table(TABLE)).toMatchObject({ + version: 3, + state: "ready", + fingerprint: EditsV3.fingerprint, + indexes: { by_file: ["path"], by_path: ["path", "at"] }, + }); + expect(await commitsAfter(0)).toEqual([["table", "index x2"]]); + }); + + it("a table at the newest version, ready, is used as recorded: nothing is written", async () => { + await filled(1); + const seq = store.lastSeq; + await prepared(EditsV1); + expect(store.lastSeq).toBe(seq); + }); +}); + +describe("eager migrations", () => { + it("one version at a time: (A), chunks of 100 in key order, (Z) with the later edits and ready", async () => { + await filled(250); + const seq = store.lastSeq; + const edits = await prepared(EditsV3); + expect(await commitsAfter(seq)).toEqual([ + ["alter x2", "table"], + ["update x100"], + ["update x100"], + ["update x50", "table"], + ["alter", "index", "table"], + ]); + expect(store.table(TABLE)).toMatchObject({ + version: 3, + state: "ready", + columns: { id: { type: "str" }, path: { type: "str" }, at: { type: "int" } }, + indexes: { by_file: ["path"], by_path: ["path", "at"] }, + }); + const rows = await all(edits, EditsV3); + expect(rows).toHaveLength(250); + expect(rows[4]).toEqual({ id: "e004", path: "d1/f4", at: 4 }); + }); + + it("the data step reads the columns as they stand at its version and fills the one it adds", async () => { + await filled(3); + const edits = await prepared(EditsV2); + expect(await all(edits, EditsV2)).toEqual([ + { id: "e000", path: "d0/f0", at: 0, dir: "d0" }, + { id: "e001", path: "d1/f1", at: 1, dir: "d1" }, + { id: "e002", path: "d2/f2", at: 2, dir: "d2" }, + ]); + }); + + it("the migration's rows carry the internal form: the patch and the version stamped as $v", async () => { + await filled(1); + const seq = store.lastSeq; + await prepared(EditsV2); + const [, chunk] = await store.outbox!.read(seq, 10, context); + expect(chunk!.writes).toEqual([ + { kind: "update", table: TABLE, key: { id: "e000" }, set: { dir: "d0" }, columns: {}, count: 1, version: 2 }, + expect.objectContaining({ kind: "table", version: 2, state: "ready" }), + ]); + }); + + it("an empty table still runs (A) and (Z); a version without a data step is one commit", async () => { + await filled(0); + const seq = store.lastSeq; + await prepared(EditsV3); + expect(await commitsAfter(seq)).toEqual([["alter x2", "table"], ["table"], ["alter", "index", "table"]]); + }); + + it("a dropped column's indexes are dropped first, in the same commit", async () => { + await filled(2); + const Dropped = evolveTable(EditsV1, (t) => t.drop("file")); + const seq = store.lastSeq; + const edits = await prepared(Dropped); + const [commit] = await store.outbox!.read(seq, 10, context); + expect(commit!.writes.slice(0, 2)).toEqual([ + { kind: "dropIndex", table: TABLE, name: "by_file" }, + { kind: "alter", table: TABLE, version: 2, edit: { kind: "drop", name: "file" } }, + ]); + expect(store.table(TABLE)).toMatchObject({ columns: { id: { type: "str" }, at: { type: "int" } }, indexes: {} }); + expect(await all(edits, Dropped)).toEqual([ + { id: "e000", at: 0 }, + { id: "e001", at: 1 }, + ]); + }); + + it("an update step is one updateWhere over the rows below its version, stamping it, with ready", async () => { + await filled(5); + const Bumped = evolveTable(EditsV1, (t) => t.update({ where: (e) => lt(e.at, 3), set: { file: "old" } })); + const seq = store.lastSeq; + const edits = await prepared(Bumped); + const commits = await store.outbox!.read(seq, 10, context); + expect(commits.map(({ writes }) => writes.map(({ kind }) => kind))).toEqual([ + ["table"], + ["updateWhere", "table"], + ]); + expect(commits[1]!.writes[0]).toMatchObject({ kind: "updateWhere", count: 3, version: 2, set: { file: "old" } }); + expect((await all(edits, Bumped)).map((row) => (row as { file: string }).file)).toEqual([ + "old", + "old", + "old", + "d0/f3", + "d1/f4", + ]); + }); + + it("a json(...) version change records the column's version: no DDL, cells migrate on access", async () => { + const Notes1 = defineTable("edits", { + version: 1, + columns: { id: str(), meta: opt(json(obj({ a: int() }), { version: 1 })) }, + key: ["id"], + }); + const Notes2 = evolveTable(Notes1, (t) => + t.json( + "meta", + json(obj({ a: int(), b: int() }), { version: 2, migrate: (old) => ({ ...(old as { a: number }), b: 0 }) }), + ), + ); + store = await memoryStore({ outbox: true }); + const v1 = await prepared(Notes1); + const staged: StagedWrite[] = []; + new TableChanges(v1, staged).put({ id: "a", meta: { a: 1 } }); + await store.commit(staged, OPEN, context); + const seq = store.lastSeq; + const v2 = await prepared(Notes2); + const [commit] = await store.outbox!.read(seq, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "alter", + table: TABLE, + version: 2, + edit: { kind: "json", name: "meta", column: expect.objectContaining({ type: "json", version: 2 }) }, + }, + expect.objectContaining({ kind: "table", version: 2, state: "ready" }), + ]); + expect(await v2.get({ id: "a" }, context)).toEqual({ id: "a", meta: { a: 1, b: 0 } }); + }); +}); + +/** + * Records the `setTimeout` calls the migration's own code makes (its stack passes through `table-migration.ts`); timers + * of the test runner or other work under load pass through unrecorded (fix round 2: the test watched every timer). + */ +function migrationTimers(): { readonly calls: string[]; restore(): void } { + const calls: string[] = []; + const original = globalThis.setTimeout; + const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((...args: Parameters) => { + const stack = new Error().stack ?? ""; + if (stack.includes("src/next/store/table-migration.ts")) calls.push(stack); + return original(...args); + }) as typeof setTimeout); + return { calls, restore: () => spy.mockRestore() }; +} + +describe("the chunk yield", () => { + it("yields between chunks without Node's setImmediate (Workers, browsers), through a message, not a timer", async () => { + await filled(3 * MIGRATION_CHUNK); + const saved = globalThis.setImmediate; + // Runtimes without it: Cloudflare Workers, browsers. + Reflect.deleteProperty(globalThis, "setImmediate"); + const timers = migrationTimers(); + try { + await prepared(EditsV2); + expect(timers.calls).toEqual([]); + } finally { + timers.restore(); + globalThis.setImmediate = saved; + } + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); + + it("yields between chunks without a timer (no 1 ms clamp), while timers due run in between (a latency guard)", async () => { + await filled(10 * MIGRATION_CHUNK); + let fired = false; + // Due during the migration: it fires between chunks (a microtask-only loop would starve it). + setTimeout(() => { + fired = true; + }, 0); + const timers = migrationTimers(); + try { + await prepared(EditsV2); + expect(timers.calls).toEqual([]); + } finally { + timers.restore(); + } + expect(fired).toBe(true); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); +}); + +describe("patches", () => { + it("a rows patch writes a JSON null into a json(...) cell and clears an opt column with undefined", async () => { + const Cells1 = defineTable("edits", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }), note: opt(str()) }, + key: ["id"], + }); + const Cells2 = evolveTable(Cells1, (t) => + t.add("more", opt(json(nul(), { version: 1 }))).rows(() => ({ more: null, note: undefined })), + ); + store = await memoryStore(); + const v1 = await prepared(Cells1); + const staged: StagedWrite[] = []; + new TableChanges(v1, staged).put({ id: "a", data: null, note: "n" }); + await store.commit(staged, OPEN, context); + const v2 = await prepared(Cells2); + expect(await v2.get({ id: "a" }, context)).toEqual({ id: "a", data: null, more: null }); + }); +}); + +describe("resumption", () => { + it("a data step that throws leaves the table migrating with its committed chunks; the next run continues", async () => { + await filled(250); + let fail = true; + const Failing = evolveTable(EditsV1, (t) => + t.rows((row) => { + if (fail && row.id === id(150)) throw new Error("crash"); + return { file: `${row.file}!` }; + }), + ); + expect(await rejection(prepared(Failing))).toMatchObject({ message: "crash" }); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + const seq = store.lastSeq; + fail = false; + const edits = await prepared(Failing); + expect(await commitsAfter(seq)).toEqual([["update x100"], ["update x50", "table"]]); + const files = (await all(edits, Failing)).map((row) => (row as { file: string }).file); + // Every row patched once: the first chunk's rows were skipped on the second run. + expect(files.every((file) => file.endsWith("!") && !file.endsWith("!!"))).toBe(true); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); + + it("a refused commit (the activation abandoned) stops the migration between chunks", async () => { + await filled(MIGRATION_CHUNK * 2 + 1); + let admitted = 2; + const fence = { admit: () => admitted-- > 0, adopt: () => {} }; + const migration = prepareTables([{ definition: EditsV2, table: TABLE }], store, fence, "app.store", context); + expect(await rejection(migration)).toMatchObject({ code: "released" }); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + }); +}); + +describe("refusals", () => { + it.each<[string, () => Promise, string]>([ + ["a newer version recorded", async () => prepared(EditsV1), "newer than the code's 1"], + [ + "a version below the history's base", + async () => prepared(defineTable("edits", { version: 7, columns: { id: str() }, key: ["id"] })), + "older than the history's base 7", + ], + [ + "a history edited in place", + async () => prepared(evolveTable(EditsV1, (t) => t.rename("file", "name"))), + "recorded with another definition", + ], + ])("%s (the store at version 2) throws before anything is written", async (_, prepare, message) => { + await filled(1); + await prepared(EditsV2); + const seq = store.lastSeq; + expect(await rejection(prepare())).toMatchObject({ message: expect.stringContaining(message) }); + expect(store.lastSeq).toBe(seq); + }); +}); diff --git a/packages/chord/test/next/store/table-writes.test.ts b/packages/chord/test/next/store/table-writes.test.ts new file mode 100644 index 000000000..7fae21403 --- /dev/null +++ b/packages/chord/test/next/store/table-writes.test.ts @@ -0,0 +1,234 @@ +// Table writes (slice 6a §6, §9): schema writes, row writes as the outbox records them, and alter writes. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { add, eq, prefix, replacePrefix } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import type { TableEdit } from "../../../src/next/store/table-history.ts"; +import { stageEdit, stageIndex } from "../../../src/next/store/table-writes.ts"; +import { memoryStore, metered, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 2, + columns: { + id: str(), + path: str(), + hits: int({ default: 0 }), + note: opt(str()), + meta: opt(json(obj({}), { version: 3 })), + }, + key: ["id"], + indexes: { by_path: ["path"] }, +}); +const TABLE = "app.store:edits"; +const FENCE = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +async function commitWith(fill: (changes: TableChanges) => void): Promise { + const staged: StagedWrite[] = []; + fill(new TableChanges(handle, staged)); + await store.commit(staged, FENCE, context); +} + +let handle: Awaited>; + +async function opened(): Promise { + store = await memoryStore({ outbox: true }); + handle = await tableIn(store, Edits); +} + +describe("schema writes", () => { + it("record the table and its indexes; the table write carries version, state, columns, key, fingerprint", async () => { + await opened(); + const [commit] = await store.outbox!.read(0, 1, context); + expect(commit!.writes).toEqual([ + { + kind: "table", + table: TABLE, + version: 2, + state: "ready", + columns: Edits.record, + key: ["id"], + fingerprint: Edits.fingerprint, + }, + { kind: "index", table: TABLE, name: "by_path", columns: ["path"] }, + ]); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready", indexes: { by_path: ["path"] } }); + const indexes = await store.read( + (executor) => executor.all(`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ?`, TABLE), + context, + ); + expect(indexes).toContainEqual({ name: `${TABLE}:by_path` }); + }); + + it("an index write for a table the store does not have refuses the step (D4: the store goes on)", async () => { + await opened(); + const write = stageIndex({ kind: "index", table: "app.store:nope", name: "x", columns: ["id"] }); + await expect(store.commit([write], FENCE, context)).rejects.toThrow("not in the store"); + expect(await store.read(async () => 1, context)).toBe(1); + }); +}); + +describe("row writes in the outbox", () => { + it("carry what a replica needs: put with v and json versions, update and delete with counts, the canonical set", async () => { + await opened(); + await commitWith((changes) => { + changes.put({ id: "a", path: "x/1", meta: {} }); + changes.put({ id: "b", path: "x/2" }); + changes.update({ id: "a" }, (e: { hits: never }) => ({ hits: add(e.hits, 1), note: undefined, meta: {} })); + changes.update({ id: "zz" }, { note: "n" }); + changes.delete({ id: "b" }); + changes.delete({ id: "b" }); + }); + const [, commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "put", + table: TABLE, + row: { id: "a", path: "x/1", hits: 0, meta: {} }, + version: 2, + columns: { meta: 3 }, + }, + { kind: "put", table: TABLE, row: { id: "b", path: "x/2", hits: 0 }, version: 2, columns: {} }, + { + kind: "update", + table: TABLE, + key: { id: "a" }, + set: { hits: { add: [{ col: "hits" }, 1] }, note: null, meta: { json: {} } }, + columns: { meta: 3 }, + count: 1, + }, + { kind: "update", table: TABLE, key: { id: "zz" }, set: { note: "n" }, columns: {}, count: 0 }, + { kind: "delete", table: TABLE, key: { id: "b" }, count: 1 }, + { kind: "delete", table: TABLE, key: { id: "b" }, count: 0 }, + ]); + }); + + it("a put is an upsert: the whole row replaced", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "x", hits: 5, note: "n" })); + await commitWith((changes) => changes.put({ id: "a", path: "y" })); + expect(await handle.get({ id: "a" }, context)).toEqual({ id: "a", path: "y", hits: 0 }); + }); + + it("updateWhere and deleteWhere are one statement each, recorded with where, set and count", async () => { + await opened(); + await commitWith((changes) => { + for (let i = 0; i < 5; i++) changes.put({ id: `e${i}`, path: i < 3 ? `a/${i}` : `b/${i}` }); + }); + const { counts, context: counting } = await metered(); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + changes.updateWhere( + (e: { path: never }) => prefix(e.path, "a/"), + (e: { path: never }) => ({ path: replacePrefix(e.path, "a/", "c/") }), + ); + changes.deleteWhere((e: { path: never }) => eq(e.path, "b/4")); + await store.commit(staged, FENCE, counting); + // The two bulk writes, the outbox commit, lastSeq. + expect(counts()["chord.store.statements"]).toBe(4); + const [, , commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "updateWhere", + table: TABLE, + where: { prefix: ["path", "a/"] }, + set: { path: { replacePrefix: [{ col: "path" }, "a/", "c/"] } }, + columns: {}, + count: 3, + }, + { kind: "deleteWhere", table: TABLE, where: { eq: ["path", "b/4"] }, count: 1 }, + ]); + expect((await handle.page(query(Edits), 10, context)).items.map((r) => r.path)).toEqual([ + "c/0", + "c/1", + "c/2", + "b/3", + ]); + }); +}); + +describe("alter writes", () => { + /** Applies `edits` to the edits table as the migration to version 3 does, one commit. */ + async function altered(...edits: TableEdit[]): Promise { + await store.commit( + edits.map((edit) => stageEdit(TABLE, 3, edit)), + FENCE, + context, + ); + } + + /** The physical columns of the edits table. */ + async function physical(): Promise { + const rows = await store.read( + (executor) => executor.all<{ readonly name: string }>(`SELECT name FROM pragma_table_info('${TABLE}')`), + context, + ); + return rows.map(({ name }) => name); + } + + it("rename a column: a json(...) one's version column, the key and the indexes follow", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + await altered({ kind: "rename", from: "meta", to: "info" }, { kind: "rename", from: "path", to: "file" }); + expect(await physical()).toEqual(["id", "file", "hits", "note", "info", "info$v", "$v"]); + expect(store.table(TABLE)).toMatchObject({ indexes: { by_path: ["file"] } }); + expect(Object.keys(store.table(TABLE)!.columns)).toEqual(["id", "file", "hits", "note", "info"]); + }); + + it("add a column: existing rows get its default; an opt(...) json(...) one adds its version column", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + await altered( + { kind: "add", name: "level", column: { type: "int", default: 3 } }, + { kind: "add", name: "extra", column: { type: "json", optional: true, version: 1, fingerprint: "f" } }, + ); + expect(await physical()).toEqual([ + "id", + "path", + "hits", + "note", + "meta", + "meta$v", + "$v", + "level", + "extra", + "extra$v", + ]); + const rows = await store.read((executor) => executor.all(`SELECT level FROM "${TABLE}"`), context); + expect(rows).toEqual([{ level: 3 }]); + expect(store.table(TABLE)!.columns.level).toEqual({ type: "int", default: 3 }); + }); + + it("drop a column (with its version column); a json edit changes the record only", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + const column = { type: "json", optional: true, version: 4, fingerprint: "g" } as const; + await altered({ kind: "drop", name: "note" }, { kind: "json", name: "meta", column }); + expect(await physical()).toEqual(["id", "path", "hits", "meta", "meta$v", "$v"]); + expect(store.table(TABLE)!.columns.meta).toEqual(column); + await altered({ kind: "drop", name: "meta" }); + expect(await physical()).toEqual(["id", "path", "hits", "$v"]); + }); + + it("are recorded as alter writes with the migration's version; index edits as index and dropIndex", async () => { + await opened(); + const seq = store.lastSeq; + await altered({ kind: "dropIndex", name: "by_path" }, { kind: "drop", name: "path" }); + const [commit] = await store.outbox!.read(seq, 1, context); + expect(commit!.writes).toEqual([ + { kind: "dropIndex", table: TABLE, name: "by_path" }, + { kind: "alter", table: TABLE, version: 3, edit: { kind: "drop", name: "path" } }, + ]); + }); +}); diff --git a/packages/chord/test/next/store/table.test.ts b/packages/chord/test/next/store/table.test.ts new file mode 100644 index 000000000..70acab99d --- /dev/null +++ b/packages/chord/test/next/store/table.test.ts @@ -0,0 +1,66 @@ +// `defineTable` (slice 6a §5): what a declaration materializes, the checks it makes, and its fingerprint. + +import { describe, expect, it } from "vitest"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const columns = { id: str(), path: str(), at: int(), note: opt(str()), meta: json(obj({}), { version: 1 }) }; + +describe("defineTable", () => { + it("materializes columns, key, indexes and the record", () => { + const Edits = defineTable("edits", { version: 2, columns, key: ["id"], indexes: { by_path: ["path", "at"] } }); + expect(Edits.name).toBe("edits"); + expect(Edits.version).toBe(2); + expect(Edits.key).toEqual(["id"]); + expect([...Edits.indexes]).toEqual([["by_path", ["path", "at"]]]); + expect(Object.keys(Edits.record)).toEqual(["id", "path", "at", "note", "meta"]); + expect(Edits.rows.physical).toEqual(["id", "path", "at", "note", "meta", "meta$v", "$v"]); + }); + + it("checks the table name, the version, key, index names and index columns", () => { + expect(() => defineTable("Edits", { version: 1, columns, key: ["id"] })).toThrow("expected lowercase"); + expect(() => defineTable("a:b", { version: 1, columns, key: ["id"] })).toThrow("expected lowercase"); + expect(defineTable("@pi/file-history.edits", { version: 1, columns, key: ["id"] }).name).toBe( + "@pi/file-history.edits", + ); + expect(() => defineTable("edits", { version: 0, columns, key: ["id"] })).toThrow("positive integer"); + expect(() => defineTable("edits", { version: 1, columns, key: [] })).toThrow("no column"); + expect(() => defineTable("edits", { version: 1, columns, key: ["note" as "id"] })).toThrow( + "non-opt str() or int()", + ); + expect(() => defineTable("edits", { version: 1, columns, key: ["meta" as "id"] })).toThrow( + "non-opt str() or int()", + ); + expect(() => defineTable("edits", { version: 1, columns, key: ["id", "id"] })).toThrow("occurs twice"); + expect(() => defineTable("edits", { version: 1, columns, key: ["nope" as "id"] })).toThrow('no column "nope"'); + expect(() => defineTable("edits", { version: 1, columns, key: ["id"], indexes: { ByPath: ["path"] } })).toThrow( + "[a-z0-9_]+", + ); + expect(() => + defineTable("edits", { version: 1, columns, key: ["id"], indexes: { by: ["note" as "path"] } }), + ).toThrow("non-opt"); + }); + + it("fingerprints columns, key and indexes, not their declaration order", () => { + const a = defineTable("edits", { version: 1, columns, key: ["id"], indexes: { x: ["path"], y: ["at"] } }); + const reordered = { meta: columns.meta, note: columns.note, at: columns.at, path: columns.path, id: columns.id }; + const b = defineTable("edits", { + version: 1, + columns: reordered, + key: ["id"], + indexes: { y: ["at"], x: ["path"] }, + }); + expect(b.fingerprint).toBe(a.fingerprint); + expect(defineTable("edits", { version: 1, columns, key: ["id"] }).fingerprint).not.toBe(a.fingerprint); + expect( + defineTable("edits", { version: 1, columns, key: ["id", "at"], indexes: { x: ["path"], y: ["at"] } }) + .fingerprint, + ).not.toBe(a.fingerprint); + const otherMeta = { ...columns, meta: json(obj({ a: int() }), { version: 1 }) }; + expect( + defineTable("edits", { version: 1, columns: otherMeta, key: ["id"], indexes: { x: ["path"], y: ["at"] } }) + .fingerprint, + ).not.toBe(a.fingerprint); + }); +}); diff --git a/packages/chord/test/next/store/text.test.ts b/packages/chord/test/next/store/text.test.ts new file mode 100644 index 000000000..f6db55f3f --- /dev/null +++ b/packages/chord/test/next/store/text.test.ts @@ -0,0 +1,62 @@ +// Text rules (slice 6a §5): code point order, stored text (lone surrogates replaced, NUL refused), prefix bounds. + +import { describe, expect, it } from "vitest"; +import { compareText, prefixUpperBound, storedText } from "../../../src/next/store/text.ts"; + +describe("compareText", () => { + it("orders by code point: a supplementary character after every BMP character, U+E000 to U+FFFF included", () => { + const sorted = ["\u{10000}", "\uFFFF", "\uE000", "\uD7FF", "z", "a", "", "ab"].sort(compareText); + expect(sorted).toEqual(["", "a", "ab", "z", "\uD7FF", "\uE000", "\uFFFF", "\u{10000}"]); + expect(compareText("a", "a")).toBe(0); + expect(compareText("\u{1F600}", "\u{1F601}")).toBeLessThan(0); + }); + + it("agrees with UTF-8 byte order", () => { + const encode = (text: string) => [...new TextEncoder().encode(text)]; + const texts = ["\u{1F600}", "\uFF5E", "\u00E9", "e", "\u{10FFFF}", "\uE000", "aa", "a"]; + const byBytes = [...texts].sort((a, b) => { + const [x, y] = [encode(a), encode(b)]; + for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i]! - y[i]!; + return x.length - y.length; + }); + expect([...texts].sort(compareText)).toEqual(byBytes); + }); +}); + +describe("storedText", () => { + it("keeps well-formed text as it is and replaces a lone surrogate with U+FFFD", () => { + const text = "a\u{1F600}b"; + expect(storedText(text, "v")).toBe(text); + expect(storedText("a\uD800b", "v")).toBe("a\uFFFDb"); + }); + + it("refuses NUL with invalid/args", () => { + expect(() => storedText("a\0b", "edits.path")).toThrow( + expect.objectContaining({ code: "invalid", reason: "args" }), + ); + }); +}); + +describe("prefixUpperBound", () => { + it("is the next code point after the last one", () => { + expect(prefixUpperBound("a/")).toBe("a0"); + expect(prefixUpperBound("a\u{1F600}")).toBe("a\u{1F601}"); + expect(prefixUpperBound("\uFFFF")).toBe("\u{10000}"); + }); + + it("skips the surrogates and carries past U+10FFFF", () => { + expect(prefixUpperBound("\uD7FF")).toBe("\uE000"); + expect(prefixUpperBound("a\u{10FFFF}")).toBe("b"); + expect(prefixUpperBound("\u{10FFFF}\u{10FFFF}")).toBeUndefined(); + expect(prefixUpperBound("")).toBeUndefined(); + }); + + it("bounds exactly the strings starting with the prefix, in code point order", () => { + const texts = ["a", "a/", "a/x", "a/\u{1F600}", "a/\uFFFF", "a0", "a.", "b"]; + for (const prefix of ["a/", "a", "a/\uFFFF"]) { + const upper = prefixUpperBound(prefix)!; + const inRange = texts.filter((t) => compareText(t, prefix) >= 0 && compareText(t, upper) < 0); + expect(inRange).toEqual(texts.filter((t) => t.startsWith(prefix))); + } + }); +}); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts new file mode 100644 index 000000000..20732c04f --- /dev/null +++ b/packages/chord/test/next/support.ts @@ -0,0 +1,1034 @@ +// Test support: cost counters through the in-memory telemetry adapter, GC checks, and hosts, links, watches and +// rejections for the slice tests. + +import { DatabaseSync } from "node:sqlite"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, beforeAll, beforeEach, describe, expect, vi } from "vitest"; +import { BACKGROUND_CONTEXT, type Context, withTelemetry } from "../../src/context/index.ts"; +import type { Op } from "../../src/delta/index.ts"; +import type { JsonValue } from "../../src/json.ts"; +import { + type Address, + arr, + bool, + bytes, + type Contract, + hostKind, + int, + local, + method, + obj, + opt, + type ParamsOf, + remote, + str, +} from "../../src/next/contracts/index.ts"; +import { currentVariants, variantNames, variantOrder } from "../../src/next/contracts/internal.ts"; +import { ChordError, isChordError, PluginError, type Principal } from "../../src/next/data/index.ts"; +import { PROTOCOL_VERSION } from "../../src/next/data/internal.ts"; +import { defineState } from "../../src/next/documents/index.ts"; +import type { Attempt } from "../../src/next/host/attempt.ts"; +import { + type Ctx, + createHost, + defineFacet, + type FacetHost, + type FacetSelf, + type Host, + type HostError, + type PluginFailure, + type RefState, + type RouteHandle, + type Self, + type StoreSelf, +} from "../../src/next/host/index.ts"; +import type { ChordObject } from "../../src/next/host/object.ts"; +import { type Compression, type Link, memoryLinkPair, type PartLimits } from "../../src/next/link/index.ts"; +import { + binaryCodec, + type Codec, + checkMessage, + jsonPart, + type Message, + openPart, + type Part, + parsePart, + textCodec, +} from "../../src/next/link/internal.ts"; +import type { ArchivePart } from "../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../src/next/store/backend.ts"; +import { changeDocument, createDocument, retireDocument } from "../../src/next/store/engine.ts"; +import { json } from "../../src/next/store/json-column.ts"; +import type { StagedWrite } from "../../src/next/store/step.ts"; +import { type ChordStore, openStore, type Store } from "../../src/next/store/store.ts"; +import { defineTable, type TableDefinition } from "../../src/next/store/table.ts"; +import { TableChanges } from "../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../src/next/store/table-evolve.ts"; +import { TableHandle, type TableOwner } from "../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../src/next/store/table-migration.ts"; +import { nodeCompression } from "../../src/next-node/compression.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import type { NativeWatchHandle, WatchCall, WatchHandle } from "../../src/watch.ts"; + +/** Counts of the `chord.data.*` counters. */ +export type Counts = { walks: number; validations: number; compilations: number }; + +/** Runs `work` with a counting context and returns the counters it added. */ +export async function counted(work: (context: Context) => void): Promise { + const telemetry = new InMemoryTelemetryContext(); + await telemetry.startSpan({ name: "test" }, (span) => work(withTelemetry(span, BACKGROUND_CONTEXT))); + const counts: Counts = { walks: 0, validations: 0, compilations: 0 }; + for (const { name, value } of telemetry.getMetrics()) { + const counter = name.replace("chord.data.", "") as keyof Counts; + counts[counter] += value; + } + return counts; +} + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; + +/** Expects every referent to be collected once nothing else holds it. */ +export async function expectCollected(...refs: WeakRef[]): Promise { + for (let round = 0; round < 10 && refs.some((ref) => ref.deref() !== undefined); round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(refs.map((ref) => ref.deref() === undefined)).toEqual(refs.map(() => true)); +} + +/** The heap in use once pending work ran and garbage was collected, repeatedly. */ +export async function settledHeap(): Promise { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + return process.memoryUsage().heapUsed; +} + +/** `true` when `A` and `B` are the same type; for type-level tests where `expectTypeOf` cannot compare (bytes). */ +export type Equal = (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 ? true : false; + +/** A telemetry adapter, a context counting through it, and the counters recorded so far, by name. */ +export async function metered(): Promise<{ + readonly telemetry: InMemoryTelemetryContext; + readonly context: Context; + counts(): Record; +}> { + const telemetry = new InMemoryTelemetryContext(); + // A settled span still records through its adapter's metrics. + const span = await telemetry.startSpan({ name: "test" }, (span) => span); + const counts = (): Record => { + const result: Record = {}; + for (const { name, value } of telemetry.getMetrics()) result[name] = (result[name] ?? 0) + value; + return result; + }; + return { telemetry, context: withTelemetry(span, BACKGROUND_CONTEXT), counts }; +} + +/** Waits until queued watch frames have been delivered. */ +export function delivered(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +/** Starts `watch` and collects its frames as `[value, ops]`. */ +export function framesOf(watch: WatchHandle): [T, readonly Op[]][] { + const frames: [T, readonly Op[]][] = []; + watch.start(async (call) => { + if (call.kind === "value") frames.push([call.value, call.ops]); + }); + return frames; +} + +/** Starts `watch` and collects every call it gets: values (the `retired` flag included) and failures. */ +export function changesOf(watch: WatchHandle): WatchCall[] { + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + return calls; +} + +/** The values of `calls` (a failure as `{ failed: message }`), for short expectations. */ +export function valuesOf(calls: readonly WatchCall[]): (T | { readonly failed: string })[] { + return calls.map((call) => (call.kind === "value" ? call.value : { failed: (call.error as Error).message })); +} + +/** + * Fails the live attempt `self` as a runtime failure of its owner does (slice 6a §1.4); `objects` is the test file's + * `instancesOf(ChordObject, "attempt")`, whose objects other than `self`'s ignore it. + */ +export function failLive(objects: readonly WeakRef[], self: Self, error: ChordError): void { + // The factory's `self` is its attempt. + for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, { kind: "broken", error }); +} + +/** A local type whose objects' attempts a test ends by hand (`restartable`); it ends with its last reference. */ +export const Restartable = local("app.restartable", { + version: 1, + params: { id: int() }, + members: { touch: method() }, +}); + +/** A local type with empty params whose attempts a test ends by hand (`storeRestartable`): it may persist (P3). */ +export const StoreRestartable = local("app.store", { version: 1, params: {}, members: { touch: method() } }); + +/** + * Serves `Restartable` with `factory` per attempt (its number from 1) and gets object 1 live. `fail()` fails the live + * attempt as a runtime failure of its owner does (slice 6a §1.4); `retry()` asks for the next attempt and resolves + * once it is live; `restart()` does both. `objects` is the test file's `instancesOf(ChordObject, "attempt")`; `serve` serves other + * types on the same host. + */ +export function restartable( + objects: readonly WeakRef[], + factory: (attempt: number, self: Self) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void } = {}, +) { + return restartableOf(Restartable, { id: 1 }, objects, factory, options); +} + +/** As `restartable`, for `StoreRestartable`: its factories may declare tables and persisted documents. */ +export function storeRestartable( + objects: readonly WeakRef[], + factory: (attempt: number, self: StoreSelf) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void } = {}, +) { + return restartableOf(StoreRestartable, {}, objects, factory as (attempt: number, self: Self) => void, options); +} + +async function restartableOf( + Type: C, + params: ParamsOf, + objects: readonly WeakRef[], + factory: (attempt: number, self: Self) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void }, +) { + const attempts: Self[] = []; + let started = 0; + const { host, errors } = await loaded((h) => { + options.serve?.(h); + // Both types have the one method `touch`; the factory's `self` is the attempt either way. + h.serve(Type as typeof Restartable, (_, self) => { + factory(++started, self); + attempts.push(self); + return { touch: async () => null }; + }); + }, options); + const get = () => host.get(Type as typeof Restartable, params as ParamsOf, BACKGROUND_CONTEXT); + const reference = await get(); + const fail = (): void => { + failLive(objects, attempts.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + }; + const retry = async (): Promise => { + reference.retry(); + (await get()).release(); + }; + const restart = async (): Promise => { + fail(); + await retry(); + }; + return { host, errors, reference, attempts, fail, retry, restart }; +} + +/** A host with background errors collected, loaded with one facet (`@test/app`) whose setup is `setup`. */ +export async function loaded( + setup: (host: FacetHost, self: FacetSelf) => void, + options: { + readonly telemetry?: InMemoryTelemetryContext; + readonly context?: Context; + readonly closeTimeout?: number; + readonly readyTimeout?: number; + readonly store?: Store; + } = {}, +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { + const errors: HostError[] = []; + const { telemetry, closeTimeout, readyTimeout, store } = options; + const host = createHost({ telemetry, closeTimeout, readyTimeout, store, onError: (error) => errors.push(error) }); + const facets = [{ name: "@test/app", facet: defineFacet(setup) }]; + const { failed } = await host.load(facets, options.context ?? BACKGROUND_CONTEXT); + return { host, errors, failed }; +} + +/** + * A host with background errors collected, loaded with one facet per plugin, in the order given (`setups` by plugin + * name). + */ +export async function loadedPlugins( + setups: Readonly void>>, + options: { readonly telemetry?: InMemoryTelemetryContext; readonly readyTimeout?: number } = {}, +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { + const errors: HostError[] = []; + const host = createHost({ ...options, onError: (error) => errors.push(error) }); + const facets = Object.entries(setups).map(([name, setup]) => ({ name, facet: defineFacet(setup) })); + const { failed } = await host.load(facets, BACKGROUND_CONTEXT); + return { host, errors, failed }; +} + +/** A store on a fresh SQLite `:memory:` database, as Chord's layers see it. */ +export async function memoryStore(options?: { readonly outbox?: boolean }): Promise { + const db = await openNodeSqliteDatabase(":memory:"); + return (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore; +} + +/** A statement of the Node facade's transactions a test can make fail. */ +export type TransactionStatement = "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK"; + +/** + * A Node SQLite facade on `:memory:` whose next `statement` throws, once per `fail` (the D4 matrix: a failed + * `BEGIN`, `COMMIT` or `ROLLBACK`, as the real facade sees one). + */ +export function faultyDatabase(): { readonly db: NodeSqliteDatabase; fail(statement: TransactionStatement): void } { + const database = new DatabaseSync(":memory:"); + const failing = new Set(); + const exec = database.exec.bind(database); + database.exec = (sql: string): void => { + if (!failing.delete(sql)) { + exec(sql); + return; + } + // A failed `ROLLBACK` here still rolls back, so the database stays usable for the rest of the test. + if (sql === "ROLLBACK") exec(sql); + throw new Error(`injected: ${sql} failed`); + }; + return { db: new NodeSqliteDatabase(database), fail: (statement) => void failing.add(statement) }; +} + +/** A store over `faultyDatabase()`, and its `fail`. */ +export async function faultyStore(options?: { + readonly outbox?: boolean; +}): Promise<{ readonly store: ChordStore; fail(statement: TransactionStatement): void }> { + const { db, fail } = faultyDatabase(); + return { store: (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore, fail }; +} + +/** + * Holds `store`'s next commit step inside its backend transaction, after its admission: `entered` resolves there, + * `release()` lets it finish; later commits run as usual. + */ +export function holdNextCommit(store: ChordStore): { readonly entered: Promise; release(): void } { + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + store.commit = commit; + const held: StagedWrite = { + order: "write", + apply: async () => { + entered.resolve(); + await gate.promise; + return [{ kind: "dropIndex", table: "x", name: "y" }]; + }, + }; + return commit([...writes, held], fence, ctx); + }; + return { entered: entered.promise, release: () => gate.resolve() }; +} + +/** The owner of a test's table handles: its failures collected. */ +export function tableOwner(failures: ChordError[] = []): TableOwner & { readonly failures: ChordError[] } { + return { label: "app.store", scope: {}, assertLive: () => {}, fail: (error) => void failures.push(error), failures }; +} + +/** Creates `definition` (and its indexes) in `store` as `app.store:` and returns a handle to it. */ +export async function tableIn( + store: ChordStore, + definition: TableDefinition, + owner = tableOwner(), +): Promise { + const table = `app.store:${definition.name}`; + const fence = { admit: () => true, adopt: () => {} }; + await prepareTables([{ definition, table }], store, fence, "app.store", BACKGROUND_CONTEXT); + return new TableHandle(definition, table, store, owner); +} + +/** The error `promise` rejects with, as ChordError JSON when it is one. */ +export async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + return isChordError(error) ? error.toJSON() : error; + } + throw new Error("expected a rejection"); +} + +/** Fakes `setTimeout` and `Date` (from 0) in the calling `describe`; `setImmediate` stays real for `flushed()`. */ +export function fakeTimers(): void { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"], now: 0 }); + }); + afterEach(() => { + vi.useRealTimers(); + }); +} + +/** Waits until every pending promise callback has run. */ +export function flushed(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +/** The kinds and objects of the reported `activation_failed` errors, with their causes' kinds. */ +export function failures(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => + error.kind === "activation_failed" ? [`${error.object}: ${error.failure.kind}`] : [], + ); +} + +/** The principals of the remote tests. */ +export type TestPrincipal = + | { kind: "owner" } + | { kind: "guest"; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +/** + * A server (principal `server`, loaded with one facet whose setup is `setup`) and a client connected to it, the + * client's link stamped `child`; background errors collected per host. + */ +export async function linked( + setup: (host: FacetHost) => void, + options: { + readonly child?: TestPrincipal; + readonly context?: Context; + readonly telemetry?: InMemoryTelemetryContext; + readonly readyTimeout?: number; + } = {}, +): Promise<{ + readonly server: Host; + readonly client: Host; + readonly errors: { readonly server: HostError[]; readonly client: HostError[] }; + readonly links: readonly [Link, Link]; +}> { + const context = options.context ?? BACKGROUND_CONTEXT; + const errors = { server: [] as HostError[], client: [] as HostError[] }; + const { telemetry, readyTimeout } = options; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + telemetry, + readyTimeout, + onError: (error) => errors.server.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet(setup) }], context); + const client = createHost({ telemetry, readyTimeout, onError: (error) => errors.client.push(error) }); + await client.load([], context); + const links = memoryLinkPair(); + server.accept(links[0], { principal: options.child ?? { kind: "owner" }, codec: testCodec() }, context); + client.connect(links[1], { principal: { kind: "server" }, codec: testCodec() }, context); + return { server, client, errors, links }; +} + +/** The incarnation a raw peer greets with. */ +export const RAW_PEER = "00000000-0000-7000-8000-000000000000"; + +/** + * A peer speaking the wire by hand, in the current test codec, at the other end of `link`: it sends `hello` (unless + * `hello: false`, announcing `calls`, default 256), records what arrives, acks every `bound` and `state` with its origin + * (unless `ack: false`: then `ack()` sends one), and sends what a test forges. + */ +export class RawPeer { + readonly received: Message[] = []; + closed = false; + readonly codec: Codec; + readonly #link: Link; + #waiters: (() => void)[] = []; + + constructor( + link: Link, + options: { + readonly hello?: boolean; + readonly calls?: number; + readonly ack?: boolean; + readonly codec?: Codec; + } = {}, + ) { + this.#link = link; + this.codec = options.codec ?? testCodec(); + link.onMessage((bytes) => { + const message = checkMessage(this.codec.decode(bytes), BACKGROUND_CONTEXT); + this.received.push(message); + if (options.ack !== false && (message.kind === "bound" || message.kind === "state")) this.ack(message.origin); + for (const wake of this.#waiters.splice(0)) wake(); + }); + link.onClose(() => { + this.closed = true; + for (const wake of this.#waiters.splice(0)) wake(); + }); + if (options.hello !== false) { + this.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: options.calls ?? 256 }); + } + } + + send(message: Message): void { + this.#link.send(this.codec.encode(message)); + } + + /** Acknowledges one `bound` or `state` message of `origin` (none: the peer's own). */ + ack(origin?: number): void { + this.send(origin === undefined ? { kind: "ack" } : { kind: "ack", origin }); + } + + close(): void { + this.#link.close(); + } + + /** Sends a message as raw bytes. */ + sendBytes(message: Uint8Array): void { + this.#link.send(message); + } + + /** The first received message of `kind` not taken yet, waiting for it; throws if the link closes first. */ + async next(kind: K): Promise> { + for (;;) { + const index = this.received.findIndex((message) => message.kind === kind); + if (index >= 0) return this.received.splice(index, 1)[0] as Extract; + if (this.closed) throw new Error(`link closed before ${kind}`); + await new Promise((resolve) => this.#waiters.push(resolve)); + } + } + + /** The kinds received and not taken, in order. */ + kinds(): string[] { + return this.received.map(({ kind }) => kind); + } +} + +/** A part's value as its owner reads it: opened (inflated when deflated) without limits, and parsed. */ +export function partValue(part: Part): unknown { + const unlimited = { part: Number.POSITIVE_INFINITY, decoded: Number.POSITIVE_INFINITY }; + return parsePart(openPart(part, unlimited, nodeCompression, BACKGROUND_CONTEXT)); +} + +/** The codecs every protocol test runs with. */ +export const CODECS: readonly Codec[] = [binaryCodec, textCodec]; + +let currentCodec: Codec = binaryCodec; + +/** The codec of the protocol test running now (`forEachCodec`); links in tests use it at both ends. */ +export function testCodec(): Codec { + return currentCodec; +} + +/** Runs `suite` once per codec, as `binary codec` and `text codec`. */ +export function forEachCodec(suite: () => void): void { + for (const codec of CODECS) { + describe(`${codec.name} codec`, () => { + beforeAll(() => { + currentCodec = codec; + }); + suite(); + }); + } +} + +/** The wire's indexes of a remote contract's variant (2b): the variant's, and its state members' and methods' by name. */ +export function wireNames(Type: Contract, variant: string) { + const names = variantNames(currentVariants(Type).get(variant)!); + const indexes = (list: readonly string[]) => Object.fromEntries(list.map((name, index) => [name, index])); + return { + variant: variantOrder(Type).indexOf(variant), + state: indexes(names.state), + methods: indexes(names.methods), + /** A message's members by index as an object by name, their parts parsed. */ + named: (members: readonly { readonly member: number; readonly ops?: Part; readonly value?: Part }[]) => + Object.fromEntries(members.map(({ member, ops, value }) => [names.state[member], partValue((ops ?? value)!)])), + }; +} + +/** A server host as in `linked` and a raw peer on a link it accepted as `child`, with `limits`. */ +export async function rawLinked( + setup: (host: FacetHost) => void, + options: { + readonly child?: Principal; + readonly hello?: boolean; + readonly readyTimeout?: number; + readonly limits?: PartLimits; + readonly compression?: Compression; + /** What the raw peer is to the host: a child (`accept`, the default) or its parent (`connect`). */ + readonly role?: "child" | "parent"; + } = {}, +): Promise<{ readonly server: Host; readonly peer: RawPeer; readonly errors: HostError[] }> { + const errors: HostError[] = []; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + readyTimeout: options.readyTimeout, + compression: options.compression, + onError: (error) => errors.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet(setup) }], BACKGROUND_CONTEXT); + const [a, b] = memoryLinkPair(); + const child = (options.child ?? { kind: "owner" }) as TestPrincipal; + const link = { principal: child, codec: testCodec(), limits: options.limits }; + if (options.role === "parent") server.connect(a, link, BACKGROUND_CONTEXT); + else server.accept(a, link, BACKGROUND_CONTEXT); + return { server, peer: new RawPeer(b, { hello: options.hello }), errors }; +} + +/** Waits until messages and closes in flight on memory links (one task per hop) have arrived. */ +export async function transported(): Promise { + for (let hop = 0; hop < 6; hop++) await flushed(); +} + +/** The variants of the notes types: an editor variant for owners, a reader one for guests. */ +const NOTES_VARIANTS = { + editor: { + state: { notes: arr(obj({ text: str() })), count: int() }, + methods: { + add: method(obj({ text: str() }), int()), + fail: method(obj({ how: str() })), + wait: method(), + echo: method(obj({ data: bytes() }), bytes()), + }, + }, + reader: { state: { notes: arr(obj({ text: str() })), count: int() }, methods: { wait: method() } }, +}; + +/** The host kind of the test servers, where every test type but `PlacedNotes` lives. */ +export const Server = hostKind("server"); + +/** `Server()` as a hold names it on the wire: the address of every test type served on the server. */ +export const SERVER_ADDRESS = { kind: "server", params: {} }; + +/** The remote type of the link tests: notes per session, an editor variant for owners, a reader one for guests. */ +export const Notes = remote("app.notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: NOTES_VARIANTS, +}); + +/** The kind of the relay tests' workers: one per session. */ +export const Session = hostKind("session", { id: str() }); + +/** `Notes` placed on the session's worker: the relay tests' type. */ +export const PlacedNotes = remote("app.placed-notes", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: NOTES_VARIANTS, +}); + +const NotesDoc = defineState("notes", { + schema: obj({ list: arr(obj({ text: str() })) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** The shared store behind `Notes` (§6): the notes of a session, once, whoever views them. */ +export const NoteStore = local("app.note-store", { + version: 1, + params: { session: str() }, + members: { list: arr(obj({ text: str() })), add: method(obj({ text: str() }), int()) }, +}); + +/** What `serveNotes` records: the handlers' contexts of `wait` calls, and every factory run of `Notes`. */ +export type NotesProbe = { readonly waits: Ctx[]; readonly factories: string[] }; + +/** + * Serves `NoteStore` and `Notes` over it: `notes` and `count` are derives over the store's list; guests get the reader + * variant. `wait` resolves when its context is aborted; `fail` throws a ChordError (`chord`), a PluginError (`plugin`) + * or a plain Error. + */ +export function serveNotes( + host: FacetHost, + Type: typeof Notes | typeof PlacedNotes = Notes, +): NotesProbe { + const probe: NotesProbe = { waits: [], factories: [] }; + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + list: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push({ text }), ctx); + return doc.value.list.length; + }, + }; + }); + // The two types differ in their name and host address only. + host.serve(Type as typeof Notes, ({ session }, principal, self) => { + probe.factories.push(`${session} ${principal.kind}`); + const store = self.get(NoteStore, { session }); + const notes = self.derive([store.list], ([list]) => list); + const count = self.derive([store.list], ([list]) => list.length); + const wait = async (ctx: Ctx) => { + probe.waits.push(ctx); + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + return null; + }; + if (principal.kind === "guest") return { variant: "reader", members: { notes, count, wait } }; + return { + variant: "editor", + members: { + notes, + count, + wait, + add: async (args, ctx) => store.add(args, ctx), + fail: async ({ how }) => { + if (how === "chord") throw new ChordError({ code: "forbidden", message: "not yours" }); + if (how === "plugin") throw new PluginError("quota", "too many", { limit: 1 }); + throw new Error("plain"); + }, + echo: async ({ data }) => data, + }, + }; + }); + return probe; +} + +/** Adds a note to the session's store from the server itself; the reference it takes keeps the store. */ +export async function addNote(server: Host, session: string, text: string): Promise { + const store = await server.get(NoteStore, { session }, BACKGROUND_CONTEXT); + await store.add({ text }, BACKGROUND_CONTEXT); +} + +/** A client connected to a raw peer playing the provider (the link's limits `limits`), after the handshake. */ +export async function rawProvider( + options: { readonly readyTimeout?: number; readonly limits?: PartLimits; readonly compression?: Compression } = {}, +) { + const errors: HostError[] = []; + const client = createHost({ + readyTimeout: options.readyTimeout, + compression: options.compression, + onError: (error) => errors.push(error), + }); + await client.load([], BACKGROUND_CONTEXT); + const peer = await rawParent(client, options.limits); + return { client, peer, errors }; +} + +/** + * Connects `host` to a new raw peer playing its parent (a first link, or the next after a drop), the link's limits + * `limits`, after the handshake. + */ +export async function rawParent(host: Host, limits?: PartLimits): Promise { + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec(), limits }, BACKGROUND_CONTEXT); + const peer = new RawPeer(b); + await peer.next("hello"); + return peer; +} + +/** Starts a `ref.state` watch and collects the states it delivers, by kind (failed ones with their error's message). */ +export function statesOf(watch: NativeWatchHandle): string[] { + const states: string[] = []; + watch.start(async (call) => { + if (call.kind !== "value") return; + const state = call.value; + states.push(state.kind === "failed" ? `failed: ${state.cause.error.message}` : state.kind); + }); + return states; +} + +/** The wire's indexes of `Notes`'s editor variant. */ +export const EDITOR = wireNames(Notes, "editor"); + +/** Answers the next hold with `bound`: the editor variant with `notes` and `count`. */ +export async function bindNext(peer: RawPeer, notes: { text: string }[] = []) { + const { hold } = await peer.next("hold"); + bindHold(peer, hold, notes); + return hold; +} + +/** Sends `bound` for `hold`: the editor variant with `notes` and `count`. */ +export function bindHold(peer: RawPeer, hold: number, notes: { text: string }[] = []): void { + const members = [ + { member: EDITOR.state.notes!, value: jsonPart(notes) }, + { member: EDITOR.state.count!, value: jsonPart(notes.length) }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); +} + +/** + * Records weakly every instance of a class that calls `method` from now on, for GC tests of objects a test cannot + * reach (holds, remote objects). Patches the prototype for the rest of the test file. + */ +export function instancesOf( + Class: { readonly prototype: T }, + method: keyof T & string, +): WeakRef[] { + const instances: WeakRef[] = []; + const prototype = Class.prototype as Record unknown>; + const original = prototype[method]!; + prototype[method] = function (this: T, ...args: unknown[]) { + instances.push(new WeakRef(this)); + return original.apply(this, args); + }; + return instances; +} + +/** What a route's `resolve` returns in the relay tests. */ +export type Resolved = { + readonly link: Link; + readonly principal: TestPrincipal; + readonly trusted?: boolean; + readonly codec?: Codec; + readonly limits?: PartLimits; +}; + +/** What `relayed` builds: the hosts, the links the server holds, the resolves it ran and the errors per host. */ +export type Relayed = { + readonly tab: Host; + readonly server: Host; + /** The server's route to `Session`'s addresses. */ + readonly route: RouteHandle; + /** The worker of each resolved session address, the latest one per key. */ + readonly workers: Map>; + /** The server's end of each resolved route link, in order. */ + readonly routeLinks: Link[]; + /** The server's end of the tab's link. */ + readonly tabLink: Link; + /** The address keys `resolve` ran for, in order. */ + readonly resolves: string[]; + readonly errors: { readonly tab: HostError[]; readonly server: HostError[]; readonly worker: HostError[] }; +}; + +/** + * Three hosts: a tab connected to a server, and a server routing `Session` to an in-process worker per address (made + * by each `resolve`, connected before it loads, W6) that serves `PlacedNotes` and `Notes`'s store. The tab acts as + * `tab` (owner by default); `authorize` admits everyone by default; route links are trusted when `trusted`. + * `server` and `worker` add setups of their own; `resolve` wraps the default one (`worker()` makes the worker). + */ +export async function relayed( + options: { + readonly tab?: TestPrincipal; + readonly authorize?: (principal: TestPrincipal, address: { readonly key: string }) => boolean; + readonly trusted?: boolean; + readonly codecs?: { readonly tab?: Codec; readonly route?: Codec }; + readonly server?: (host: FacetHost) => void; + readonly worker?: (host: FacetHost) => void; + readonly resolve?: ( + address: Address, + worker: () => Resolved, + context: Context, + ) => Promise; + readonly context?: Context; + /** The server's and the workers' `revokedBy`. */ + readonly revokedBy?: (principal: TestPrincipal) => readonly JsonValue[]; + /** The server's `revokedBy` when it differs from the workers'. */ + readonly serverRevokedBy?: (principal: TestPrincipal) => readonly JsonValue[]; + /** The compressor of each host: tab, server, workers. */ + readonly compression?: { + readonly tab?: Compression; + readonly server?: Compression; + readonly worker?: Compression; + }; + /** The limits of the tab's link at the server. */ + readonly limits?: PartLimits; + } = {}, +): Promise { + const context = options.context ?? BACKGROUND_CONTEXT; + const errors = { tab: [] as HostError[], server: [] as HostError[], worker: [] as HostError[] }; + const workers = new Map>(); + const routeLinks: Link[] = []; + const resolves: string[] = []; + const { revokedBy, compression = {} } = options; + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy: options.serverRevokedBy ?? revokedBy, + compression: compression.server, + onError: (error) => errors.server.push(error), + }); + const codec = options.codecs?.route ?? testCodec(); + const worker = (address: Address): Resolved => { + resolves.push(address.key); + const [a, b] = memoryLinkPair(); + const principal = { kind: "worker", session: address.params.id } as const; + const host = createHost({ + address, + principal, + revokedBy, + compression: compression.worker, + onError: (error) => errors.worker.push(error), + }); + host.connect(b, { principal: { kind: "server" }, trusted: options.trusted, codec }, context); + const facet = defineFacet((h) => { + serveNotes(h, PlacedNotes); + options.worker?.(h); + }); + void host.load([{ name: "@test/worker", facet }], context); + workers.set(address.key, host); + routeLinks.push(a); + return { link: a, principal, trusted: options.trusted, codec }; + }; + const resolve = options.resolve ?? (async (_address, made) => made()); + const route = server.route( + Session, + { + authorize: options.authorize ?? (() => true), + resolve: (address, routing) => resolve(address, () => worker(address), routing), + }, + context, + ); + await server.load( + [{ name: "@test/server", facet: defineFacet((h) => options.server?.(h)) }], + context, + ); + const tab = createHost({ compression: compression.tab, onError: (error) => errors.tab.push(error) }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + const tabCodec = options.codecs?.tab ?? testCodec(); + const accepted = { principal: options.tab ?? { kind: "owner" }, codec: tabCodec, limits: options.limits } as const; + server.accept(a, accepted, context); + tab.connect(b, { principal: { kind: "server" }, codec: tabCodec }, context); + return { tab, server, route, workers, routeLinks, tabLink: a, resolves, errors }; +} + +/** + * A `resolve` for `relayed` whose workers are raw peers playing the provider, collected in `peers` in order; their + * links trusted when `trusted`. + */ +export function rawWorkers( + peers: RawPeer[], + options: { + readonly calls?: number; + readonly ack?: boolean; + readonly trusted?: boolean; + readonly limits?: PartLimits; + } = {}, +): (address: Address) => Promise { + return async (address) => { + const [a, b] = memoryLinkPair(); + peers.push(new RawPeer(b, options)); + const principal = { kind: "worker", session: address.params.id } as const; + return { link: a, principal, codec: testCodec(), trusted: options.trusted, limits: options.limits }; + }; +} + +/** A host's incarnation, which its handshakes send. */ +export function incarnationOf(host: Host): string { + // Every host is a `ChordHost`. + return (host as unknown as { readonly incarnation: string }).incarnation; +} + +/** Counts the bytes every `DecompressionStream` puts out while installed; `restore` puts the real one back. */ +export function countInflated(): { readonly bytes: number; restore(): void } { + const Real = globalThis.DecompressionStream; + const counter = { bytes: 0, restore: () => vi.unstubAllGlobals() }; + class Counted { + readonly writable: WritableStream; + readonly readable: ReadableStream; + constructor(format: CompressionFormat) { + const real = new Real(format); + this.writable = real.writable; + this.readable = real.readable.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + counter.bytes += chunk.byteLength; + controller.enqueue(chunk); + }, + }), + ); + } + } + vi.stubGlobal("DecompressionStream", Counted); + return counter; +} + +/** The tables of `archiveStore`: edits (v1 with an index; v2 adds `done`) and notes, whose v2 migration crashed. */ +export const ArchiveTables = (() => { + const EditsV1 = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + note: opt(str()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, + }); + const EditsV2 = evolveTable(EditsV1, (t) => t.add("done", bool({ default: false }))); + const NotesV1 = defineTable("notes", { version: 1, columns: { id: int(), text: str() }, key: ["id"] }); + const NotesV2 = evolveTable(NotesV1, (t) => + t.rows((row) => { + if (row.id === 120) throw new Error("crash"); + return { text: `${row.text}!` }; + }), + ); + return { EditsV1, EditsV2, NotesV1, NotesV2 }; +})(); + +/** + * A store holding a bit of everything an archive carries: `app.store:edits` at version 2 with 150 rows (two + * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `app.store:notes` left + * `migrating` at version 2 after one committed chunk, and documents: a singleton, keyed ones with deltas after their + * base, and a retired one. + */ +export async function archiveStore(options?: { readonly outbox?: boolean }): Promise { + const { EditsV1, EditsV2, NotesV1, NotesV2 } = ArchiveTables; + const store = await memoryStore(options); + const fence = { admit: () => true, adopt: () => {} }; + const edits = await tableAt(store, "app.store:edits", EditsV1); + const notes = await tableAt(store, "app.store:notes", NotesV1); + const staged: StagedWrite[] = []; + const editChanges = new TableChanges(edits, staged); + for (let i = 0; i < 150; i++) { + editChanges.put({ + id: `e${String(i).padStart(3, "0")}`, + path: `p/${i % 4}`, + at: i, + ...(i % 2 === 0 ? { note: `n${i}` } : {}), + ...(i % 3 === 0 ? { meta: { a: i } } : {}), + }); + } + const noteChanges = new TableChanges(notes, staged); + for (let i = 0; i < 150; i++) noteChanges.put({ id: i, text: `t${i}` }); + await store.commit(staged, fence, BACKGROUND_CONTEXT); + await prepareTables([{ definition: EditsV2, table: "app.store:edits" }], store, fence, "app", BACKGROUND_CONTEXT); + await prepareTables( + [{ definition: NotesV2, table: "app.store:notes" }], + store, + fence, + "app", + BACKGROUND_CONTEXT, + ).catch(() => {}); + const address = { namespace: "chord", owner: "app.store", kind: "memo" }; + const base = (value: JsonValue) => ({ version: 1, fingerprint: "f1", value }); + let ids: readonly number[] = []; + const created = { + admit: () => true, + adopt: (_: number, created: readonly number[]) => { + ids = created; + }, + }; + await store.commit( + ["", "a", "b", "gone"].map((key) => + createDocument({ ...address, key }, base({ title: key }), JSON.stringify({ title: key })), + ), + created, + BACKGROUND_CONTEXT, + ); + const [, a, b, gone] = ids; + const delta = (title: string) => ({ kind: "delta", version: 1, ops: [["s", ["title"], title]] }) as const; + await store.commit( + [changeDocument(a!, 1, delta("a2"), JSON.stringify(delta("a2").ops)), retireDocument(gone!, 1)], + fence, + BACKGROUND_CONTEXT, + ); + await store.commit([changeDocument(a!, 2, delta("a3"), JSON.stringify(delta("a3").ops))], fence, BACKGROUND_CONTEXT); + await store.commit([changeDocument(b!, 1, delta("b2"), JSON.stringify(delta("b2").ops))], fence, BACKGROUND_CONTEXT); + return store; +} + +/** `definition` prepared as `table` in `store`, and a handle to it. */ +async function tableAt(store: ChordStore, table: string, definition: TableDefinition): Promise { + await prepareTables( + [{ definition, table }], + store, + { admit: () => true, adopt: () => {} }, + "app", + BACKGROUND_CONTEXT, + ); + return new TableHandle(definition, table, store, tableOwner()); +} + +/** Every part of `store`'s export. */ +export async function exported(store: Store): Promise { + const parts: ArchivePart[] = []; + for await (const part of store.export(BACKGROUND_CONTEXT)) parts.push(part); + return parts; +} diff --git a/packages/chord/test/output.test.ts b/packages/chord/test/output.test.ts index 00e7d647e..b35a62bf6 100644 --- a/packages/chord/test/output.test.ts +++ b/packages/chord/test/output.test.ts @@ -57,6 +57,18 @@ describe("tool output bounds", () => { expect(sanitizeOutput("a\0b\tc\nd\re\u0007f\ufff9g\ufffbh😀")).toBe("ab\tc\ndefgh😀"); }); + // #11604 + it("removes whole ANSI escape sequences, not only their ESC bytes", () => { + expect(sanitizeOutput("\x1b[31mERROR\x1b[0m\n")).toBe("ERROR\n"); + expect(sanitizeOutput("\x1b[1;38;5;208mx\x1b[?25l\x1b[2K\x1b(By\x1b7z")).toBe("xyz"); + expect(sanitizeOutput("\x1b]8;;https://pi.dev\x07link\x1b]8;;\x1b\\ \x1b]0;title\x07ok")).toBe("link ok"); + const output = new OutputBuffer({ retain: "tail", maxBytes: 1024, maxLines: 100 }); + output.push("\x1b[3"); + output.push("1mERROR\x1b[0"); + output.push("m\n"); + expect(output.snapshot().text).toBe("ERROR\n"); + }); + it("keeps output within the limits unchanged", () => { expect(bound("a\nb\n", head(2))).toEqual({ kept: "a\nb\n", droppedBytes: 0, droppedLines: 0 }); expect(bound("a\nb", tail(2))).toEqual({ kept: "a\nb", droppedBytes: 0, droppedLines: 0 }); diff --git a/packages/chord/test/placement.test.ts b/packages/chord/test/placement.test.ts index 869bfa655..3e3a7bd8e 100644 --- a/packages/chord/test/placement.test.ts +++ b/packages/chord/test/placement.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { bind, @@ -22,17 +22,17 @@ const tick = () => new Promise((resolve) => setImmediate(resolve)); /** Lives on `session:` + its input's `session`. */ const Note = remote("t.note", { version: 1, - input: Type.Object({ session: Type.String(), id: Type.String() }), + input: obj({ session: str(), id: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** Lives on `server`. */ const Catalog = remote("t.catalog", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** A server with two session hosts as children; every host serves what it may and reports its name. */ @@ -126,9 +126,9 @@ describe("placement", () => { it("the host comes from the normalized input: unknown fields and defaults do not change it", async () => { const Doc = remote("t.doc-default", { version: 1, - input: Type.Object({ session: Type.String({ default: "S1" }) }), + input: obj({ session: str({ default: "S1" }) }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); const { server, s1, close } = cluster(); s1.serve(Doc, () => ({ variant: "main", where: () => "session:S1" })); @@ -143,7 +143,7 @@ describe("placement", () => { const at = (host: string) => remote("t.moved", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(host), variants: { main: { methods: { ping: method() } } }, }); @@ -155,7 +155,7 @@ describe("placement", () => { api.serve( remote("t.moved", { version, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(place), variants: { main: { methods: { ping: method() } } }, }), @@ -170,7 +170,7 @@ describe("placement", () => { // A new version that `accepts` version 1 but lives elsewhere: no adapter makes version 1 consumers bind. const moved = remote("t.moved", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { methods: { ping: method() } } }, }); @@ -182,7 +182,7 @@ describe("placement", () => { }); it("types: the family's field is a required string of the input, and remote contracts need a host", () => { - const input = Type.Object({ session: Type.String(), count: Type.Number(), note: Type.Optional(Type.String()) }); + const input = obj({ session: str(), count: num(), note: opt(str()) }, { open: true }); const variants = { main: {} }; remote("t.ok", { version: 1, input, host: hostFamily("session:", "session"), variants }); // @ts-expect-error: no such field diff --git a/packages/chord/test/principals.test.ts b/packages/chord/test/principals.test.ts index 4aef22cb3..aab011dab 100644 --- a/packages/chord/test/principals.test.ts +++ b/packages/chord/test/principals.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -26,8 +26,8 @@ const principalKey = (principal: Principal): string => const Who = local("t.who", { version: 1, - input: Type.Object({}), - members: { who: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { who: method(undefined, str()) }, }); describe("principals", () => { @@ -86,9 +86,9 @@ describe("principals", () => { it("a child's parent acts as the principal the child gave `connect`; relayed callers keep theirs", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("worker"), - variants: { main: { methods: { who: method(undefined, Type.String()) } } }, + variants: { main: { methods: { who: method(undefined, str()) } } }, }); const server = createHost({ name: "server", self: { kind: "host" }, principalKey }); const [down, up] = memoryLinkPair(); diff --git a/packages/chord/test/provider-cost.test.ts b/packages/chord/test/provider-cost.test.ts index 470c74b20..50a8bf533 100644 --- a/packages/chord/test/provider-cost.test.ts +++ b/packages/chord/test/provider-cost.test.ts @@ -1,7 +1,7 @@ // Entry 8: what a change costs the provider. The diff is computed once per change; each feed (one per connection and // member) encodes it with its own stateful encoder and its link serializes it. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, it, vi } from "vitest"; import type * as delta from "../src/delta/index.ts"; import { type Ctx, createHost, hostNamed, memoryLinkPair, method, remote } from "../src/index.ts"; @@ -30,16 +30,16 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const List = remote("t.e8-cost", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { items: Type.Array(Type.String()) }, methods: { add: method(Type.String()) } } }, + variants: { main: { state: { items: arr(str()) }, methods: { add: method(str()) } } }, }); describe("provider cost", () => { it("one change to K clients: K encodes (one per feed), one frame per link, the same ops on every link", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(List, (_input, _principal, self) => { - const items = self.state("items", Type.Array(Type.String()), []); + const items = self.state("items", arr(str()), []); return { variant: "main", items, diff --git a/packages/chord/test/readiness.test.ts b/packages/chord/test/readiness.test.ts index a781a6e34..b34b2540c 100644 --- a/packages/chord/test/readiness.test.ts +++ b/packages/chord/test/readiness.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { createHost, @@ -15,8 +15,8 @@ import { const tick = () => new Promise((resolve) => setImmediate(resolve)); const unexpectedError = (error: unknown) => assert.fail(String(error)); -const Eager = local("ready.eager", { version: 1, input: Type.Object({}) }); -const Lazy = local("ready.lazy", { version: 1, input: Type.Object({}) }); +const Eager = local("ready.eager", { version: 1, input: obj({}, { open: true }) }); +const Lazy = local("ready.lazy", { version: 1, input: obj({}, { open: true }) }); // Startup must not report ready while an eager object has failed. describe("host: eager readiness", () => { @@ -162,7 +162,7 @@ describe("host: eager readiness", () => { it("ready includes direct host serves, but not later serves", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const First = local("ready.first", { version: 1, input: Type.Object({}) }); + const First = local("ready.first", { version: 1, input: obj({}, { open: true }) }); let created = 0; try { host.serve( @@ -201,7 +201,7 @@ describe("host: eager readiness", () => { }); const Status = remote("ready.status", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { default: {} }, }); diff --git a/packages/chord/test/reload.test.ts b/packages/chord/test/reload.test.ts index f2931f0c9..b9998b316 100644 --- a/packages/chord/test/reload.test.ts +++ b/packages/chord/test/reload.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, type Schema, str, unknown } from "@earendil-works/pi-schema"; import { describe, it, onTestFinished, vi } from "vitest"; import { type Ctx, @@ -30,15 +30,15 @@ function latch() { const Greeter = local("t.greeter", { version: 1, - input: Type.Object({ id: Type.Optional(Type.String({ default: "a" })) }), - members: { greet: method(Type.String(), Type.String()), count: Type.Unknown() }, + input: obj({ id: opt(str({ default: "a" })) }, { open: true }), + members: { greet: method(str(), str()), count: unknown() }, }); -const CountV1 = Type.Object({ n: Type.Number() }); +const CountV1 = obj({ n: num() }, { open: true }); interface GreeterSpec { readonly greeting: string; - readonly schema?: typeof CountV1 | ReturnType; + readonly schema?: typeof CountV1 | Schema; readonly initial?: object; readonly bump?: (draft: Record) => void; readonly options?: ServeOptions; @@ -129,7 +129,7 @@ describe("reload", () => { const writing = greeterFacet({ greeting: "hi", extra: (self) => { - const probe = self.state("probe", Type.Object({}), {}); + const probe = self.state("probe", obj({}, { open: true }), {}); probe.replace(captured!, {}); }, }); @@ -164,7 +164,7 @@ describe("reload", () => { await host.hold(Greeter, {}).ready; const changed = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, }); await assert.rejects( @@ -191,7 +191,7 @@ describe("reload", () => { const froms: number[] = []; const v2 = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, bump: (draft) => { draft.total = (draft.total ?? 0) + 1; @@ -218,16 +218,16 @@ describe("reload", () => { const host = quiet(); const Pair = local("t.pair", { version: 1, - input: Type.Object({}), - members: { a: Type.Unknown(), b: Type.Unknown(), bump: method() }, + input: obj({}, { open: true }), + members: { a: unknown(), b: unknown(), bump: method() }, }); const migrated: string[] = []; const facet = (version: number): Facet => ({ name: "pair", setup: (api) => api.serve(Pair, (_input, _principal, self) => { - const a = self.state("a", Type.Number(), 1); - const b = self.state("b", Type.Number(), 1, { + const a = self.state("a", num(), 1); + const b = self.state("b", num(), 1, { version, migrate: (old, from) => { migrated.push(`b ${from}->${version}`); @@ -294,7 +294,7 @@ describe("reload", () => { await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); log.push(`stop ${greeting}`); }); - const count = self.state("count", Type.Number(), 0, { version }); + const count = self.state("count", num(), 0, { version }); return { count, greet: (name: string) => `${greeting} ${name}` }; }), }); @@ -340,7 +340,7 @@ describe("reload", () => { open: () => void log.push(`open cache ${rev}`), close: () => void log.push(`close cache ${rev}`), }); - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { count, greet: (name: string) => `${lock.value} ${name}` }; }), }); @@ -523,21 +523,21 @@ describe("reload", () => { it("checks contracts at reload: forgotten bumps, live consumers, accepts adapters", async () => { const DocV1 = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.String() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: str() }, { open: true })) } } }, }); const DocBroken = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.Number() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: num() }, { open: true })) } } }, }); const DocV2 = remote("t.rdoc", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.Object({ body: Type.String() })) } } }, + variants: { main: { methods: { fetch: method(undefined, obj({ body: str() }, { open: true })) } } }, }); const host = quiet(); host.install({ @@ -590,14 +590,14 @@ describe("reload", () => { const log: string[] = []; const Sum = local("t.sum", { version: 1, - input: Type.Object({}), - members: { sum: Type.Object({ v: Type.Number() }) }, + input: obj({}, { open: true }), + members: { sum: obj({ v: num() }, { open: true }) }, }); const facet = (factor: number): Facet => ({ name: "sum", setup: (api) => api.serve(Sum, (_input, _principal, self) => { - const sum = self.state("sum", Type.Object({ v: Type.Number() }), { v: 0 }); + const sum = self.state("sum", obj({ v: num() }, { open: true }), { v: 0 }); self.on( "add", events, @@ -632,7 +632,7 @@ describe("reload", () => { it("keeps registry tables per code version", async () => { const host = quiet(); const Routes = defineRegistry("t.reload-routes"); - const Router = local("t.router", { version: 1, input: Type.Object({}) }); + const Router = local("t.router", { version: 1, input: obj({}, { open: true }) }); const facet = (value: string): Facet => ({ name: "router", setup: (api) => @@ -672,8 +672,8 @@ describe("reload", () => { await ref.ready; const Clock = local("t.clock", { version: 1, - input: Type.Object({}), - members: { now: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { now: method(undefined, num()) }, }); const clockFacet: Facet = { name: "clock", setup: (api) => api.serve(Clock, () => ({ now: () => 42 })) }; const consumer = (fail: boolean): Facet => ({ diff --git a/packages/chord/test/requests.test.ts b/packages/chord/test/requests.test.ts index fef1f2a30..5e3cc5abb 100644 --- a/packages/chord/test/requests.test.ts +++ b/packages/chord/test/requests.test.ts @@ -1,6 +1,6 @@ // Regression tests for resolved application requests (#1-#10), numbered as in their test names. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { @@ -36,19 +36,19 @@ function latch() { return { promise, open }; } -const View = Type.Object({ text: Type.String() }); +const View = obj({ text: str() }, { open: true }); describe("application requests", () => { it("#1 a held object's state member can be returned as this object's state member", async () => { const Conv = remote("t.conv", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { view: View }, methods: { set: method(Type.String()) } } }, + variants: { main: { state: { view: View }, methods: { set: method(str()) } } }, }); const Chat = remote("t.chat", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { view: View } } }, }); @@ -95,11 +95,11 @@ describe("application requests", () => { it("#2 facets reach their own host through FacetApi.root and FacetApi.registry", async () => { const Config = defineRegistry("t.config"); - const Settings = local("t.settings", { version: 1, input: Type.Object({}) }); + const Settings = local("t.settings", { version: 1, input: obj({}, { open: true }) }); const Reader = local("t.reader", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, str()) }, }); const captured: FacetApi[] = []; const facet = (value: string): Facet => ({ @@ -135,8 +135,8 @@ describe("application requests", () => { it("#3 a listener keeps its subscription across reloads when resource and select are unchanged", async () => { const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Array(Type.String()) }, + input: obj({}, { open: true }), + members: { seen: arr(str()) }, }); // A source that accepts one subscriber at a time (like a Durable watch). let subscribers = 0; @@ -155,7 +155,7 @@ describe("application requests", () => { name: "watcher", setup: (api) => api.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Array(Type.String()), []); + const seen = self.state("seen", arr(str()), []); const watchResource = self.resource("watch", { version: 1, open: () => single }); // A new adapter object per call, and a new arrow per factory run. self.on( @@ -181,7 +181,7 @@ describe("application requests", () => { }); it("#3 a changed select resubscribes", async () => { - const Watcher = local("t.watcher2", { version: 1, input: Type.Object({}) }); + const Watcher = local("t.watcher2", { version: 1, input: obj({}, { open: true }) }); let subscribes = 0; const source = { subscribe: () => { @@ -210,10 +210,10 @@ describe("application requests", () => { it("#4 replace diffs a new revision that shares structure; unrelated values replace", async () => { const Mirror = local("t.mirror", { version: 1, - input: Type.Object({}), - members: { doc: Type.Unknown(), set: method(Type.Unknown()) }, + input: obj({}, { open: true }), + members: { doc: unknown(), set: method(unknown()) }, }); - const Doc = Type.Object({ entries: Type.Array(Type.String()), status: Type.String() }); + const Doc = obj({ entries: arr(str()), status: str() }, { open: true }); let lastOptions: { diff?: boolean } | undefined; const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Mirror, (_input, _principal, self) => { @@ -256,9 +256,9 @@ describe("application requests", () => { it("#5 roots of a child that connected behind the server are admitted in the server's code version", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { codeVersion: method(undefined, Type.Number()) } } }, + variants: { main: { methods: { codeVersion: method(undefined, num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); const echoFacet: Facet = { @@ -293,7 +293,7 @@ describe("application requests", () => { it("#6 self.expose: an opaque member backed by a resource, readable once live", async () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ readonly queries: string[] }>("t.db@1") }, }); const gate = latch(); @@ -315,7 +315,7 @@ describe("application requests", () => { await ref.ready; assert.equal(ref.db, native); - const Other = local("t.other", { version: 1, input: Type.Object({}) }); + const Other = local("t.other", { version: 1, input: obj({}, { open: true }) }); host.serve(Other, (_input, _principal, self) => { self.expose({ key: "x", value: 1 }); return {}; @@ -326,9 +326,9 @@ describe("application requests", () => { it("#7 a cluster reload reports what each child prepared", async () => { const Greeter = remote("t.greeter", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), - variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + variants: { main: { methods: { greet: method(str(), str()) } } }, }); const greeter = (text: string): Facet => ({ name: "greeter", @@ -362,13 +362,13 @@ describe("application requests", () => { it("#8 a held remote reference keeps its state without listeners; a later subscriber sees the current value", async () => { const Counter = remote("t.count8", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { set: method(Type.Number()) } } }, + variants: { main: { state: { n: num() }, methods: { set: method(num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Counter, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 0); + const n = self.state("n", num(), 0); return { variant: "main", n, @@ -404,7 +404,7 @@ describe("application requests", () => { it("#9 references report status changes: live, loading while a link reconnects, gone, released", async () => { const Thing = remote("t.thing9", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: {} }, }); @@ -449,9 +449,9 @@ describe("application requests", () => { const log: string[] = []; const Remote = remote("t.remote10", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { note: method(Type.String()) } } }, + variants: { main: { methods: { note: method(str()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Remote, () => ({ @@ -462,10 +462,10 @@ describe("application requests", () => { })); const Proc = local("t.proc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { tracker: opaque>("t.tracker@1") }, }); - const Chat = local("t.chat10", { version: 1, input: Type.Object({}) }); + const Chat = local("t.chat10", { version: 1, input: obj({}, { open: true }) }); const [down, up] = memoryLinkPair(); server.accept(down, { principal: { worker: "w" }, kind: "host", name: "w" }); const worker = createHost({ diff --git a/packages/chord/test/review.test.ts b/packages/chord/test/review.test.ts index 554a24097..418d592ea 100644 --- a/packages/chord/test/review.test.ts +++ b/packages/chord/test/review.test.ts @@ -1,7 +1,6 @@ // Regression tests for Sol's phase 1a review (findings #1-#11) and the requested race tests. import assert from "node:assert/strict"; -import { Type } from "typebox"; -import { Value } from "typebox/value"; +import { arr, bool, compile, num, obj, opt, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -18,6 +17,7 @@ import { } from "../src/index.ts"; import { fits } from "../src/schema.ts"; import { CountingLink } from "./fixtures/counting.ts"; +import { typedLit } from "./fixtures/schemas.ts"; const tick = () => new Promise((resolve) => setImmediate(resolve)); const ticks = async (n = 10) => { @@ -51,8 +51,8 @@ function emitter(options: { failSubscribe?: boolean } = {}) { const Item = local("t.item", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { version: method(undefined, Type.String()) }, + input: obj({ id: str() }, { open: true }), + members: { version: method(undefined, str()) }, }); describe("review: reload commit", () => { @@ -82,15 +82,15 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, dev: true, onError: () => {} }); const V1 = remote("t.counted", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ n: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ n: num() }, { open: true }) } } }, }); const V2 = remote("t.counted", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ total: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ total: num() }, { open: true }) } } }, }); host.install({ name: "counted", @@ -99,7 +99,7 @@ describe("review: reload commit", () => { V1, (_input, _principal, self) => ({ variant: "main", - count: self.state("count", Type.Object({ n: Type.Number() }), { n: 3 }), + count: self.state("count", obj({ n: num() }, { open: true }), { n: 3 }), }), { linger: 60_000 }, ), @@ -117,7 +117,7 @@ describe("review: reload commit", () => { variant: "main", count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -160,8 +160,8 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Other = local("t.other", { version: 1, - input: Type.Object({}), - members: { version: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { version: method(undefined, str()) }, }); const facet = (version: string): Facet => ({ name: "pair", @@ -184,8 +184,8 @@ describe("review: reload commit", () => { describe("review: leases, invocations, activation", () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), - members: { count: Type.Unknown() }, + input: obj({}, { open: true }), + members: { count: unknown() }, }); it("#4 a running listener holds a lease: the gate waits for it before migrating", async () => { @@ -197,7 +197,7 @@ describe("review: leases, invocations, activation", () => { name: "store", setup: (api) => api.serve(Store, (_input, _principal, self) => { - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 1 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 1 }); self.on("slow", events, async () => { await go.promise; seen.push(count.value); @@ -218,7 +218,7 @@ describe("review: leases, invocations, activation", () => { api.serve(Store, (_input, _principal, self) => ({ count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -238,8 +238,8 @@ describe("review: leases, invocations, activation", () => { it("#5 disposal waits for running methods before closing resources (released and forced)", async () => { const Worker = local("t.busy", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { work: method(undefined, Type.Boolean()) }, + input: obj({ id: str() }, { open: true }), + members: { work: method(undefined, bool()) }, }); const log: string[] = []; let go = latch(); @@ -291,7 +291,9 @@ describe("review: leases, invocations, activation", () => { const host = createHost({ self: { kind: "host" }, onError: (error) => errors.push(error) }); const Routes = defineRegistry("t.review-routes"); const log: string[] = []; - const types = ["t.eager-a", "t.eager-b"].map((name) => local(name, { version: 1, input: Type.Object({}) })); + const types = ["t.eager-a", "t.eager-b"].map((name) => + local(name, { version: 1, input: obj({}, { open: true }) }), + ); for (const type of types) { host.serve( type, @@ -321,7 +323,7 @@ describe("review: leases, invocations, activation", () => { let runs = 0; host.serve(Store, (_input, _principal, self) => { runs++; - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); self.resource("r", { version: 1, open: () => void log.push("open"), @@ -345,8 +347,8 @@ describe("review: leases, invocations, activation", () => { describe("review: prepare isolation and contracts", () => { const Db = local("t.db", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.Unknown()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, unknown()) }, }); it("#7 a prepare rerun cannot touch retained resources, even open ones", async () => { @@ -372,7 +374,7 @@ describe("review: prepare isolation and contracts", () => { it("#7 objects of types a reload adds start listeners and tasks only at commit", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Fresh = local("t.fresh", { version: 1, input: Type.Object({}) }); + const Fresh = local("t.fresh", { version: 1, input: obj({}, { open: true }) }); host.install({ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v1" })) }); await Promise.all(["a", "b"].map((id) => host.hold(Item, { id }).ready)); const plugins = (fail: boolean): Facet[] => [ @@ -408,20 +410,20 @@ describe("review: prepare isolation and contracts", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const P1 = remote("t.provider", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { get: method(undefined, Type.String()) } } }, + variants: { main: { methods: { get: method(undefined, str()) } } }, }); const P2 = remote("t.provider", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.String()) } } }, + variants: { main: { methods: { fetch: method(undefined, str()) } } }, }); const Consumer = local("t.consumer", { version: 1, - input: Type.Object({}), - members: { ask: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { ask: method(undefined, str()) }, }); host.install({ name: "both", @@ -493,7 +495,7 @@ describe("review: prepare isolation and contracts", () => { it("#10 retirement closes dropped resources before releasing dropped holds", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Dep = local("t.dep", { version: 1, input: Type.Object({}) }); + const Dep = local("t.dep", { version: 1, input: obj({}, { open: true }) }); const facet = (withDecls: boolean): Facet => ({ name: "item", setup: (api) => { @@ -535,48 +537,49 @@ describe("review: prepare isolation and contracts", () => { assert.deepEqual(log, ["own closed", "dep closed"]); }); - it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with Value.Check", () => { + it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with compile", () => { const cases = [ - [Type.String(), Type.String({ not: { const: "x" } }), "x"], - [Type.Array(Type.Number()), Type.Array(Type.Number(), { contains: Type.Literal(1) }), [2]], + [str(), { ...str(), not: { const: "x" } }, "x"], + [arr(num()), { ...arr(num()), contains: typedLit(1) }, [2]], [ - Type.Object({ a: Type.String() }), - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), + obj({ a: str() }, { open: true }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, { a: "s", b: "t" }, ], [ - Type.Object({ x: Type.String() }), - Type.Object({}, { patternProperties: { "^x$": Type.Number() } }), + obj({ x: str() }, { open: true }), + { ...obj({}, { open: true }), patternProperties: { "^x$": num() } }, { x: "s" }, ], - [Type.Number(), Type.Intersect([Type.Number(), Type.Number({ minimum: 1 })]), 0], + [num(), { allOf: [num(), num({ minimum: 1 })] }, 0], [ - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), - Type.Object({ a: Type.String(), b: Type.Optional(Type.String()) }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, + obj({ a: str(), b: opt(str()) }, { open: true }), { a: "", b: 1 }, ], ] as const; for (const [sub, sup, value] of cases) { - assert.ok(Value.Check(sub, value) && !Value.Check(sup, value), `bad counterexample ${JSON.stringify(value)}`); + assert.ok( + compile(sub).check(value) && !compile(sup).check(value), + `bad counterexample ${JSON.stringify(value)}`, + ); assert.equal(fits(sub, sup), false, `fits(${JSON.stringify(sub)}, ${JSON.stringify(sup)})`); } // Additive changes still fit. - assert.ok( - fits(Type.Object({ a: Type.String() }), Type.Object({ a: Type.String(), b: Type.Optional(Type.Number()) })), - ); - assert.ok(fits(Type.Object({ a: Type.String(), extra: Type.Number() }), Type.Object({ a: Type.String() }))); + assert.ok(fits(obj({ a: str() }, { open: true }), obj({ a: str(), b: opt(num()) }, { open: true }))); + assert.ok(fits(obj({ a: str(), extra: num() }, { open: true }), obj({ a: str() }, { open: true }))); }); it("#11 arguments are stripped to what the consumer knows, then to what the provider knows", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Strict = remote("t.strict", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), variants: { main: { methods: { - echo: method(Type.Object({ a: Type.String() }, { additionalProperties: false }), Type.Unknown()), + echo: method(obj({ a: str() }), unknown()), }, }, }, @@ -590,13 +593,13 @@ describe("review: prepare isolation and contracts", () => { describe("review: ctx races", () => { const Target = local("t.target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { hit: method() }, }); it("an aborted task ctx cannot start fresh calls", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Runner = local("t.runner", { version: 1, input: Type.Object({}) }); + const Runner = local("t.runner", { version: 1, input: obj({}, { open: true }) }); let hits = 0; const go = latch(); let outcome: unknown; @@ -650,7 +653,7 @@ describe("review: ctx races", () => { it("a remote reference refuses a foreign ctx and a ctx not made by Chord, without calling", async () => { const RemoteTarget = remote("t.remote-target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { state: {}, methods: { hit: method() } } }, }); diff --git a/packages/chord/test/sqlite/cloudflare.test.ts b/packages/chord/test/sqlite/cloudflare.test.ts index 0e7521d25..1202d8f49 100644 --- a/packages/chord/test/sqlite/cloudflare.test.ts +++ b/packages/chord/test/sqlite/cloudflare.test.ts @@ -1,7 +1,12 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../src/context/index.ts"; +import { sqliteBackend } from "../../src/next/store/backend.ts"; +import { refusalOf } from "../../src/next/store/refusal.ts"; +import type { StagedWrite } from "../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../src/next/store/store.ts"; import { DurableObjectSqliteDatabase } from "../../src/sqlite/cloudflare.ts"; -import { durableObjectStorage } from "./helpers.ts"; +import { durableObjectStorage, RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Durable Object SQLite driver", () => { it("binds and returns BLOB values as Uint8Array", async () => { @@ -14,6 +19,15 @@ describe("Durable Object SQLite driver", () => { }); }); + it("reports the rows a statement changed, without index writes", async () => { + const database = new DurableObjectSqliteDatabase(durableObjectStorage()); + expect(await runChanges(database)).toEqual(RUN_CHANGES); + const inTransaction = await database.transaction((handle) => + handle.runCounted("INSERT INTO changes_probe VALUES (?, ?, ?)", "t", "t/1", 1), + ); + expect(inTransaction).toBe(1); + }); + it("reads only the cursor's first row for get", async () => { const operations: string[] = []; const database = new DurableObjectSqliteDatabase( @@ -114,3 +128,72 @@ describe("Durable Object SQLite driver", () => { expect(sqlite.prepare("SELECT id FROM items").all()).toEqual([]); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** The test storage, its `transaction` failing after the closure resolved (a failed commit) when asked. */ + function failingCommit(): { storage: ReturnType; fail(): void } { + const storage = durableObjectStorage(); + const transaction = storage.transaction.bind(storage); + let failing = false; + return { + storage: { + ...storage, + transaction: async (closure) => { + const result = await transaction(closure); + if (!failing) return result; + failing = false; + throw new Error("commit failed"); + }, + }, + fail: () => { + failing = true; + }, + }; + } + + it("rejects with the callback's own error after a rollback, another one when the commit fails", async () => { + const { storage, fail } = failingCommit(); + const database = new DurableObjectSqliteDatabase(storage); + const own = new Error("refused"); + await expect(database.transaction(async () => Promise.reject(own))).rejects.toBe(own); + fail(); + await expect(database.transaction(async () => 1)).rejects.toThrow("commit failed"); + }); + + it("a store on it tells a CHECK refusal by its message, its error having no extended code", async () => { + const storage = durableObjectStorage(); + const exec = storage.sql.exec.bind(storage.sql); + // The Durable Object binding's errors carry a message only. + const withoutCodes: typeof storage = { + ...storage, + sql: { + exec: (query, ...bindings) => { + try { + return exec(query, ...bindings); + } catch (error) { + throw new Error((error as Error).message); + } + }, + }, + }; + const store = (await openStore( + sqliteBackend(new DurableObjectSqliteDatabase(withoutCodes)), + BACKGROUND_CONTEXT, + )) as ChordStore; + await store.read( + (executor) => executor.exec('CREATE TABLE "checked" (n INTEGER CHECK (n < 10))'), + BACKGROUND_CONTEXT, + ); + const refused: StagedWrite = { + order: "write", + apply: async (step) => { + await step.run('INSERT INTO "checked" VALUES (11)'); + return []; + }, + }; + const fence = { admit: () => true, adopt: () => {} }; + const error = await store.commit([refused], fence, BACKGROUND_CONTEXT).catch((thrown: unknown) => thrown); + expect(refusalOf(error, "x")).toMatchObject({ kind: "check" }); + expect(await store.read(async () => 1, BACKGROUND_CONTEXT)).toBe(1); + }); +}); diff --git a/packages/chord/test/sqlite/helpers.ts b/packages/chord/test/sqlite/helpers.ts index 042696e45..48e855eab 100644 --- a/packages/chord/test/sqlite/helpers.ts +++ b/packages/chord/test/sqlite/helpers.ts @@ -1,5 +1,6 @@ import { DatabaseSync } from "node:sqlite"; import type { DurableObjectSqliteStorage } from "../../src/sqlite/cloudflare.ts"; +import type { SqliteExecutor } from "../../src/sqlite/index.ts"; type SqlValue = ArrayBuffer | string | number | null; @@ -63,3 +64,34 @@ export function durableObjectStorage( }, }; } + +/** + * The rows each kind of statement changes, as `runCounted` reports them: on a table with two indexes, so a driver that counted + * index writes (a Durable Object cursor's `rowsWritten`) would report more. + */ +export async function runChanges(database: SqliteExecutor): Promise> { + await database.exec( + "CREATE TABLE changes_probe (id TEXT PRIMARY KEY, path TEXT NOT NULL, n INTEGER NOT NULL) STRICT, WITHOUT ROWID", + ); + await database.exec("CREATE INDEX changes_probe_path ON changes_probe (path, id)"); + await database.exec("CREATE INDEX changes_probe_n ON changes_probe (n, id)"); + for (let i = 0; i < 10; i++) await database.run("INSERT INTO changes_probe VALUES (?, ?, ?)", `k${i}`, `a/${i}`, i); + const changes = (sql: string, ...params: (string | number)[]) => database.runCounted(sql, ...params); + return { + insert: await changes("INSERT INTO changes_probe VALUES (?, ?, ?)", "x", "a/x", 99), + update: await changes("UPDATE changes_probe SET path = ? WHERE id = ?", "b/x", "x"), + none: await changes("UPDATE changes_probe SET n = 0 WHERE id = ?", "absent"), + bulk: await changes("UPDATE changes_probe SET path = 'c/' || substr(path, 3) WHERE path >= 'a/' AND path < 'a0'"), + upsert: await changes( + "INSERT INTO changes_probe VALUES (?, ?, ?) ON CONFLICT (id) DO UPDATE SET n = excluded.n", + "x", + "b/x", + 5, + ), + delete: await changes("DELETE FROM changes_probe WHERE id = ?", "x"), + bulkDelete: await changes("DELETE FROM changes_probe WHERE path >= 'c/' AND path < 'c0'"), + }; +} + +/** What `runChanges` reports on every driver. */ +export const RUN_CHANGES = { insert: 1, update: 1, none: 0, bulk: 10, upsert: 1, delete: 1, bulkDelete: 10 }; diff --git a/packages/chord/test/sqlite/node.test.ts b/packages/chord/test/sqlite/node.test.ts index 11098d1c5..8666b3f5d 100644 --- a/packages/chord/test/sqlite/node.test.ts +++ b/packages/chord/test/sqlite/node.test.ts @@ -1,6 +1,8 @@ -import { describe, expect, it } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it, vi } from "vitest"; import type { SqliteExecutor } from "../../src/sqlite/index.ts"; -import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Node SQLite driver", () => { it("configures connection defaults and accepts explicit settings", async () => { @@ -18,6 +20,35 @@ describe("Node SQLite driver", () => { } }); + it("keeps the most recently used prepared statements and prepares evicted ones again", async () => { + const database = await openNodeSqliteDatabase(":memory:"); + const prepare = vi.spyOn(DatabaseSync.prototype, "prepare"); + try { + for (let i = 0; i < 300; i++) await database.get(`SELECT ${i} AS n`); + const prepared = prepare.mock.calls.length; + expect(await database.get("SELECT 299 AS n")).toEqual({ n: 299 }); + expect(prepare.mock.calls.length).toBe(prepared); + expect(await database.get("SELECT 0 AS n")).toEqual({ n: 0 }); + expect(prepare.mock.calls.length).toBe(prepared + 1); + } finally { + prepare.mockRestore(); + await database.close(); + } + }); + + it("reports the rows a statement changed, without index writes", async () => { + const database = await openNodeSqliteDatabase(":memory:"); + try { + expect(await runChanges(database)).toEqual(RUN_CHANGES); + const inTransaction = await database.transaction((handle) => + handle.runCounted("INSERT INTO changes_probe VALUES (?, ?, ?)", "t", "t/1", 1), + ); + expect(inTransaction).toBe(1); + } finally { + await database.close(); + } + }); + it("commits work done through the transaction handle and closes idempotently", async () => { const database = await openNodeSqliteDatabase(":memory:"); await database.transaction(async (transaction) => { @@ -167,3 +198,48 @@ describe("Node SQLite driver", () => { await database.close(); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** A facade on `:memory:` whose next `statement` throws (and a `ROLLBACK` still rolls back). */ + function faulty(statement?: "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK") { + const database = new DatabaseSync(":memory:"); + const exec = database.exec.bind(database); + let failing = statement; + database.exec = (sql: string): void => { + if (sql !== failing) { + exec(sql); + return; + } + failing = undefined; + if (sql === "ROLLBACK") exec(sql); + throw new Error(`${sql} failed`); + }; + return new NodeSqliteDatabase(database); + } + + it("rejects with the callback's own error after a rollback: the caller knows it never committed", async () => { + const database = faulty(); + const own = new Error("refused"); + await database.exec("CREATE TABLE t (n INTEGER)"); + const outcome = database.transaction(async (handle) => { + await handle.run("INSERT INTO t VALUES (1)"); + throw own; + }); + await expect(outcome).rejects.toBe(own); + expect(await database.all("SELECT n FROM t")).toEqual([]); + }); + + it.each(["BEGIN IMMEDIATE", "COMMIT"] as const)("rejects with another error when %s fails", async (statement) => { + const database = faulty(statement); + const outcome = database.transaction(async () => 1); + await expect(outcome).rejects.toThrow(`${statement} failed`); + }); + + it("rejects with an AggregateError, not the callback's error, when its ROLLBACK fails", async () => { + const database = faulty("ROLLBACK"); + const own = new Error("refused"); + const error = await database.transaction(async () => Promise.reject(own)).catch((thrown: unknown) => thrown); + expect(error).toBeInstanceOf(AggregateError); + expect(error).not.toBe(own); + }); +}); diff --git a/packages/chord/test/state-init.test.ts b/packages/chord/test/state-init.test.ts index 77f101d5e..24d086aac 100644 --- a/packages/chord/test/state-init.test.ts +++ b/packages/chord/test/state-init.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, bool, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, createHost, type Facet, isChordError, local, method } from "../src/index.ts"; @@ -19,16 +19,16 @@ function latch() { const Notes = local("t.notes", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { - notes: Type.Array(Type.String()), - count: Type.Number(), - has: method(Type.String(), Type.Boolean()), - add: method(Type.String()), + notes: arr(str()), + count: num(), + has: method(str(), bool()), + add: method(str()), }, }); -const NotesSchema = Type.Array(Type.String()); +const NotesSchema = arr(str()); describe("state init", () => { it("runs after the resources are open and before the object goes live: nothing sees another value", async () => { @@ -124,7 +124,7 @@ describe("state init", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Source = local("t.source", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { all: method(undefined, NotesSchema) }, }); host.serve(Source, () => ({ all: () => ["from source"] })); @@ -155,7 +155,7 @@ describe("state init", () => { return ["loaded"]; }, }); - const counter = self.state("counter", Type.Number(), 0, { version: spec.countVersion ?? 1 }); + const counter = self.state("counter", num(), 0, { version: spec.countVersion ?? 1 }); const extra = spec.extra; const more = extra === undefined diff --git a/packages/chord/test/state.test.ts b/packages/chord/test/state.test.ts index 656ac0601..2f48c8799 100644 --- a/packages/chord/test/state.test.ts +++ b/packages/chord/test/state.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { type Ctx, createHost, isChordError, local, method, type StateEvent } from "../src/index.ts"; @@ -22,15 +22,15 @@ function emitter() { }; } -const Item = Type.Object({ id: Type.Number(), label: Type.String() }); -const Items = Type.Object({ items: Type.Array(Item), count: Type.Number() }); +const Item = obj({ id: num(), label: str() }, { open: true }); +const Items = obj({ items: arr(Item), count: num() }, { open: true }); const List = local("t.list", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { list: Items, - labels: Type.Array(Type.String()), + labels: arr(str()), push: method(Item), set: method(Items), }, @@ -84,11 +84,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Counter = local("t.prim", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), inc: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), inc: method(undefined, num()) }, }); host.serve(Counter, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); return { n: nState, inc: (ctx: Ctx) => { @@ -117,11 +117,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Box = local("t.box", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { put: method() }, }); host.serve(Box, (_input, _principal, self) => { - const list = self.state("list", Type.Array(Type.Number()), []); + const list = self.state("list", arr(num()), []); return { put: (ctx: Ctx) => { // The classic slip: an expression-bodied arrow returns push()'s length. @@ -133,10 +133,10 @@ describe("state cells", () => { }); it("validates initial values always and writes only in dev mode", async () => { - const Bad = local("t.bad-initial", { version: 1, input: Type.Object({}) }); + const Bad = local("t.bad-initial", { version: 1, input: obj({}, { open: true }) }); const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Bad, (_input, _principal, self) => { - self.state("x", Type.Object({ n: Type.Number() }), { n: "no" } as never); + self.state("x", obj({ n: num() }, { open: true }), { n: "no" } as never); return {}; }); await assert.rejects(host.hold(Bad, {}).ready, /initial value of state x/); @@ -157,13 +157,13 @@ describe("state cells", () => { it("rejects writes from inside a factory and placeholder use before activation", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Writer = local("t.writer", { version: 1, input: Type.Object({}) }); + const Writer = local("t.writer", { version: 1, input: obj({}, { open: true }) }); let captured: Ctx | undefined; await host.root((ctx) => { captured = ctx; }); host.serve(Writer, (_input, _principal, self) => { - const x = self.state("x", Type.Object({}), {}); + const x = self.state("x", obj({}, { open: true }), {}); const r = self.resource("r", { version: 1, open: () => 1 }); assert.throws( () => r.value, @@ -180,13 +180,13 @@ describe("state cells", () => { const events = emitter(); const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Object({ sum: Type.Number() }) }, + input: obj({}, { open: true }), + members: { seen: obj({ sum: num() }, { open: true }) }, }); const roots = new Set(); let taskAborted = false; host.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Object({ sum: Type.Number() }), { sum: 0 }); + const seen = self.state("seen", obj({ sum: num() }, { open: true }), { sum: 0 }); const sourceResource = self.resource("source", { version: 1, open: () => events }); self.on( "onEvent", @@ -223,11 +223,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Echo = local("t.reentrant", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), d: Type.Number(), set: method(Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), d: num(), set: method(num()) }, }); host.serve(Echo, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); const d = self.derive([nState], (n) => n * 2); // Subscribed first: re-enters the write while revision 1 is being published. self.on("bump", nState, (n, ctx) => { diff --git a/packages/chord/test/telemetry-sqlite-store.test.ts b/packages/chord/test/telemetry-sqlite-store.test.ts index 4da872546..34e142c57 100644 --- a/packages/chord/test/telemetry-sqlite-store.test.ts +++ b/packages/chord/test/telemetry-sqlite-store.test.ts @@ -59,6 +59,7 @@ describe("SQLite telemetry reads", () => { const counted: SqliteExecutor = { exec: (sql) => tx.exec(sql), run: (sql, ...params) => tx.run(sql, ...params), + runCounted: (sql, ...params) => tx.runCounted(sql, ...params), get: (sql: string, ...params: SqliteValue[]) => tx.get(sql, ...params), all: (sql: string, ...params: SqliteValue[]) => { queries++; diff --git a/packages/chord/test/tutorial.test.ts b/packages/chord/test/tutorial.test.ts index 55c0dfe17..72f98e1dd 100644 --- a/packages/chord/test/tutorial.test.ts +++ b/packages/chord/test/tutorial.test.ts @@ -1,7 +1,7 @@ // example:imports import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, @@ -51,23 +51,23 @@ function checkExamples(section: string): void { // example:02-contract const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); // end-example:02-contract // example:03-contract -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -107,13 +107,13 @@ const todoFactory = todoStore(); // example:05-views const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -190,13 +190,13 @@ it("04: holds, asynchronous resources and release", { timeout: 5000 }, async () // example:04-database const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -306,8 +306,8 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( // example:07 const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -316,7 +316,7 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -345,7 +345,7 @@ it("08: plugins contribute registry entries and expose callbacks", { timeout: 50 onTestFinished(() => host.close()); // example:08 const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); - const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); + const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -444,7 +444,7 @@ it("09: compatible reload, migration and an old-contract adapter", { timeout: 50 const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { diff --git a/packages/chord/test/types.test.ts b/packages/chord/test/types.test.ts index f64f0ab98..9df86f431 100644 --- a/packages/chord/test/types.test.ts +++ b/packages/chord/test/types.test.ts @@ -1,25 +1,25 @@ // Entry 8: reference types. State members are plain values (a reference you hold has its state); a remote reference is a // union discriminated by `variant`, so another variant's members are reachable only after narrowing. Checked by tsc. -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, expectTypeOf, it } from "vitest"; import { type Ctx, type Host, hostNamed, local, method, type Readable, type Ref, remote } from "../src/index.ts"; const Doc = remote("t.e8-types-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - reader: { state: { text: Type.String() } }, - editor: { state: { text: Type.String() }, methods: { edit: method(Type.String()) } }, + reader: { state: { text: str() } }, + editor: { state: { text: str() }, methods: { edit: method(str()) } }, }, }); const Single = remote("t.e8-types-single", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { bump: method() } } }, + variants: { main: { state: { n: num() }, methods: { bump: method() } } }, }); -const Local = local("t.e8-types-local", { version: 1, input: Type.Object({}), members: { n: Type.Number() } }); +const Local = local("t.e8-types-local", { version: 1, input: obj({}, { open: true }), members: { n: num() } }); describe("reference types", () => { it("state members are T; other variants' members need narrowing; gets are awaited", () => { diff --git a/packages/chord/test/variants.test.ts b/packages/chord/test/variants.test.ts index 363f50361..3b8c2359d 100644 --- a/packages/chord/test/variants.test.ts +++ b/packages/chord/test/variants.test.ts @@ -1,7 +1,7 @@ // Entry 8: a binding applies a shape only once its new state members have snapshots; members outside the variant // throw `not_in_variant`; a re-hold after a link loss keeps the last values readable and sends one fresh unit. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -24,12 +24,12 @@ async function until(check: () => boolean, attempts = 200): Promise { const Doc = remote("t.e8-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - a: { state: { x: Type.Number() }, methods: { bump: method() } }, - b: { state: { x: Type.Number(), y: Type.String() }, methods: { bump: method() } }, - c: { state: { y: Type.String() }, methods: { bump: method() } }, + a: { state: { x: num() }, methods: { bump: method() } }, + b: { state: { x: num(), y: str() }, methods: { bump: method() } }, + c: { state: { y: str() }, methods: { bump: method() } }, }, }); @@ -39,8 +39,8 @@ function docFacet(variant: "a" | "b" | "c"): Facet { name: "doc", setup: (api) => api.serve(Doc, (_input, _principal, self) => { - const x = self.state("x", Type.Number(), 1); - const y = self.state("y", Type.String(), "why"); + const x = self.state("x", num(), 1); + const y = self.state("y", str(), "why"); const bump = (ctx: Ctx) => { x.replace(ctx, x.value + 1); }; diff --git a/packages/chord/test/watch.test.ts b/packages/chord/test/watch.test.ts index 76b54401b..d3b9c2416 100644 --- a/packages/chord/test/watch.test.ts +++ b/packages/chord/test/watch.test.ts @@ -1,7 +1,7 @@ // Tests for Chord's watch handles: the shared lifecycle through `Watch`, and what `NativeWatch` does differently. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import { afterEach, describe, expect, expectTypeOf, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { BACKGROUND_CONTEXT, type Context, @@ -12,7 +12,7 @@ import { import { applyImmutable, type Op, track } from "../src/delta/index.ts"; import { MAX_PENDING_OPS } from "../src/delta/pending.ts"; import type { JsonValue } from "../src/json.ts"; -import { NativeWatch, Watch, type WatchEnd } from "../src/watch.ts"; +import { NativeWatch, type NativeWatchCall, Watch, type WatchCall, type WatchEnd } from "../src/watch.ts"; const context = BACKGROUND_CONTEXT; @@ -25,6 +25,28 @@ function deferred(): { readonly promise: Promise; readonly resolve: () => return { promise, resolve: () => resolve() }; } +/** A source's frame that is not a retirement. */ +function frame(value: T, ops: readonly Op[]): WatchValue { + return { kind: "value", value, ops, retired: false }; +} + +/** A source's retirement frame. */ +function retirement(value: T, ops: readonly Op[] = []): WatchValue { + return { kind: "value", value, ops, retired: true }; +} + +type WatchValue = Extract, { kind: "value" }>; + +/** A listener for value calls only: a `failed` call throws. */ +function onValue( + listener: (call: Extract, context: Context) => Promise, +): (call: C, context: Context) => Promise { + return (call, deliveryContext) => { + if (call.kind !== "value") throw new Error(`Unexpected ${call.kind} call`); + return listener(call as Extract, deliveryContext); + }; +} + /** A watch whose detach and reports are recorded. */ function recordedWatch(value: T) { const recorded = { detached: 0, reported: [] as unknown[] }; @@ -39,10 +61,12 @@ function recordedWatch(value: T) { /** A started watch collecting its calls. */ function collectingWatch(value: T) { const { watch, recorded } = recordedWatch(value); - const frames: Array<{ value: T; ops: readonly Op[] }> = []; - watch.start(async (delivered, ops) => { - frames.push({ value: delivered, ops }); - }); + const frames: Array<{ value: T; ops: readonly Op[]; retired?: true }> = []; + watch.start( + onValue(async ({ value: delivered, ops, retired }) => { + frames.push(retired ? { value: delivered, ops, retired } : { value: delivered, ops }); + }), + ); return { watch, recorded, frames }; } @@ -69,23 +93,24 @@ const collect = (() => { describe("Watch delivery", () => { it("keeps the acquisition value until start, then delivers the frames since in one call", async () => { const { watch } = recordedWatch({ n: 0, text: "" }); - watch.advance({ n: 1, text: "" }, [["s", ["n"], 1]], context); + watch.advance(frame({ n: 1, text: "" }, [["s", ["n"], 1]]), context); watch.advance( - { n: 2, text: "a" }, - [ + frame({ n: 2, text: "a" }, [ ["s", ["n"], 2], ["a", ["text"], "a"], - ], + ]), context, ); - watch.advance({ n: 2, text: "ab" }, [["a", ["text"], "b"]], context); + watch.advance(frame({ n: 2, text: "ab" }, [["a", ["text"], "b"]]), context); await flush(); expect(watch.value).toEqual({ n: 0, text: "" }); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); expect(frames).toEqual([]); await flush(); expect(frames).toEqual([ @@ -103,14 +128,16 @@ describe("Watch delivery", () => { it("passes a frame's ops on by identity to a listener that keeps up", async () => { const { watch } = recordedWatch(0); const delivered: Array = []; - watch.start(async (_value, ops) => { - delivered.push(ops); - }); + watch.start( + onValue(async ({ ops }) => { + delivered.push(ops); + }), + ); const first: readonly Op[] = [["r", 1]]; const second: readonly Op[] = [["r", 2]]; - watch.advance(1, first, context); + watch.advance(frame(1, first), context); await flush(); - watch.advance(2, second, context); + watch.advance(frame(2, second), context); await flush(); expect(delivered).toHaveLength(2); expect(delivered[0]).toBe(first); @@ -119,7 +146,7 @@ describe("Watch delivery", () => { it("never invokes the listener inline", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); expect(frames).toEqual([]); await flush(); expect(frames).toEqual([{ value: 1, ops: [["r", 1]] }]); @@ -130,16 +157,18 @@ describe("Watch delivery", () => { const gate = deferred(); const calls: string[] = []; const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - calls.push(`start ${JSON.stringify(value)}`); - frames.push({ value, ops }); - if (frames.length === 1) await gate.promise; - calls.push(`end ${JSON.stringify(value)}`); - }); - watch.advance({ text: "a" }, [["a", ["text"], "a"]], context); + watch.start( + onValue(async ({ value, ops }) => { + calls.push(`start ${JSON.stringify(value)}`); + frames.push({ value, ops }); + if (frames.length === 1) await gate.promise; + calls.push(`end ${JSON.stringify(value)}`); + }), + ); + watch.advance(frame({ text: "a" }, [["a", ["text"], "a"]]), context); await flush(); - watch.advance({ text: "ab" }, [["a", ["text"], "b"]], context); - watch.advance({ text: "abc" }, [["a", ["text"], "c"]], context); + watch.advance(frame({ text: "ab" }, [["a", ["text"], "b"]]), context); + watch.advance(frame({ text: "abc" }, [["a", ["text"], "c"]]), context); await flush(); expect(calls).toEqual(['start {"text":"a"}']); gate.resolve(); @@ -153,11 +182,11 @@ describe("Watch delivery", () => { const key = createContextKey("frame"); const controller = new AbortController(); const delivered: Context[] = []; - watch.start(async (_value, _ops, deliveryContext) => { + watch.start(async (_change, deliveryContext) => { delivered.push(deliveryContext); }); - watch.advance(1, [["r", 1]], withContextValue(key, "first", context)); - watch.advance(2, [["r", 2]], withContextValue(key, "second", withAbortSignal(controller.signal, context))); + watch.advance(frame(1, [["r", 1]]), withContextValue(key, "first", context)); + watch.advance(frame(2, [["r", 2]]), withContextValue(key, "second", withAbortSignal(controller.signal, context))); await flush(); expect(delivered).toHaveLength(1); expect(delivered[0]?.value(key)).toBe("second"); @@ -180,18 +209,20 @@ describe("Watch with a slow listener", () => { edit(change.state as Doc); const prepared = change.prepare(); tracker.adopt(prepared); - watch.advance(tracker.value, prepared.ops, context); + watch.advance(frame(tracker.value, prepared.ops), context); }; let replica: Doc = tracker.value; const calls: Array<{ commits: number; ops: number }> = []; let commits = 0; - watch.start(async (value, ops) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - calls.push({ commits, ops: ops.length }); - // A listener taking a second per call, as a slow client would. - await new Promise((resolve) => setTimeout(resolve, 1000)); - }); + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + calls.push({ commits, ops: ops.length }); + // A listener taking a second per call, as a slow client would. + await new Promise((resolve) => setTimeout(resolve, 1000)); + }), + ); // Ten seconds of a streamed reply at 60 tokens per second, a new entry every second. for (let tick = 0; tick < 600; tick++) { if (tick % 60 === 0) commit((draft) => void draft.entries.push({ text: "", status: "streaming" })); @@ -217,7 +248,7 @@ function advanceDistinct(watch: Watch, count: number): JsonValue { const values: number[] = []; for (let index = 0; index < count; index++) { values.push(index); - watch.advance({ k: [...values] }, [["s", ["k", index], index]], context); + watch.advance(frame({ k: [...values] }, [["s", ["k", index], index]]), context); } return { k: values }; } @@ -240,11 +271,13 @@ describe("Watch overflow", () => { const { watch } = recordedWatch({ k: [] }); const gate = deferred(); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - if (frames.length === 1) await gate.promise; - }); - watch.advance({ k: [], first: true }, [["s", ["first"], true]], context); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + if (frames.length === 1) await gate.promise; + }), + ); + watch.advance(frame({ k: [], first: true }, [["s", ["first"], true]]), context); await flush(); const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); gate.resolve(); @@ -259,8 +292,8 @@ describe("Watch overflow", () => { const { watch, frames } = collectingWatch({ k: [] }); advanceDistinct(watch, MAX_PENDING_OPS + 1); await flush(); - watch.advance({ n: 1 }, [["r", { n: 1 }]], context); - watch.advance({ n: 2 }, [["s", ["n"], 2]], context); + watch.advance(frame({ n: 1 }, [["r", { n: 1 }]]), context); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); await flush(); expect(frames.at(-1)).toEqual({ value: { n: 2 }, @@ -273,77 +306,409 @@ describe("Watch overflow", () => { }); describe("Watch retirement", () => { - it('delivers `[["r", null]]` as the last call and then ends retired', async () => { + it("reports a retirement in the next call with the value the source gives, and goes on", async () => { const { watch, frames, recorded } = collectingWatch(1); const settled = endOf(watch); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); - watch.retire(context); - watch.advance(3, [["r", 3]], context); + watch.advance(retirement(2), context); + await flush(); + watch.advance(frame(3, [["r", 3]]), context); await flush(); expect(frames).toEqual([ { value: 2, ops: [["r", 2]] }, - { value: null, ops: [["r", null]] }, + { value: 2, ops: [], retired: true }, + { value: 3, ops: [["r", 3]] }, ]); - expect(watch.value).toBeNull(); - expect(settled.end).toEqual({ reason: "retired" }); - expect(recorded.detached).toBe(1); + expect(watch.value).toBe(3); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + await watch.stop(); }); - it("drops pending frames, also an overflow, for the retirement", async () => { - for (const pending of [3, MAX_PENDING_OPS + 1]) { - const { watch } = recordedWatch({ k: [] }); - advanceDistinct(watch, pending); - watch.retire(context); - const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(frames).toEqual([{ value: null, ops: [["r", null]] }]); - } + it("folds a retirement and the frames around it into one call that reports it", async () => { + const { watch } = recordedWatch({ n: 0 }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance(retirement({ n: 1 }), context); + watch.advance(frame({ m: 1 }, [["r", { m: 1 }]]), context); + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ kind: "value", value: { m: 1 }, ops: [["r", { m: 1 }]], retired: true }]); + watch.advance(frame({ m: 2 }, [["s", ["m"], 2]]), context); + await flush(); + expect(changes[1]).toEqual({ kind: "value", value: { m: 2 }, ops: [["s", ["m"], 2]], retired: false }); + await watch.stop(); }); - it("ends retired only on its source's retirement, never on a null value", async () => { + it("keeps reporting the retirement past an overflow", async () => { + const { watch } = recordedWatch({ k: [] }); + watch.advance(retirement({ k: [] }), context); + const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ kind: "value", value: newest, ops: [["r", newest]], retired: true }]); + await watch.stop(); + }); + + it("treats a null value as a value, never as a retirement", async () => { const { watch, frames } = collectingWatch(1); const settled = endOf(watch); - watch.advance(null, [["r", null]], context); + watch.advance(frame(null, [["r", null]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); expect(frames).toEqual([ { value: null, ops: [["r", null]] }, { value: 2, ops: [["r", 2]] }, ]); expect(settled.end).toBeUndefined(); - expect(await watch.stop()).toEqual({ reason: "stopped" }); - }); - - it("is only available to watches whose values include null", () => { - const check = (numbers: Watch, documents: Watch<{ n: number } | null>) => { - // @ts-expect-error `retire` delivers `null`, which a number watch cannot hold. - numbers.retire(context); - documents.retire(context); - }; - expectTypeOf(check).toBeFunction(); + expect(await watch.stop()).toEqual({ kind: "stopped" }); }); it("discards a retirement still pending when the source ends", async () => { const { watch } = recordedWatch(1); - watch.retire(context); - watch.end(); - expect(await watch.closed).toEqual({ reason: "ended" }); + watch.advance(retirement(1), context); + watch.end({ kind: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value).toBe(1); }); }); +describe("Watch failure calls", () => { + /** A started watch recording every call, failures included. */ + function recordingWatch(value: T) { + const { watch, recorded } = recordedWatch(value); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + return { watch, recorded, calls }; + } + + it("delivers a failure as a call, stays open, keeps the last value, and replaces the root on recovery", async () => { + const { watch, recorded, calls } = recordingWatch({ n: 0 }); + const settled = endOf(watch); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + await flush(); + const error = new Error("cannot compute"); + watch.fail(error, context); + await flush(); + expect(calls[1]).toEqual({ kind: "failed", error }); + expect(watch.value).toEqual({ n: 1 }); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); + await flush(); + expect(calls[2]).toEqual({ kind: "value", value: { n: 2 }, ops: [["r", { n: 2 }]], retired: false }); + // Recovered: frames pass on as the source made them again. + const next = frame({ n: 3 }, [["s", ["n"], 3]]); + watch.advance(next, context); + await flush(); + expect(calls[3]).toBe(next); + expect(watch.value).toEqual({ n: 3 }); + await watch.stop(); + }); + + it("takes a failure through `advance()` as through `fail()`", async () => { + const { watch, calls } = recordingWatch(0); + const failure: WatchCall = { kind: "failed", error: "down" }; + watch.advance(failure, context); + await flush(); + expect(calls).toEqual([failure]); + expect(calls[0]).toBe(failure); + await watch.stop(); + }); + + it("folds pending values followed by a failure into the failure", async () => { + const { watch } = recordedWatch({ n: 0 }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); + const first = new Error("first"); + const second = new Error("second"); + watch.fail(first, context); + watch.fail(second, context); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + await flush(); + expect(calls).toEqual([{ kind: "failed", error: second }]); + expect(watch.value).toEqual({ n: 0 }); + await watch.stop(); + }); + + it("folds a pending failure followed by values into one root replacement with the newest value", async () => { + const { watch } = recordedWatch({ n: 0 }); + const key = createContextKey("frame"); + watch.fail(new Error("down"), withContextValue(key, "failure", context)); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance(retirement({ n: 1 }), context); + watch.advance(frame({ m: 2 }, [["r", { m: 2 }]]), withContextValue(key, "newest", context)); + const calls: Array<{ call: WatchCall; frame: string | undefined }> = []; + watch.start(async (call, deliveryContext) => { + calls.push({ call, frame: deliveryContext.value(key) }); + }); + await flush(); + expect(calls).toEqual([ + { call: { kind: "value", value: { m: 2 }, ops: [["r", { m: 2 }]], retired: true }, frame: "newest" }, + ]); + await watch.stop(); + }); + + it("delivers a failure that arrives during a call after it, then the recovery", async () => { + const { watch } = recordedWatch(0); + const gate = deferred(); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + watch.advance(frame(1, [["r", 1]]), context); + await flush(); + watch.advance(frame(2, [["r", 2]]), context); + watch.fail("down", context); + gate.resolve(); + await flush(); + watch.advance(frame(3, [["r", 3]]), context); + await flush(); + expect(calls.map((call) => (call.kind === "value" ? call.value : call.error))).toEqual([1, "down", 3]); + expect(watch.value).toBe(3); + await watch.stop(); + }); + + it("drops a pending failure when the source ends, and ignores one after", async () => { + const { watch, calls } = recordingWatch(0); + watch.fail("down", context); + watch.end({ kind: "ended" }); + watch.fail("later", context); + await flush(); + expect(calls).toEqual([]); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("passes every lone value call on by identity, so a value call costs the watch no allocation", async () => { + const { watch, calls } = recordingWatch(0); + const sent: WatchCall[] = []; + for (let n = 1; n <= 1_000; n++) { + const call = frame(n, [["r", n]]); + sent.push(call); + watch.advance(call, context); + await Promise.resolve(); + await Promise.resolve(); + } + await flush(); + expect(calls).toHaveLength(sent.length); + for (const [index, call] of calls.entries()) expect(call).toBe(sent[index]); + await watch.stop(); + }); +}); + +describe("Watch end after delivery", () => { + it("takes no more frames, delivers what is pending, then ends `ended` and only then detaches", async () => { + const { watch, frames, recorded } = collectingWatch(1); + const settled = endOf(watch); + watch.advance(retirement(1), context); + watch.endAfterDelivery(); + expect(recorded.detached).toBe(0); + // The source sends nothing more: a later frame is not taken. + watch.advance(frame(2, [["r", 2]]), context); + await flush(); + expect(frames).toEqual([{ value: 1, ops: [], retired: true }]); + expect(settled.end).toEqual({ kind: "ended" }); + expect(recorded.detached).toBe(1); + }); + + it("waits for start to deliver, and ends at once with nothing pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance(frame(2, [["r", 2]]), context); + watch.endAfterDelivery(); + await flush(); + const settled = endOf(watch); + await flush(); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + const values: JsonValue[] = []; + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(values).toEqual([2]); + expect(recorded.detached).toBe(1); + + const idle = collectingWatch(1); + idle.watch.endAfterDelivery(); + expect(await idle.watch.closed).toEqual({ kind: "ended" }); + expect(idle.frames).toEqual([]); + expect(idle.recorded.detached).toBe(1); + }); + + it("ends a never-started watch at its source's end, dropping what is pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance(retirement(1), context); + watch.endAfterDelivery(); + const failure = new Error("source failed"); + watch.end({ kind: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); + expect(recorded.detached).toBe(1); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + }); + + /** A started watch whose listener is in a call held by `gate`, with nothing pending, then `endAfterDelivery()`. */ + async function endingWhileRunning(settle: () => Promise) { + const { watch, recorded } = recordedWatch(0); + const gate = deferred(); + const values: JsonValue[] = []; + let returned = false; + watch.start( + onValue(async ({ value }) => { + values.push(value); + await gate.promise; + await settle(); + returned = true; + }), + ); + watch.advance(retirement(1, [["r", 1]]), context); + await flush(); + expect(values).toEqual([1]); + watch.endAfterDelivery(); + const settled = endOf(watch); + await flush(); + return { watch, recorded, values, settled, release: gate.resolve, returned: () => returned }; + } + + it("ends `ended` only after the running call returns when nothing is pending, detached then", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(ending.settled.end).toBeUndefined(); + expect(ending.recorded.detached).toBe(0); + ending.release(); + expect(await ending.watch.closed).toEqual({ kind: "ended" }); + expect(ending.returned()).toBe(true); + expect(ending.values).toEqual([1]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `listener_error` and reports it when the running call throws after `endAfterDelivery()`", async () => { + const failure = new Error("listener failed"); + const ending = await endingWhileRunning(async () => { + throw failure; + }); + expect(ending.settled.end).toBeUndefined(); + ending.release(); + expect(await ending.watch.closed).toEqual({ kind: "listener_error", error: failure }); + expect(ending.recorded.reported).toEqual([failure]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `stopped` when stopped while ending, without awaiting the running call", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(await ending.watch.stop()).toEqual({ kind: "stopped" }); + expect(ending.recorded.detached).toBe(1); + expect(ending.returned()).toBe(false); + ending.release(); + await flush(); + expect(await ending.watch.closed).toEqual({ kind: "stopped" }); + }); +}); + +describe("Watch call objects", () => { + /** A frame context carrying `payload` and an abort signal, as a producer's is. */ + function producerContext(payload: object) { + const controller = new AbortController(); + return withAbortSignal( + controller.signal, + withContextValue(createContextKey("payload"), payload, context), + ); + } + + it("hands a listener plain data only: `{ kind, value, ops, retired }`, cloneable and serializable", async () => { + const { watch } = recordedWatch({ n: 0 }); + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), producerContext({})); + await flush(); + // Folded: the watch builds the change. + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), producerContext({})); + watch.advance(retirement({ n: 2 }), producerContext({})); + await flush(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["kind", "value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + expect(JSON.parse(JSON.stringify(change))).toEqual(change); + } + expect(changes[1]).toEqual({ kind: "value", value: { n: 2 }, ops: [["s", ["n"], 2]], retired: true }); + await watch.stop(); + }); + + it("passes a frame delivered alone on as the source made it", async () => { + const { watch } = recordedWatch(0); + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + const alone = frame(1, [["r", 1]]); + watch.advance(alone, context); + await flush(); + expect(changes[0]).toBe(alone); + await watch.stop(); + }); + + it("keeps no frame context alive through a call a listener keeps", async () => { + const kept: Array | NativeWatchCall> = []; + /** Deliver one frame whose context alone references `payload`, keep the change, and stop the watch. */ + const deliver = async (native: boolean): Promise> => { + const payload = { big: "x".repeat(1000) }; + if (native) { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(1, producerContext(payload)); + await flush(); + expect(Object.keys(kept.at(-1)!)).toEqual(["kind", "value"]); + expect(structuredClone(kept.at(-1))).toEqual({ kind: "value", value: 1 }); + await watch.stop(); + } else { + const { watch } = recordedWatch(0); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(frame(1, [["r", 1]]), producerContext(payload)); + await flush(); + await watch.stop(); + } + return new WeakRef(payload); + }; + for (const native of [false, true]) { + const payload = await deliver(native); + await collect(); + expect(kept).toHaveLength(native ? 2 : 1); + expect(payload.deref()).toBeUndefined(); + } + }); +}); + describe("Watch ends", () => { it("stops idempotently and rejects a repeated or late start", async () => { const { watch, recorded } = recordedWatch(0); watch.start(async () => {}); expect(() => watch.start(async () => {})).toThrow("Watch is already started"); - expect(await watch.stop()).toEqual({ reason: "stopped" }); - expect(await watch.stop()).toEqual({ reason: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); expect(recorded.detached).toBe(1); const late = recordedWatch(0).watch; @@ -353,53 +718,55 @@ describe("Watch ends", () => { it("ends `ended` or `failed` from its source, dropping pending frames", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); - watch.end(); + watch.advance(frame(1, [["r", 1]]), context); + watch.end({ kind: "ended" }); await flush(); expect(frames).toEqual([]); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); const failure = new Error("source failed"); const failed = recordedWatch(0).watch; - failed.end({ error: failure }); - failed.end(); - expect(await failed.closed).toEqual({ reason: "failed", error: failure }); + failed.end({ kind: "failed", error: failure }); + failed.end({ kind: "ended" }); + expect(await failed.closed).toEqual({ kind: "failed", error: failure }); }); it("stops future delivery without aborting or awaiting an in-flight callback", async () => { const { watch } = recordedWatch(0); const gate = deferred(); const calls: string[] = []; - watch.start(async (value) => { - calls.push(`start ${value}`); - await gate.promise; - calls.push(`end ${value}`); - }); - watch.advance(1, [["r", 1]], context); + watch.start( + onValue(async ({ value }) => { + calls.push(`start ${value}`); + await gate.promise; + calls.push(`end ${value}`); + }), + ); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); - expect(await watch.stop()).toEqual({ reason: "stopped" }); + watch.advance(frame(2, [["r", 2]]), context); + expect(await watch.stop()).toEqual({ kind: "stopped" }); gate.resolve(); await flush(); expect(calls).toEqual(["start 1", "end 1"]); }); - it("ends `cancelled` when its acquisition signal aborts, also when already aborted", async () => { + it("ends `aborted` when its acquisition signal aborts, also when already aborted", async () => { const controller = new AbortController(); const { watch, recorded } = recordedWatch(0); - watch.observeCancellation(controller.signal); - expect(() => watch.observeCancellation(controller.signal)).toThrow("Watch cancellation is already installed"); + watch.observeAbort(controller.signal); + expect(() => watch.observeAbort(controller.signal)).toThrow("Watch abort is already installed"); controller.abort(); - expect(await watch.closed).toEqual({ reason: "cancelled" }); + expect(await watch.closed).toEqual({ kind: "aborted" }); expect(recorded.detached).toBe(1); const aborted = recordedWatch(0).watch; - aborted.observeCancellation(AbortSignal.abort()); - expect(await aborted.closed).toEqual({ reason: "cancelled" }); + aborted.observeAbort(AbortSignal.abort()); + expect(await aborted.closed).toEqual({ kind: "aborted" }); - const cancelled = recordedWatch(0).watch; - cancelled.cancel(); - expect(await cancelled.closed).toEqual({ reason: "cancelled" }); + const direct = recordedWatch(0).watch; + direct.abort(); + expect(await direct.closed).toEqual({ kind: "aborted" }); }); it("ends `listener_error` on a throw, reports it, and delivers nothing more", async () => { @@ -407,16 +774,18 @@ describe("Watch ends", () => { const failure = new Error("listener failed"); const gate = deferred(); const values: JsonValue[] = []; - watch.start(async (value) => { - values.push(value); - await gate.promise; - throw failure; - }); - watch.advance(1, [["r", 1]], context); + watch.start( + onValue(async ({ value }) => { + values.push(value); + await gate.promise; + throw failure; + }), + ); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); gate.resolve(); - expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await watch.closed).toEqual({ kind: "listener_error", error: failure }); expect(values).toEqual([1]); expect(recorded.reported).toEqual([failure]); expect(recorded.detached).toBe(1); @@ -425,10 +794,10 @@ describe("Watch ends", () => { it("wraps a thrown non-Error in an Error", async () => { const { watch } = recordedWatch(0); watch.start(() => Promise.reject("plain")); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); const end = await watch.closed; - expect(end.reason).toBe("listener_error"); - expect(end.reason === "listener_error" && end.error.message).toBe("plain"); + expect(end.kind).toBe("listener_error"); + expect(end.kind === "listener_error" && end.error.message).toBe("plain"); }); }); @@ -444,9 +813,9 @@ describe("Watch references", () => { () => void source, ); const listener = async (): Promise => {}; - watch.observeCancellation(controller.signal); + watch.observeAbort(controller.signal); watch.start(listener); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); end(watch); return { @@ -467,9 +836,9 @@ describe("Watch references", () => { for (const end of [ (watch: Watch) => void watch.stop(), - (watch: Watch) => watch.cancel(), - (watch: Watch) => watch.end(), - (watch: Watch) => watch.retire(context), + (watch: Watch) => watch.abort(), + (watch: Watch) => watch.end({ kind: "ended" }), + (watch: Watch) => watch.endAfterDelivery(), ]) { const ended = await acquire(end); await flush(); @@ -480,6 +849,61 @@ describe("Watch references", () => { } }); + it("keeps no ops a fold or an overflow dropped, while it waits", async () => { + /** Feed an unstarted watch four tracker batches of 4,000 sets each: the third overflows, the fourth is ignored. */ + const overflow = () => { + const initial: Record = {}; + for (let i = 0; i < 16_000; i++) initial[`k${i}`] = 0; + const tracker = track>(initial); + const watch = new Watch( + initial, + () => {}, + () => {}, + ); + const batches: Array> = []; + for (let round = 0; round < 4; round++) { + const change = tracker.beginChange(); + for (let i = round * 4_000; i < (round + 1) * 4_000; i++) change.state[`k${i}`] = 1; + const prepared = change.prepare(); + tracker.adopt(prepared); + batches.push(new WeakRef(prepared.ops)); + watch.advance(frame(tracker.value, prepared.ops), context); + } + return { watch, batches, newest: tracker.value }; + }; + const { watch, batches, newest } = overflow(); + await collect(); + expect(batches.map((batch) => batch.deref() === undefined)).toEqual([true, true, true, true]); + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ kind: "value", value: newest, ops: [["r", newest]], retired: false }]); + await watch.stop(); + }); + + it("delivers a native backlog that never drains in order", async () => { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + const values: number[] = []; + let next = 50; + // Each call queues one more value behind a backlog of 50, so the queue never empties for 1,000 values. + watch.start( + onValue(async ({ value }) => { + values.push(value); + if (next < 1_000) watch.advance(next++, context); + }), + ); + for (let n = 0; n < 50; n++) watch.advance(n, context); + await flush(); + expect(values).toEqual(Array.from({ length: 1_000 }, (_, n) => n)); + await watch.stop(); + }); + it("releases its pending queue and leaf index once delivered or ended", async () => { /** Two pending frames, folded, whose ops alone reference `payload`; `settle` delivers or ends them. */ const pending = async (settle: (watch: Watch, gate: () => void) => Promise) => { @@ -494,10 +918,10 @@ describe("Watch references", () => { watch.start(async () => { if (++calls === 1) await gate.promise; }); - watch.advance(1, [["s", ["n"], 1]], context); + watch.advance(frame(1, [["s", ["n"], 1]]), context); await flush(); - watch.advance(2, [["s", ["o"], payload]], context); - watch.advance(3, [["s", ["n"], 3]], context); + watch.advance(frame(2, [["s", ["o"], payload]]), context); + watch.advance(frame(3, [["s", ["n"], 3]]), context); await settle(watch, gate.resolve); return { watch, payload: new WeakRef(payload) }; }; @@ -529,9 +953,11 @@ describe("NativeWatch", () => { ); const key = createContextKey("frame"); const delivered: Array<{ value: { run: () => number }; frame: string | undefined }> = []; - watch.start(async (value, deliveryContext) => { - delivered.push({ value, frame: deliveryContext.value(key) }); - }); + watch.start( + onValue(async ({ value }, deliveryContext) => { + delivered.push({ value, frame: deliveryContext.value(key) }); + }), + ); watch.advance(first, withContextValue(key, "first", context)); watch.advance(second, withContextValue(key, "second", context)); expect(delivered).toEqual([]); @@ -551,9 +977,11 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(n, context); const values: number[] = []; - watch.start(async (value) => { - values.push(value); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); await flush(); expect(values).toEqual([101]); }); @@ -567,9 +995,11 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(`value ${n}`, context); const values: string[] = []; - watch.start(async (value) => { - values.push(value); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); await flush(); expect(values).toEqual(["replacement"]); }); @@ -582,20 +1012,34 @@ describe("NativeWatch", () => { () => {}, ); const stopped = create(); - expect(await stopped.stop()).toEqual({ reason: "stopped" }); + expect(await stopped.stop()).toEqual({ kind: "stopped" }); - const cancelled = create(); - cancelled.observeCancellation(AbortSignal.abort()); - expect(await cancelled.closed).toEqual({ reason: "cancelled" }); + const aborted = create(); + aborted.observeAbort(AbortSignal.abort()); + expect(await aborted.closed).toEqual({ kind: "aborted" }); const ended = create(); - ended.end(); - expect(await ended.closed).toEqual({ reason: "ended" }); + ended.end({ kind: "ended" }); + expect(await ended.closed).toEqual({ kind: "ended" }); + + const finishing = create(); + const finished: number[] = []; + finishing.advance(1, context); + finishing.advance(2, context); + finishing.endAfterDelivery(); + finishing.advance(3, context); + finishing.start( + onValue(async ({ value }) => { + finished.push(value); + }), + ); + expect(await finishing.closed).toEqual({ kind: "ended" }); + expect(finished).toEqual([1, 2]); const error = new Error("source failed"); const failed = create(); - failed.end({ error }); - expect(await failed.closed).toEqual({ reason: "failed", error }); + failed.end({ kind: "failed", error }); + expect(await failed.closed).toEqual({ kind: "failed", error }); const reported: unknown[] = []; const throwing = new NativeWatch( @@ -606,10 +1050,44 @@ describe("NativeWatch", () => { const failure = new Error("listener failed"); throwing.start(() => Promise.reject(failure)); throwing.advance(1, context); - expect(await throwing.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await throwing.closed).toEqual({ kind: "listener_error", error: failure }); expect(reported).toEqual([failure]); }); + it("delivers a failure as a call, stays open, and folds it with the values around it", async () => { + const reported: unknown[] = []; + const watch = new NativeWatch( + 0, + () => {}, + (error) => reported.push(error), + ); + const calls: NativeWatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + watch.advance(1, context); + await flush(); + // Values followed by a failure: the failure; a failure followed by values: the values, in order. + watch.advance(2, context); + watch.advance(3, context); + watch.fail("down", context); + await flush(); + expect(watch.value).toBe(1); + watch.fail("again", context); + watch.advance(4, context); + watch.advance(5, context); + await flush(); + expect(calls).toEqual([ + { kind: "value", value: 1 }, + { kind: "failed", error: "down" }, + { kind: "value", value: 4 }, + { kind: "value", value: 5 }, + ]); + expect(watch.value).toBe(5); + expect(reported).toEqual([]); + await watch.stop(); + }); + it("retains neither its source nor its listener once stopped, while still referenced", async () => { const acquire = async (stop: boolean) => { const source = { value: 1 }; diff --git a/packages/chord/test/websocket.test.ts b/packages/chord/test/websocket.test.ts index 8cb1e95c2..b5bc4389b 100644 --- a/packages/chord/test/websocket.test.ts +++ b/packages/chord/test/websocket.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import type { IncomingMessage } from "node:http"; -import { Type } from "typebox"; +import { arr, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { WebSocketServer } from "ws"; import { @@ -17,12 +17,12 @@ import { const Board = remote("t.board", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { - state: { notes: Type.Array(Type.String()) }, - methods: { add: method(Type.String()), who: method(undefined, Type.Unknown()) }, + state: { notes: arr(str()) }, + methods: { add: method(str()), who: method(undefined, unknown()) }, }, }, }); @@ -31,7 +31,7 @@ describe("WebSocket links", () => { it("serves a browser-style client over a real socket, stamped by the authenticator", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, principal, self) => { - const notes = self.state("notes", Type.Array(Type.String()), []); + const notes = self.state("notes", arr(str()), []); return { variant: "main", notes, @@ -112,7 +112,7 @@ describe("WebSocket links: packed payloads", () => { const notes = Array.from({ length: 40 }, (_, i) => `${i}:${"x".repeat(4000)}`); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, _principal, self) => { - const state = self.state("notes", Type.Array(Type.String()), notes); + const state = self.state("notes", arr(str()), notes); return { variant: "main", notes: state, add: () => {}, who: () => notes }; }); const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index decd59f9e..6c6de2736 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Changed + +- README: the codemode tool example builds its parameters with `@earendil-works/pi-schema` instead of TypeBox. + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/codemode/README.md b/packages/codemode/README.md index e4fa87786..d97bd6bf5 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -275,7 +275,7 @@ import { type CodemodeTool, renderDeclarations, } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; const sandboxTools: CodemodeTool[] = agentTools.map((tool) => ({ name: tool.name, @@ -293,7 +293,7 @@ const codemodeTool: AgentTool = { name: "codemode", label: "Codemode", description: `Run JavaScript that calls tools as \`await tools.(args)\`. Output with text() or return.\n\n${renderDeclarations({ tools: sandboxTools })}`, - parameters: Type.Object({ code: Type.String() }), + parameters: obj({ code: str() }), execute: async (_toolCallId, { code }, signal) => { const sandbox = new CodemodeSandbox({ tools: sandboxTools }); try { diff --git a/packages/codemode/package.json b/packages/codemode/package.json index b1f0fea9c..006549146 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "description": "Sandboxed JavaScript execution where the only capability is calling injected tools", "type": "module", "main": "./dist/index.js", @@ -42,7 +42,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -70,7 +70,6 @@ "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 246c1225e..36af238b5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,17 +2,53 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### New Features + +- **Extension update changelogs**: see what changed in installed extension packages on the next fresh launch after an update. Package authors can choose the changelog path, and `collapseChangelog` condenses the notices. See [Show a changelog](docs/packages.md#show-a-changelog). +- **Faster streaming**: large tool calls are parsed incrementally, and code-heavy answers and tool rows reuse highlighting and rendered Markdown instead of repeating the work on every update. See [Keep rendering responsive](docs/tui.md#keep-rendering-responsive). +- **Custom self-update commands**: use `PI_SELF_UPDATE_COMMAND` or `selfUpdateCommand` to update pi through a wrapper or another package manager. See [Custom update command](docs/cli.md#custom-update-command). +- **GPT-6.1 Terra**: available through OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search, mid-conversation thinking-level changes, and thinking `off`. Codex offers a 500K context window. See [Models](docs/cli.md#models). + +### Breaking Changes + +- Tool argument validation runs on `@earendil-works/pi-schema`. TypeBox features outside JSON Schema (`Type.Refine()`, `Type.BigInt()`, `Type.Undefined()`, codecs, `Format.Set()` formats) no longer apply to tool arguments; see [Tool parameter schemas and TypeBox](docs/extensions.md#tool-parameter-schemas-and-typebox). Plain JSON Schema tools, including MCP tools, now get the same coercion as TypeBox tools, such as converting numeric strings and wrapping a single value into an array. Validation failures the model sees list pi-schema's messages (`- path: missing`, `- limit: expected number, got string`) instead of TypeBox's (`must have required properties path`, `must be number`). + ### Added +- Extension packages can now show their changelog after an update. Pi reads `pi.changelogPath` from the package's `package.json`, or `CHANGELOG.md` in the package root, and on the next fresh launch shows entries newer than the last seen version under "Extension Updates". The `collapseChangelog` setting condenses this to one line per package ([#5958](https://github.com/earendil-works/pi/issues/5958), [#8790](https://github.com/earendil-works/pi/pull/8790)) +- `pi.exec()` accepts a `maxBuffer` option that limits stdout and stderr combined, and its result has a `truncated` field that is true when output exceeded the limit ([#11360](https://github.com/earendil-works/pi/issues/11360)) - Device-code login (for example GitHub Copilot) now opens the verification page and copies the code when you press Enter. Providers can turn either action off with `openBrowserOnConfirm: false` or `copyCodeOnConfirm: false` ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) +- `pi update` can update Pi through a wrapper or another package manager. Set `PI_SELF_UPDATE_COMMAND` (for example `mise use -g npm:{packageName}@{version}`) or the global `selfUpdateCommand` setting; `pi update` runs it when pi.dev reports a newer release and fills `{packageName}`, `{version}`, and `{installSpec}` from that release. See [Custom update command](docs/cli.md#custom-update-command) ([#7756](https://github.com/earendil-works/pi/issues/7756), [#5607](https://github.com/earendil-works/pi/issues/5607)) +- Extensions can format the user's configured keybindings with `keyText()` and `keyHint()` from the tool render context and the message and entry renderer options. Importing these functions from `@earendil-works/pi-coding-agent` can resolve the extension's own copy of the package, which does not see pi's keybindings, so hints such as `keyHint("app.tools.expand", "to expand")` rendered without the key. The docs now recommend the injected functions ([#4748](https://github.com/earendil-works/pi/issues/4748)) +- Added persisted session-entry `messageId` and ISO `timestamp` to `MarkdownTransformContext`. They are available when rendering saved entries and undefined for live rendering ([#7952](https://github.com/earendil-works/pi/pull/7952) by [@vipentti](https://github.com/vipentti)) +- Added a `blocked` option to inline `ctx.ui.custom()` components so approval prompts and other custom dialogs can report that pi is waiting for user input. Overlays ignore the option. See [Custom screens and overlays](docs/tui.md#use-custom-screens-and-overlays) ([#11428](https://github.com/earendil-works/pi/issues/11428)) +- Added `Usage.cost.totalFromProvider` to distinguish billed totals from catalog estimates in SDK and extension usage data; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added concrete `responseModel` metadata to messages from pi-messages providers such as Radius, including when a virtual model is requested. See [AssistantMessage](docs/message-types.md#assistantmessage) ([#11265](https://github.com/earendil-works/pi/pull/11265)) +- Added GPT-6.1 Terra for OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search and mid-conversation thinking-level changes. It supports thinking `off`; Codex offers a 500K context window ### Changed +- Extensions keep importing `Type`, `Static` and `TSchema` from `@earendil-works/pi-ai`: Pi maps that import to pi-ai's compat entry plus the TypeBox it bundles, now that pi-ai itself no longer re-exports TypeBox, and its `StringEnum` there stays a TypeBox `TUnsafe`. The built-in tool factories (`createReadTool()` and the others) return `TypeBoxAgentTool`, which keeps `params` typed for TypeBox. +- Tightened system prompt - `pi update` for installs from the pi.dev installer no longer passes `--min-release-age=0` to `npm ci`. The release lockfile pins every package, so the flag had no effect, and security scanners that block it stopped the update ([#11262](https://github.com/earendil-works/pi/issues/11262)) - `/model` and `/scoped-models` now show a model's name next to its ID when the name adds words the ID lacks, such as a renamed `models.json` override ([#5916](https://github.com/earendil-works/pi/issues/5916), [#11347](https://github.com/earendil-works/pi/pull/11347)) +- MCP OAuth sign-in now uses pi's Client ID Metadata Document by default when the authorization server supports it for public clients, and registers a client dynamically otherwise, like Codex, Claude Code, and opencode. Clients registered before stay in use while their redirect URI matches. The new default `oauth.clientRegistration: "auto"` registers when `clientName` or a non-default `callbackUrl` is set; `"dcr"` always registers. `pi mcp add --oauth-client-registration` sets the mode ([#11426](https://github.com/earendil-works/pi/issues/11426)) +- Changed the default OpenAI model from `gpt-5.5` to `gpt-6.1-sol`, including after ChatGPT sign-in when no model is selected ([#11442](https://github.com/earendil-works/pi/issues/11442)) +- Updated the built-in model catalog to models.dev v2. Realtime and image-only models are no longer offered as chat models; newer OpenAI and Google chat models are included ### Fixed +- Updated glob matching to prevent denial of service from amplified glob patterns. +- Fixed a saved "Do not trust" decision being ignored for projects without trust-requiring resources, for example a folder with only `AGENTS.md`. pi auto-trusted such projects, so `ctx.isProjectTrusted()` returned true and package commands treated them as trusted. A saved decision in `trust.json` now takes precedence; auto-trust applies only when no decision is saved ([#10947](https://github.com/earendil-works/pi/issues/10947)) +- Fixed `Shift+Enter` queueing a follow-up instead of inserting a newline in Apple Terminal setups that send `ESC` + Return. pi now checks the local Shift key state for that sequence, as it already did for plain Return; `Alt+Enter` without Shift still queues a follow-up ([#11559](https://github.com/earendil-works/pi/issues/11559)) +- Fixed the working status (OSC 7501) disappearing in Rex and iTerm2 during runs in regular TUI mode. Rendered messages wrote OSC 133 prompt markers to the terminal, which the terminal treats as a new shell prompt; regular mode no longer writes them ([#11519](https://github.com/earendil-works/pi/issues/11519)) +- Fixed compaction, branch summaries, and bug-report summaries in SDK sessions passing the stored OAuth token back to the model runtime as an explicit API key. Provider auth that treats explicit keys differently from stored OAuth now sees the OAuth login, as normal prompts do ([#11520](https://github.com/earendil-works/pi/issues/11520)) +- Fixed a redundant system prompt update (tools, rules) when an extension set the same active tools in a different order, for example a reversed `pi.setActiveTools()` list. Tools that stay active now keep their order and newly activated tools are appended ([#11495](https://github.com/earendil-works/pi/issues/11495)) +- Fixed `pi.exec()` crashing pi with `RangeError: Invalid string length` when a command printed more output than fits in a string. The default output cap is the smaller of 256 MiB and the runtime string limit; past the cap, the command is terminated and the result has `truncated: true` and a nonzero exit code ([#11360](https://github.com/earendil-works/pi/issues/11360)) +- Fixed `pi.exec()` never returning when a command ignored SIGTERM after a timeout or abort. pi now sends SIGKILL 5 seconds later, as intended +- Fixed `pi install` and `pi remove` adding ever deeper nested entries to the npm `package-lock.json` when the agent directory is reached through a symlink, such as `~/.pi` linked into a dotfiles repo. pi now passes the resolved real path to npm ([#11358](https://github.com/earendil-works/pi/issues/11358)) - MCP sign-in now rejects authorization endpoints and other OAuth metadata URLs that are not `http:` or `https:`. Before, a malicious MCP server could supply a `file:` or custom protocol URL, which pi passed to the platform URL handler. - Automatic `fd` and `rg` downloads no longer fetch x86-64 binaries on ARM32 and other architectures without a prebuilt release. pi now warns and recommends installing the tool with the system package manager ([#11209](https://github.com/earendil-works/pi/issues/11209)) - Fixed `/model` search ranking letters scattered over an earlier part of an entry above a real match, for example `opus` listing `gpt-oss-120b` and `claude-sonnet-4.5` before `claude-opus-4.1` ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) @@ -20,7 +56,67 @@ - Fixed HTML export showing a blank page for sessions with thousands of entries ([#11225](https://github.com/earendil-works/pi/issues/11225), [#11293](https://github.com/earendil-works/pi/pull/11293)) - Fixed HTML export not showing system messages: the system prompt and tools, and later section updates such as after `/reload`. `pi --export` now includes the system prompt and tools too ([#11261](https://github.com/earendil-works/pi/issues/11261), [#3464](https://github.com/earendil-works/pi/issues/3464), [#11290](https://github.com/earendil-works/pi/pull/11290)) - Fixed aborting a bash command on macOS sometimes running the next command of a list such as `long_step; next_step`. pi now stops the process group before killing it ([#11252](https://github.com/earendil-works/pi/issues/11252), [#11289](https://github.com/earendil-works/pi/pull/11289)) -- Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed large streamed tool calls taking excessive CPU time and memory because every delta re-parsed the accumulated arguments. Providers and agent proxy streams now parse each delta once, grammar tools no longer copy their entire input per delta, and interactive tool renderers run at most once per frame ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal +- Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored in slash command and file autocomplete, `/settings`, and the model, scoped models, login, fork, `/mcp`, and llama.cpp selectors. In fullscreen mode, PageUp/PageDown now move an open list or autocomplete menu instead of scrolling the transcript, and a page key bound to Ctrl+V pages autocomplete instead of pasting an image ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Fixed tool output and `!` command output showing the parameters of control sequences with a `<`, `=` or `>` prefix as text, for example `35;59;1M` from a mouse report `ESC[<35;59;1M` or `0;276;0c` from a terminal reply `ESC[>0;276;0c`. Text after the keypad-mode sequences `ESC=` and `ESC>` is still kept ([#11377](https://github.com/earendil-works/pi/issues/11377)) +- Fixed `pi -e npm:` staying on the version from the first download. On each start and `/reload`, pi now reinstalls an `-e` npm package without an exact version when the registry has a newer matching version. `pi -e git:@` likewise stayed on its first checkout and is now fetched and reset to the branch; a commit-hash ref that is already checked out is not fetched. Offline mode and failed checks keep the cached copy ([#11375](https://github.com/earendil-works/pi/issues/11375), [#11590](https://github.com/earendil-works/pi/pull/11590)) +- Fixed Windows paths being compared case-sensitively. When the working directory and home directory differed only in drive-letter casing (`c:\Users\me` vs `C:\Users\me`), skills in `~/.agents/skills` loaded twice, as project and user skills, and were reported as name collisions. The same mismatch also re-asked for saved project trust, hid sessions in a custom session directory from `--continue` and `/resume`, and stopped `pi remove` from matching configured local packages ([#10488](https://github.com/earendil-works/pi/issues/10488)) +- Fixed paths next to the home directory being shown with a broken `~` prefix, for example `/home/me2/project` shown as `~2/project` when home is `/home/me` +- Fixed `pi mcp login -l` and `pi mcp logout -l` failing with `Unknown option --local.` although `pi mcp add` and `pi mcp remove` accept `-l`. With `-l`, they now require the server to be a project server. Unknown options are now reported as typed, and a mistyped server name gets a suggestion, for example `Did you mean "gurker"?` +- Fixed `pi update` naming pnpm for installs that only live under `PNPM_HOME`, for example a wrapper under `$PNPM_HOME/global-nub/`. pi said "not managed by a global pnpm install"; it now names the package manager only when that manager owns the install ([#7756](https://github.com/earendil-works/pi/issues/7756)) +- Fixed image paste, copy, and opening the browser for sign-in failing silently on WSL when the working directory is deep. Windows programs such as `powershell.exe` and `explorer.exe` cannot start from a directory whose `\\wsl.localhost\...` path exceeds 260 characters; pi now starts these helpers from `/` ([#11592](https://github.com/earendil-works/pi/issues/11592)) +- Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592)) +- Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) +- Fixed `pi update` stopping with `git clean -fdx failed with code 1` on Windows when another pi session had a git extension's native module or executable open. Windows cannot delete such files, so the update left the new code with the old dependencies. pi now warns, installs the dependencies anyway, and retries the clean on the next update ([#11597](https://github.com/earendil-works/pi/issues/11597)) +- Fixed tool rows and streamed answers highlighting the same code again on every redraw. A running codemode row highlighted its whole script on every progress update and timer tick, a complete `write` row its whole file on every update and expand toggle, a streamed `write` its first 50 lines on every delta, and a streamed answer every earlier code block on every delta. Rows now reuse their last highlighting until the code, theme, or loaded grammars change, and assistant messages keep their rendered Markdown blocks while streaming ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed resized photos being sent as multi-megabyte PNGs even when JPEG was much smaller, causing large requests to fail or time out. Image resizing now tries the smaller encoding first ([#11551](https://github.com/earendil-works/pi/issues/11551)) +- Fixed `pi update --extensions` reporting success without warning when npm's stale registry metadata left a package at its old version. pi now reports the installed and expected versions ([#11536](https://github.com/earendil-works/pi/issues/11536)) +- Fixed skill validation warnings being labeled `[Skill conflicts]` even when there was no name collision. They are now shown under `[Skill warnings]` ([#11521](https://github.com/earendil-works/pi/issues/11521)) +- Fixed codemode tool descriptions including entire union and multi-line output types in every request. Object unions now summarize their fields and discriminators; other multi-line types point to `describeTool()` for the full declaration ([#11501](https://github.com/earendil-works/pi/issues/11501)) +- Fixed the model-scope startup notice being left in the shell after exiting fullscreen mode. It now appears inside pi's fullscreen header ([#11508](https://github.com/earendil-works/pi/issues/11508)) +- Fixed startup from a deleted working directory crashing with an uncaught `uv_cwd` stack trace. pi now asks the user to change to an existing directory ([#11509](https://github.com/earendil-works/pi/issues/11509)) +- Fixed failed session writes leaving entries in memory that were absent from disk, so later entries pointed to missing parents and resumed sessions lost earlier history and tools ([#11460](https://github.com/earendil-works/pi/issues/11460)) +- Fixed `/tree` branch summaries ignoring the session's thinking level, which could make reasoning models reject summary requests with an unsupported `none` effort ([#11444](https://github.com/earendil-works/pi/issues/11444)) +- Fixed hidden thinking with an empty hidden-thinking label still adding blank rows before answers and tool calls ([#11402](https://github.com/earendil-works/pi/issues/11402)) +- Fixed package installation and removal printing npm audit and funding advice, including `npm audit fix` instructions that could install a second copy of pi into the package directory ([#11409](https://github.com/earendil-works/pi/issues/11409)) +- Fixed replacing an extension editor while a dialog was open hiding the dialog and leaving it unanswerable. The new editor now appears after the dialog closes ([#11431](https://github.com/earendil-works/pi/issues/11431)) +- Fixed `grep` context lines being offset from ripgrep's match line numbers when a file contains a lone carriage return ([#11398](https://github.com/earendil-works/pi/issues/11398)) +- Fixed `--model /` failing for provider IDs containing slashes, such as URL-derived llama.cpp provider IDs ([#11394](https://github.com/earendil-works/pi/issues/11394)) +- Fixed slow session forking on filesystems where opening files is expensive by writing the fork through one file handle ([#11253](https://github.com/earendil-works/pi/issues/11253)) +- Fixed `pi.exec()` corrupting UTF-8 characters split across stdout or stderr chunks ([#11312](https://github.com/earendil-works/pi/issues/11312)) +- Fixed `pi.exec()` returning empty stderr when a command could not be spawned, such as a missing executable. It now preserves the spawn error message ([#11257](https://github.com/earendil-works/pi/pull/11257) by [@muyiyr](https://github.com/muyiyr)) +- Fixed a configured package failing to install crashing startup. pi now skips the package, reports an extension warning, and retries on the next start or `/reload`; explicit install and update commands still fail with a nonzero exit ([#11304](https://github.com/earendil-works/pi/pull/11304) by [@Lincoln504](https://github.com/Lincoln504)) +- Fixed the todo extension example crashing the TUI when a tool call failed argument validation ([#8057](https://github.com/earendil-works/pi/pull/8057) by [@cyzlmh](https://github.com/cyzlmh)) +- Updated `undici` to 8.12.3 and the Gondolin extension example to 0.14.0 for security advisories +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) +- Fixed Claude sessions failing with `role 'system' must precede an 'assistant' message` when a system update was followed by an empty or reasoning-only assistant reply ([#11091](https://github.com/earendil-works/pi/issues/11091)) +- Fixed Claude mid-conversation thinking-level changes failing with `Invalid effort level` on accounts whose organization caps effort below `high` ([#10063](https://github.com/earendil-works/pi/issues/10063)) +- Fixed Kimi K3.1 Pro models using Kimi K2 request settings on Kimi Coding, Moonshot, and OpenCode Go. They now send the correct reasoning effort, preserve earlier thinking, accept tools added mid-conversation, and use K3-family pricing +- Fixed OpenCode DeepSeek V4.1 Flash requests failing when long cache retention sent the unsupported `prompt_cache_retention` parameter +- Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from a model that had called tools ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock Claude requests failing when preferred strict tools exceed the provider's strict-tool limits. Requests now retry once with preferred tools non-strict while keeping required strict tools strict ([#11416](https://github.com/earendil-works/pi/issues/11416)) +- Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing earlier thinking after tool calls by using the Anthropic Messages API ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed terminal titles containing control characters being able to inject terminal escape sequences, and pi-messages streams accepting invalid content-block indices +- Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`; word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) +- Fixed display math wider than the content area being split apart by line wrapping; it now falls back to raw LaTeX ([#11483](https://github.com/earendil-works/pi/issues/11483)) +- Fixed styled words ending exactly at the wrap width adding a leading space on the next line, and strikethrough leaking into wrapped-line padding ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `/tree` branch summaries including failed assistant replies and original content that context edits had hidden or replaced. Summaries now honor the same context edits as live requests and compaction ([#11602](https://github.com/earendil-works/pi/issues/11602)) + +### Removed + +- Removed the native OS `sandbox/` extension example and its dependency. Use the [Gondolin example](examples/extensions/gondolin/) or the [containerization guide](docs/containerization.md). ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/docs/cli.md b/packages/coding-agent/docs/cli.md index c471f7576..8861d22ad 100644 --- a/packages/coding-agent/docs/cli.md +++ b/packages/coding-agent/docs/cli.md @@ -84,15 +84,15 @@ pi --continue See [Sessions and Context](sessions.md) for resuming, forking, naming, and storing sessions. - `-c`, `--continue`
- Continues the most recent session for the current project. + Continues the most recent session for the current directory. - `-r`, `--resume`
- Opens the session selector. + Opens the session selector and resumes the chosen session in its working directory. Press Tab to include sessions from other directories. - `--session `
- Opens by file path, exact ID, or partial ID. Pi searches the current project first and offers to fork a cross-project match. + Opens a session by file path or full/partial ID, and resumes it in its working directory. If the ID only matches sessions stored for another directory, Pi offers to fork it into the current directory instead. - `--session-id `
- Opens the exact project session ID or creates it if absent. IDs accept letters, numbers, `.`, `_`, and `-`. + Opens the current directory's existing session with the ID, or creates it otherwise. IDs accept letters, numbers, `.`, `_`, and `-`. - `--fork `
- Forks an existing session into a new session for the current project. + Forks a session, chosen by file path or full/partial ID, into a new session in the current directory. - `--session-dir `
Overrides storage and lookup. It takes precedence over `PI_CODING_AGENT_SESSION_DIR` and the `sessionDir` setting. - `--no-session`
@@ -282,6 +282,16 @@ Add `--force` to reinstall Pi when the selected update includes Pi. `pi update` cannot update Pi when another package manager provides it, such as Nix. Update Pi with that package manager, for example `nix profile upgrade pi`. Package and model catalog updates still work. +#### Custom update command + +If a wrapper or another package manager provides Pi, set `PI_SELF_UPDATE_COMMAND` so `pi update` runs its update command: + +```sh +export PI_SELF_UPDATE_COMMAND="mise use -g npm:{packageName}@{version}" +``` + +`pi update` runs it only when pi.dev has a newer release, replacing `{packageName}`, `{version}`, and `{installSpec}` (`{packageName}@{version}`) with that release. The value is split on spaces. For arguments with spaces, use the `selfUpdateCommand` setting in `~/.pi/agent/settings.json` instead, an argv array such as `["nix", "profile", "upgrade", "pi"]`. + ### Aliases and command options - `pi uninstall ` is an alias for `pi remove `. @@ -322,11 +332,11 @@ These commands work outside a session, so agents can run them through `bash`. Se | Command | Description | |---|---| | `pi mcp add [options] -- [args...]` | Add or replace a stdio server in `mcp.json`; `--env KEY=VALUE` (repeatable) and `--cwd ` set its environment and working directory. Arguments after the command are passed to it | -| `pi mcp add [options] --url ` | Add or replace a streamable HTTP server; `--header KEY=VALUE` (repeatable), `--bearer-token-env-var ` (sends `Authorization: Bearer ${NAME}`), `--oauth-client-id`, `--oauth-client-secret`, `--oauth-callback-port`, and `--oauth-client-name` configure authentication | +| `pi mcp add [options] --url ` | Add or replace a streamable HTTP server; `--header KEY=VALUE` (repeatable), `--bearer-token-env-var ` (sends `Authorization: Bearer ${NAME}`), `--oauth-client-id`, `--oauth-client-secret`, `--oauth-callback-port`, `--oauth-client-name`, and `--oauth-client-registration` (`auto`, `cimd`, or `dcr`) configure authentication | | `pi mcp remove ` | Remove a server from `mcp.json`; stored OAuth credentials are kept | | `pi mcp list [--json]` | Connect to every enabled server and print its state, tools, and errors; exit with `1` when a config entry is invalid or an enabled server is not connected | -| `pi mcp login [--timeout ]` | Sign in to an OAuth server: open the authorization page and wait for the browser; a terminal also accepts the pasted redirect URL. `--timeout` limits the whole sign-in, including requests to the authorization server (default 300 seconds) | -| `pi mcp logout ` | Delete the stored OAuth credentials of a server | +| `pi mcp login [-l] [--timeout ]` | Sign in to an OAuth server: open the authorization page and wait for the browser; a terminal also accepts the pasted redirect URL. `--timeout` limits the whole sign-in, including requests to the authorization server (default 300 seconds). `-l` requires the server to be defined in the project's `.pi/mcp.json` | +| `pi mcp logout [-l]` | Delete the stored OAuth credentials of a server | `add` and `remove` change `~/.pi/agent/mcp.json`, or `.pi/mcp.json` in the current directory with `--local` (`-l`). `add` also takes `--exposure ` (see [Exposure](mcp.md#control-tool-exposure)) and `--description ` and does not connect; run `pi mcp list` to check the server. diff --git a/packages/coding-agent/docs/environment-variables.md b/packages/coding-agent/docs/environment-variables.md index b485d6051..14dd3e142 100644 --- a/packages/coding-agent/docs/environment-variables.md +++ b/packages/coding-agent/docs/environment-variables.md @@ -83,6 +83,7 @@ These variables are read by Pi itself: | `PI_PACKAGE_DIR` | Override the package directory, useful for Nix/Guix store paths | | `PI_OFFLINE` | Disable automatic network activity, including model catalog refreshes | | `PI_SKIP_VERSION_CHECK` | Disable the `pi.dev` latest-version request | +| `PI_SELF_UPDATE_COMMAND` | Command `pi update` runs to update Pi itself, split on whitespace, with `{packageName}`, `{version}`, and `{installSpec}` placeholders. Takes precedence over the `selfUpdateCommand` setting. See [Custom update command](cli.md#custom-update-command) | | `PI_TELEMETRY` | Override install/update telemetry and provider attribution headers: `1`/`true`/`yes` or `0`/`false`/`no` | | `PI_CODEMODE_MAX_WORKERS` | Maximum codemode workers alive at once across all sessions in the process; default `2`. Each worker's script may use up to 256 MB. Raise it when one process runs several sessions, such as in-process subagents. Further scripts wait for a free worker. See [Codemode](codemode.md) | | `PI_CACHE_RETENTION` | Set to `long` for extended provider prompt caching where supported | diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 7580137f6..5879058c7 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -1,65 +1,323 @@ +> Pi can create extensions. Ask it to build one for your use case. + # Extensions -Extensions are TypeScript modules that add executable behavior to Pi. Use one when a workflow needs tools, commands, event handlers, model providers, session state, or terminal UI rather than instructions alone. +Extensions are TypeScript modules that extend Pi's behavior. They can subscribe to lifecycle events, register custom tools callable by the LLM, add commands, and more. An extension runs inside the Pi process with the same operating-system permissions. It can inspect prompts, tool calls, files, credentials, and session history, so load extensions only from sources you trust. -Typical extensions add an agent tool, protect paths, confirm dangerous commands, react to session events, modify context, expose a command, or display persistent status. +> **Placement for /reload:** Put extensions in `~/.pi/agent/extensions/` (global) or `.pi/extensions/` (project-local) for auto-discovery. Use `pi -e ./path.ts` only for quick tests. Extensions in auto-discovered locations can be hot-reloaded with `/reload`. + +**Key capabilities:** +- **Custom tools** - Register tools the LLM can call via `pi.registerTool()` +- **Event interception** - Block or modify tool calls, inject context, customize compaction +- **User interaction** - Prompt users via `ctx.ui` (select, confirm, input, notify) +- **Custom UI components** - Full TUI components with keyboard input via `ctx.ui.custom()` for complex interactions +- **Custom commands** - Register commands like `/mycommand` via `pi.registerCommand()` +- **Session persistence** - Store state that survives restarts via `pi.appendEntry()` +- **Custom rendering** - Control how tool calls/results and messages appear in TUI +- **Providers and MCP servers** - Register model providers, virtual models, and MCP servers + +**Example use cases:** +- Permission gates (confirm before `rm -rf`, `sudo`, etc.) +- Git checkpointing (stash at each turn, restore on branch) +- Path protection (block writes to `.env`, `node_modules/`) +- Custom compaction (summarize conversation your way) +- Conversation summaries (see `summarize.ts` example) +- Interactive tools (questions, wizards, custom dialogs) +- Stateful tools (todo lists, connection pools) +- External integrations (file watchers, webhooks, CI triggers) +- Games while you wait (see `snake.ts` example) + +See [examples/extensions/](../examples/extensions/) for working implementations. + +## Table of Contents + +- [Quick Start](#quick-start) +- [Extension Locations](#extension-locations) +- [Available Imports](#available-imports) +- [Writing an Extension](#writing-an-extension) + - [Extension Styles](#extension-styles) +- [Events](#events) + - [Lifecycle Overview](#lifecycle-overview) + - [Handler Composition](#handler-composition) + - [Startup Events](#startup-events) + - [Resource Events](#resource-events) + - [Session Events](#session-events) + - [Agent Events](#agent-events) + - [Model Events](#model-events) + - [Tool Events](#tool-events) + - [User Bash Events](#user-bash-events) + - [Input Events](#input-events) +- [ExtensionContext](#extensioncontext) +- [ExtensionCommandContext](#extensioncommandcontext) +- [ExtensionAPI Methods](#extensionapi-methods) +- [State Management](#state-management) +- [Custom Tools](#custom-tools) + - [Tool Exposure](#tool-exposure) + - [Calling Other Tools](#calling-other-tools) + - [Dynamic Tool Loading](#dynamic-tool-loading) +- [MCP Servers](#mcp-servers) +- [Custom UI](#custom-ui) +- [Error Handling](#error-handling) +- [Mode Behavior](#mode-behavior) +- [Examples Reference](#examples-reference) - - -## Create and load an extension +## Quick Start -An extension exports a default factory that receives `ExtensionAPI`. The factory registers capabilities for the current extension runtime. - -Create `~/.pi/agent/extensions/hello.ts`: +Create `~/.pi/agent/extensions/my-extension.ts`: ```typescript -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { type ExtensionAPI, isToolCallEventType } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; export default function (pi: ExtensionAPI) { + // React to events + pi.on("session_start", async (_event, ctx) => { + ctx.ui.notify("Extension loaded!", "info"); + }); + + pi.on("tool_call", async (event, ctx) => { + if (isToolCallEventType("bash", event) && event.input.command.includes("rm -rf")) { + const ok = await ctx.ui.confirm("Dangerous!", "Allow rm -rf?"); + if (!ok) return { block: true, reason: "Blocked by user" }; + } + }); + + // Register a custom tool + pi.registerTool({ + name: "greet", + label: "Greet", + description: "Greet someone by name", + parameters: Type.Object({ + name: Type.String({ description: "Name to greet" }), + }), + async execute(toolCallId, params, signal, onUpdate, ctx) { + return { + content: [{ type: "text", text: `Hello, ${params.name}!` }], + details: {}, + }; + }, + }); + + // Register a command pi.registerCommand("hello", { - description: "Show a greeting", - handler: async (name, ctx) => { - ctx.ui.notify(`Hello, ${name || "world"}!`, "info"); + description: "Say hello", + handler: async (args, ctx) => { + ctx.ui.notify(`Hello ${args || "world"}!`, "info"); }, }); } ``` -Start Pi and run `/hello`. During development, load a file directly: +Test with `--extension` (or `-e`) flag: ```bash -pi --extension ./hello.ts +pi -e ./my-extension.ts ``` -Pi uses `jiti`, so local TypeScript extensions do not need a separate compilation step. Use [Pi packages](packages.md) for distributed extensions and dependencies. - - - -## Add it to Pi +## Extension Locations -Place the extension in your user or project extensions directory. Pi loads direct TypeScript or JavaScript files and subdirectories containing an `index.ts` or `index.js` entry point. +> **Security:** Extensions run with your full system permissions and can execute arbitrary code. Only install from sources you trust. -Use a single file for a small extension and a directory for a multi-file implementation. Put npm dependencies in a nearby `package.json`. See [Configuration](configuration.md) for conventional locations and [Settings](settings.md#resources) for additional paths. +Extensions are auto-discovered from trusted locations. Project-local `.pi/extensions` entries load only after the project is trusted. -Reload replaces the extension runtime, so code after `await ctx.reload()` must not reuse state from the old runtime. Only personal and explicit command-line extensions can participate in the `project_trust` event that runs before project extensions load. +| Location | Scope | +|----------|-------| +| `~/.pi/agent/extensions/*.ts` or `*.js` | Global (all projects) | +| `~/.pi/agent/extensions/*/index.ts` or `index.js` | Global (subdirectory) | +| `~/.pi/agent/extensions/*/package.json` with a `pi.extensions` field | Global (subdirectory with manifest) | +| `.pi/extensions/...` | Project-local, same layouts | + +Discovery goes one level deep. Larger packages declare their entry points in a `package.json` manifest. + +Additional paths via `settings.json`: + +```json +{ + "packages": [ + "npm:@foo/bar@1.0.0", + "git:github.com/user/repo@v1" + ], + "extensions": [ + "/path/to/local/extension.ts", + "/path/to/local/extension/dir" + ] +} +``` + +Settings also accept built-in extensions as `builtin:` entries; see [Settings](settings.md#resources). To share extensions via npm or git as Pi packages, see [packages.md](packages.md). + +## Available Imports + +| Package | Purpose | +|---------|---------| +| `@earendil-works/pi-coding-agent` | Extension types (`ExtensionAPI`, `ExtensionContext`, events) and helpers | +| `@earendil-works/pi-agent-core` | Agent message and tool types | +| `typebox`, `typebox/compile`, `typebox/value` | Schema definitions for tool parameters | +| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types). Inside Pi it also exports TypeBox's `Type`, `Static` and `TSchema`, which the published package no longer re-exports; import them from `typebox` in new code | +| `@earendil-works/pi-ai/compat`, `/models`, `/oauth`, `/providers/all` | Provider construction, built-in APIs, and OAuth helpers | +| `@earendil-works/pi-tui` | TUI components for custom rendering | + +Pi maps these imports to its own copies, so extensions share Pi's classes and registries. The legacy `@mariozechner/*` and `@sinclair/typebox` names map to the same modules. + +npm dependencies work too. Add a `package.json` next to your extension (or in a parent directory), run `npm install`, and imports from `node_modules/` are resolved automatically. + +For distributed Pi packages installed with `pi install` (npm or git), runtime deps must be in `dependencies`; `devDependencies` are not installed. Do not list the host-provided packages above in `dependencies`. See [Declare dependencies](packages.md#declare-dependencies). + +### Tool parameter schemas and TypeBox + +Pi validates tool arguments as JSON Schema with `@earendil-works/pi-schema`, whatever built the schema; TypeBox schemas are JSON Schema and work unchanged. Before validation, loosely typed arguments are coerced: numeric strings to numbers, `"true"` to `true`, a single value to a one-item array, `null` in an optional property dropped. TypeBox features outside JSON Schema do not apply to tool arguments: + +- `Type.Refine()` refinements are not run. Check such constraints in `execute()`. +- `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every argument, since JSON has no such values. Codecs (`Type.Codec()`) do not decode: `execute()` receives the JSON value. +- Formats registered with TypeBox's `Format.Set()` are not checked. The standard formats are, plus `url` and `json-pointer-uri-fragment`. +- Property names are matched as names, not as regular expressions, and `additionalProperties` applies only to properties the object does not declare. +- A `$ref` that names nothing rejects only arguments at its place. + +Validation errors the model sees use pi-schema's messages, such as `- path: missing` or `- limit: expected number, got string`. + +Node.js built-ins (`node:fs`, `node:path`, etc.) are also available. + +## Writing an Extension + +An extension exports a default factory function that receives `ExtensionAPI`. The factory can be synchronous or asynchronous: + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +export default function (pi: ExtensionAPI) { + // Subscribe to events + pi.on("event_name", async (event, ctx) => { + // ctx.ui for user interaction + const ok = await ctx.ui.confirm("Title", "Are you sure?"); + ctx.ui.notify("Done!", "info"); + ctx.ui.setStatus("my-ext", "Processing..."); // Footer status + ctx.ui.setWidget("my-ext", ["Line 1", "Line 2"]); // Widget above editor (default) + }); + + // Register tools, commands, shortcuts, flags + pi.registerTool({ ... }); + pi.registerCommand("name", { ... }); + pi.registerShortcut("ctrl+x", { ... }); + pi.registerFlag("my-flag", { ... }); +} +``` + +Extensions are loaded via [jiti](https://github.com/unjs/jiti), so TypeScript works without compilation. + +If the factory returns a `Promise`, Pi awaits it before continuing startup. That means async initialization completes before `session_start`, before `resources_discover`, and before provider registrations queued via `pi.registerProvider()` are flushed. + +### Async factory functions + +Use an async factory for one-time startup work such as fetching remote configuration or dynamically discovering available models. + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +export default async function (pi: ExtensionAPI) { + const response = await fetch("http://localhost:1234/v1/models"); + const payload = (await response.json()) as { + data: Array<{ + id: string; + name?: string; + context_window?: number; + max_tokens?: number; + }>; + }; + + pi.registerProvider("local-openai", { + baseUrl: "http://localhost:1234/v1", + apiKey: "$LOCAL_OPENAI_API_KEY", + api: "openai-completions", + models: payload.data.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: model.context_window ?? 128000, + maxTokens: model.max_tokens ?? 4096, + })), + }); +} +``` + +This pattern makes the fetched models available during normal startup and to `pi --list-models`. For model lists that change while Pi runs, implement `refreshModels` instead; see [pi.registerProvider()](#piregisterprovidername-config). + +### Long-lived resources and shutdown + +Extension factories may run in invocations that never start a session. Do not start background resources such as processes, sockets, file watchers, or timers from the factory. + +Defer background resource startup until `session_start` or the command/tool/event that needs the resource. Register an idempotent `session_shutdown` handler to close any session-scoped resources you start. + +### Extension Styles + +**Single file** - simplest, for small extensions: + +``` +~/.pi/agent/extensions/ +└── my-extension.ts +``` + +**Directory with index.ts** - for multi-file extensions: + +``` +~/.pi/agent/extensions/ +└── my-extension/ + ├── index.ts # Entry point (exports default function) + ├── tools.ts # Helper module + └── utils.ts # Helper module +``` + +**Package with dependencies** - for extensions that need npm packages: + +``` +~/.pi/agent/extensions/ +└── my-extension/ + ├── package.json # Declares dependencies and entry points + ├── package-lock.json + ├── node_modules/ # After npm install + └── src/ + └── index.ts +``` + +```json +// package.json +{ + "name": "my-extension", + "dependencies": { + "zod": "^3.0.0", + "chalk": "^5.0.0" + }, + "pi": { + "extensions": ["./src/index.ts"] + } +} +``` + +Run `npm install` in the extension directory, then imports from `node_modules/` work automatically. + + + +## Events -## Understand the runtime lifecycle +### Lifecycle Overview -The factory can be synchronous or asynchronous. Pi waits for an asynchronous factory before startup continues, allowing it to fetch configuration or register providers needed during startup. +A prompt starts a run. Input handling comes first: -Do not start processes, sockets, watchers, or timers in the factory because some invocations load extensions without starting a session. -Start long-lived resources from `session_start` or from the command or tool that needs them. -Close session-scoped resources from an idempotent `session_shutdown` handler. +1. Extension commands (`/name`) run their handler. The `input` event does not fire. Typed and RPC prompts always check for commands; `pi.sendUserMessage()` checks only with `expandPromptTemplates: true`. +2. `input` handlers can intercept, transform, or handle the text. +3. Skill commands (`/skill:name`) and prompt templates (`/template`) expand. +4. `before_agent_start` runs, then the agent loop starts. -A normal prompt starts a run. The run repeats turns, then checks whether more work starts another low-level agent run: +The run repeats turns, then checks whether more work starts another low-level agent run: ```mermaid flowchart TD @@ -75,7 +333,7 @@ flowchart TD Continue -->|no| Settled["agent_settled"] ``` -More work means a retry, recovery, compaction retry, or queued work. +More work means a retry, final-attempt recovery, a compaction retry, or queued messages. `agent_end` is therefore not a final-idle signal; use `agent_settled` for that. An aborted run skips `agent_before_settle` and goes from `agent_end` to `agent_settled`. Pi counts as busy while a run starts, including the compaction check and `before_agent_start`, so messages sent meanwhile go into the starting run. Each turn sends one model request and executes the tool calls in the assistant response: @@ -93,36 +351,24 @@ flowchart TD The model request runs these hooks in order: -1. `context` -2. `context_with_system` -3. `before_provider_headers` -4. `before_provider_request` -5. `after_provider_response` -6. `provider_stream_event` and assistant message events +1. `context` (can modify conversation messages) +2. `context_with_system` (can modify the full transcript) +3. `before_provider_headers` (can mutate headers) +4. `before_provider_request` (can inspect or replace the payload) +5. `after_provider_response` (status and headers, before the stream is consumed) +6. `provider_stream_event` (raw parsed stream events) and `message_start` / `message_update` / `message_end` for the assistant message -Tool execution runs these hooks in order for each tool call: +A tool call that executes runs these hooks in order: 1. `tool_execution_start` -2. `tool_call` +2. `tool_call` (can block or mutate input) 3. `tool_execution_update` while the tool executes -4. `tool_result` +4. `tool_result` (can modify the result) 5. `tool_execution_end` -The lifecycle involves these event groups: +Calls that do not execute end early. An unknown tool or invalid arguments skip `tool_call`; a blocked or aborted call skips execution and `tool_result`. Each of these still gets `tool_execution_end` with an error result, so use `tool_execution_end` to observe every finalized call. -| Area | Events | -|---|---| -| Input preparation | `input`, `before_agent_start` | -| Agent run | `agent_start`, `agent_end` | -| Model turn | `turn_start`, `context`, `context_with_system`, `turn_end` | -| Provider call | `before_provider_headers`, `before_provider_request`, `after_provider_response`, `provider_stream_event` | -| Message lifecycle | `message_start`, `message_update`, `message_end` | -| Tool execution | `tool_execution_start`, `tool_call`, `tool_execution_update`, `tool_result`, `tool_execution_end` | -| Settlement | `agent_before_settle`, `agent_settled` | - -Pi counts as busy while a run starts, including the compaction check and `before_agent_start`, so messages sent meanwhile go into the starting run. Message lifecycle events also cover user and tool-result messages; the model request list places only the assistant stream where it occurs in each turn. - -Tool hooks run for each tool call, and one assistant response can start another turn by returning tool calls. Provider hooks run only when that transport exposes the corresponding stage. See the exported declarations in [`extensions/types.ts`](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/extensions/types.ts) for exact event payloads and result types. +`turn_end` can append entries and request a continuation. Threshold compaction runs before a naturally required next turn. Message events also fire for other messages, such as user, tool-result, and custom messages. Provider hooks run only when the transport exposes the corresponding stage. Session startup and replacement follow a separate lifecycle: @@ -134,73 +380,38 @@ flowchart TD Fork["/fork or /clone"] --> BeforeFork["session_before_fork"] BeforeSwitch --> Shutdown["session_shutdown"] BeforeFork --> Shutdown - Reload["reload"] --> Shutdown + Reload["/reload"] --> Shutdown + Exit["exit"] --> Shutdown Shutdown --> Replacing{"Replacing the runtime?"} - Replacing -->|yes| Replacement["load replacement extension runtime"] + Replacing -->|yes| ReplacementTrust["project_trust when the working directory is new"] + ReplacementTrust --> Replacement["load replacement extension runtime"] Replacement --> SessionStart Replacing -->|no| Stop["process exits"] SessionStart --> Discover["resources_discover"] ``` -`project_trust` runs before trusted project resources load and can run again when session replacement enters a new working directory. Retries, recovery, compaction, and queued messages can start another agent run after `agent_end`, so `agent_end` is not a final-idle signal. Reload and session replacement invalidate the old runtime. Code after `await ctx.reload()` still belongs to the old runtime and must not reuse its session-bound state. +`session_start` carries `reason: "startup" | "reload" | "new" | "resume" | "fork"`, and `session_shutdown` carries `reason: "quit" | "reload" | "new" | "resume" | "fork"`. `resources_discover` uses `reason: "reload"` after a reload and `"startup"` otherwise. Exit covers Ctrl+C, Ctrl+D, SIGHUP, and SIGTERM. `project_trust` runs before trusted project resources load and can run again when session replacement enters a new working directory. -### Choose an event by intent +Other operations change the current session without replacing the runtime: -| Goal | Event | +| Operation | Events | |---|---| -| Inspect, transform, or consume raw user input | `input` | -| Change prompt sections, active tools, or add context before a run | `before_agent_start` | -| Observe one low-level agent run | `agent_start`, `agent_end` | -| Act after one model response and its tool calls | `turn_end` | -| Filter conversation messages for one provider request | `context` | -| Transform the complete transcript, including system messages | `context_with_system` | -| Change request headers or the serialized provider payload | `before_provider_headers`, `before_provider_request` | -| Observe provider response metadata or parsed stream events | `after_provider_response`, `provider_stream_event` | -| Authorize or modify a tool call | `tool_call` | -| Redact, annotate, or replace a tool result | `tool_result` | -| Append final entries or request one last continuation | `agent_before_settle` | -| Run notification work after automatic processing has stopped | `agent_settled` | -| Initialize or release session-scoped resources | `session_start`, `session_shutdown` | +| `/compact` or automatic compaction | `session_before_compact` (can cancel or customize), then `session_compact` or `session_compact_failed` | +| `/tree` navigation | `session_before_tree` (can cancel or customize), then `session_tree` | +| `/name` or `pi.setSessionName()` | `session_info_changed` | +| `/model` or model cycling (`Ctrl+P`) | `thinking_level_select` if the level changes or is clamped, then `model_select` | +| Thinking-level change (settings, keybinding, `pi.setThinkingLevel()`) | `thinking_level_select` | +| `!` or `!!` command | `user_bash` (can intercept) | +| Extension dialog or custom screen waits for the user | `ui_prompt_start`, `ui_prompt_end` | +| Idle prompt-cache refresh | `cache_warming_decision` | +| Extension registers or removes an MCP server after startup | `mcp_servers_change` | - - -Use `agent_before_settle` when an extension must append entries or request one final continuation. Use `agent_settled` for notification-only work that should run after Pi has no automatic continuation remaining. - - - -## Choose an integration point - -| Capability | Main API | -|---|---| -| Observe or modify lifecycle behavior | `pi.on()` | -| Add a model-callable operation | `pi.registerTool()` | -| Add a `/` command | `pi.registerCommand()` | -| Add a shortcut or CLI flag | `pi.registerShortcut()` or `pi.registerFlag()` | -| Send user or custom messages | `pi.sendUserMessage()` or `pi.sendMessage()` | -| Persist non-context session data | `pi.appendEntry()` | -| Change active tools, model, or thinking level | Session control methods on `pi` | -| Add a model provider | `pi.registerProvider()` | -| Add an MCP server | `pi.registerMcpServer()` | -| Route each request to a model | [`pi.registerVirtualModel()`](virtual-models.md) | -| Add terminal rendering | Renderer registration and `ctx.ui` | -| Communicate with another extension | `pi.events` | - -Use the exported declarations in [`extensions/types.ts`](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/extensions/types.ts) for exact event, context, tool, and result types. - -## Follow the extension contracts - - + -### Events and concurrency +### Handler Composition -Handlers run in extension load and registration order. `pi.on()` returns a function that unsubscribes that registration; changes do not affect a dispatch already in progress. -Some events notify; others transform data, replace results, or cancel an operation. -Use each event’s declared result type rather than assuming every return value has an effect. - -Events cover resource discovery, sessions, agent and message lifecycle, providers, tools, and raw input. - -#### Understand handler composition +Handlers run in extension load order, then registration order. Some events notify; others transform data, replace results, or cancel an operation: | Pattern | Behavior | |---|---| @@ -213,56 +424,2318 @@ Events cover resource discovery, sessions, agent and message lifecycle, provider Notification handlers should not perform slow work on a streaming event. Transform handlers should preserve fields they do not intentionally own, and gate handlers should fail closed when their safety decision cannot be made. -`before_agent_start` exposes both the current prompt and its structured `systemPromptOptions`. `event.messageType` says what started the run: `"user"` for a prompt or `pi.sendUserMessage()`, with `prompt` and `images`; `"custom"` for `pi.sendMessage(..., { triggerTurn: true })`, with the starting `message` and no `prompt`. A returned `message` is added to either kind of run; check `messageType` to skip runs started by custom messages. Prefer changing prompt sections, selected tools, or guidelines so Pi can append a transcript delta. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt for that run while the transcript continues recording the structured sections. Providers receive the forced text as their leading system prompt. +### Startup Events -`message_end` can replace a finalized message while preserving its role. `tool_call` can mutate input or block execution. `tool_result` handlers compose, with each handler seeing prior changes. +#### project_trust - +Fired before Pi decides whether to trust a project with dynamic configs (`.pi` or `.agents/skills`). It runs during startup and when session replacement (for example `/resume`) enters a cwd whose trust has not been resolved in the current process. Only user/global extensions and CLI `-e` extensions participate; project-local extensions are not loaded until after trust is resolved. -`provider_stream_event` fires for each parsed provider stream event before Pi normalizes it. The event identifies the provider, API, and model; `event.data` is the earliest structured value available to Pi, not necessarily the original HTTP bytes or SSE frame. Treat it as read-only because mutation can affect normalization. The event is notification-only and is not persisted. +```typescript +pi.on("project_trust", async (event, ctx) => { + // event.cwd - current working directory + // ctx has a limited trust context: cwd, mode, hasUI, and select/confirm/input/notify UI helpers + if (await ctx.ui.confirm("Trust project?", event.cwd)) { + return { trusted: "yes", remember: true }; + } + return { trusted: "undecided" }; +}); +``` -Handlers are awaited in stream order, so slow handlers delay stream consumption. Handler errors are reported without changing the provider response. See [`debug-provider.ts`](../examples/extensions/debug-provider.ts) for an opt-in viewer that groups raw events by assistant message. +A `project_trust` handler must return `{ trusted: "yes" | "no" | "undecided" }`. A user/global or CLI extension that returns `"yes"` or `"no"` owns the decision; the first yes/no decision wins and suppresses the built-in trust prompt. Use `remember: true` to persist a yes/no decision; otherwise it applies only to the current process. Return `"undecided"` to let later handlers or the built-in trust flow decide. Check `ctx.hasUI` before prompting. If no handler returns yes/no, normal trust resolution continues: saved `trust.json` decisions apply first, then `defaultProjectTrust` controls whether Pi asks, trusts, or declines by default. - +### Resource Events -`context` transforms conversation messages without prompt and tool system messages; Pi restores that state afterward. Use `context_with_system` only when a request-local transformation must own the complete transcript, and keep a system message at index zero. +#### resources_discover -`turn_end` and `agent_before_settle` are actionable boundaries. Their handlers can chain proposed `custom`, `custom_message`, `context_edit`, or `compaction` entries and return `continue: true` for one next model request. Guard continuation conditions because an unconditional continuation can loop. Use the exported event declarations for the complete validation and ordering contract. +Fired after `session_start` so extensions can contribute additional skill, prompt, and theme paths. +The startup path uses `reason: "startup"`. Reload uses `reason: "reload"`. - +```typescript +pi.on("resources_discover", async (event, _ctx) => { + // event.cwd - current working directory + // event.reason - "startup" | "reload" + return { + skillPaths: ["/path/to/skills"], + promptPaths: ["/path/to/prompts"], + themePaths: ["/path/to/themes"], + }; +}); +``` -`cache_warming_decision` can override an idle prompt-cache refresh with `{ action: "warm" }` or `{ action: "stop" }`. The last handler that returns an action wins. +#### mcp_servers_change -Tool calls from one assistant message can run in parallel. -Do not assume a sibling call or result exists when another tool event runs. -Use `ctx.signal` for nested work owned by an active turn; commands and idle session events often have no operation signal. +Fired when an extension registers or unregisters an MCP server after extensions are bound. `event.servers` lists every registered server after the change. Handling this event marks the extension as the one that connects registered servers; see [MCP Servers](#mcp-servers). -A `user_bash` handler that returns `undefined` passes the command to the next handler and then to local execution if no handler handles it. Returning `operations` or `result` stops propagation. A handler failure blocks the command rather than falling through to local execution. +### Session Events + +See [Session Format](session-format.md) for session storage internals and the SessionManager API. + +#### session_start + +Fired when a session is started, loaded, or reloaded. + +```typescript +pi.on("session_start", async (event, ctx) => { + // event.reason - "startup" | "reload" | "new" | "resume" | "fork" + // event.previousSessionFile - present for "new", "resume", and "fork" + ctx.ui.notify(`Session: ${ctx.sessionManager.getSessionFile() ?? "ephemeral"}`, "info"); +}); +``` + +#### session_info_changed + +Fired when the current session display name is set via `/name`, RPC, or `pi.setSessionName()`. + +```typescript +pi.on("session_info_changed", async (event, ctx) => { + // event.name - current normalized name, or undefined if cleared + ctx.ui.notify(`Session renamed: ${event.name ?? "(none)"}`, "info"); +}); +``` + +#### session_before_switch + +Fired before starting a new session (`/new`) or switching sessions (`/resume`). + +```typescript +pi.on("session_before_switch", async (event, ctx) => { + // event.reason - "new" or "resume" + // event.targetSessionFile - session we're switching to (only for "resume") + + if (event.reason === "new") { + const ok = await ctx.ui.confirm("Clear?", "Delete all messages?"); + if (!ok) return { cancel: true }; + } +}); +``` + +After a successful switch or new-session action, Pi emits `session_shutdown` for the old extension instance, reloads and rebinds extensions for the new session, then emits `session_start` with `reason: "new" | "resume"` and `previousSessionFile`. +Do cleanup work in `session_shutdown`, then reestablish any in-memory state in `session_start`. + +#### session_before_fork + +Fired when forking via `/fork` or cloning via `/clone`. + +```typescript +pi.on("session_before_fork", async (event, ctx) => { + // event.entryId - ID of the selected entry + // event.position - "before" for /fork, "at" for /clone + return { cancel: true }; // Cancel fork/clone + // OR + return { skipConversationRestore: true }; // Reserved for future conversation restore control +}); +``` + +After a successful fork or clone, Pi emits `session_shutdown` for the old extension instance, reloads and rebinds extensions for the new session, then emits `session_start` with `reason: "fork"` and `previousSessionFile`. +Do cleanup work in `session_shutdown`, then reestablish any in-memory state in `session_start`. + + + +#### session_before_compact / session_compact / session_compact_failed + +Fired on compaction. See [compaction.md](compaction.md) for details. + +```typescript +pi.on("session_before_compact", async (event, ctx) => { + const { preparation, branchEntries, customInstructions, reason, willRetry, signal } = event; + + // reason - "manual" (/compact), "threshold", or "overflow" + // willRetry - whether the aborted turn is retried after compaction (overflow recovery) + + // Cancel: + return { cancel: true }; + + // Custom summary: + return { + compaction: { + summary: "...", + firstKeptEntryId: preparation.firstKeptEntryId, + tokensBefore: preparation.tokensBefore, + // usage: summaryResponse.usage, // Optional; included in session totals + } + }; +}); + +pi.on("session_compact", async (event, ctx) => { + // event.compactionEntry - the saved compaction + // event.fromExtension - whether extension provided it + // event.reason - "manual" (/compact), "threshold", or "overflow" + // event.willRetry - whether the aborted turn is retried after compaction (overflow recovery) +}); + +pi.on("session_compact_failed", async (event, ctx) => { + // event.reason - "manual" (/compact), "threshold", or "overflow" + // event.errorMessage - present for non-abort failures + // event.aborted - true for cancelled/aborted compactions + // event.willRetry - whether the aborted turn would have retried after compaction + // event.fromExtension - whether extension-provided compaction content was being used +}); +``` + +#### session_before_tree / session_tree + +Fired on `/tree` navigation. See [Sessions](sessions.md) for tree navigation concepts. + +```typescript +pi.on("session_before_tree", async (event, ctx) => { + const { preparation, signal } = event; + // preparation.targetId, oldLeafId, commonAncestorId, entriesToSummarize, + // userWantsSummary, customInstructions, replaceInstructions, label + return { cancel: true }; + // OR provide custom summary (used only when preparation.userWantsSummary is true): + return { + summary: { + summary: "...", + // usage: summaryResponse.usage, // Optional; included in session totals + details: {}, + }, + }; + // OR override summarization options: customInstructions, replaceInstructions, label +}); + +pi.on("session_tree", async (event, ctx) => { + // event.newLeafId, oldLeafId, summaryEntry, fromExtension +}); +``` + +#### session_shutdown + +Fired before a started session runtime is torn down. Use this to clean up resources opened from `session_start` or other session-scoped hooks. + +```typescript +pi.on("session_shutdown", async (event, ctx) => { + // event.reason - "quit" | "reload" | "new" | "resume" | "fork" + // event.targetSessionFile - destination session for session replacement flows + // Cleanup, save state, etc. +}); +``` + +### Agent Events + +#### before_agent_start + +Fired before a run starts, after input handling and expansion. Can inject a message and/or modify the system prompt. + +`event.messageType` says what started the run: `"user"` for a prompt or `pi.sendUserMessage()`, with `prompt` and `images`; `"custom"` for `pi.sendMessage(..., { triggerTurn: true })` while idle, with the starting `message` and no `prompt`. + +```typescript +pi.on("before_agent_start", async (event, ctx) => { + // event.messageType - "user" | "custom" + // event.prompt, event.images - user's prompt text and images (messageType "user") + // event.message - the custom message that starts the run (messageType "custom") + // event.systemPrompt - current chained system prompt for this handler + // (includes changes from earlier before_agent_start handlers) + // event.systemPromptOptions - structured options used to build the system prompt + // .customPrompt - exact prompt prefix from --system-prompt, SYSTEM.md, or custom templates + // .forceSystemPrompt - optional exact replacement for the complete prompt + // .selectedTools - tools currently active in the prompt + // .hiddenTools - selected tools whose declarations requests leave out + // .toolSnippets - one-line descriptions for each tool + // .toolGuidelines - rule bullets keyed by tool name + // .promptGuidelines - additional custom rule bullets + // .sections - custom XML-wrapped sections keyed by tag name + // .appendSystemPrompt - text from --append-system-prompt flags + // .cwd - working directory + // .contextFiles - AGENTS.md files and other loaded context files + // .skills - loaded skills + + return { + // Inject a persistent message (stored in session, sent to LLM) + message: { + customType: "my-extension", + content: "Additional context for the LLM", + display: true, + }, + // Replace the system prompt for this turn (chained across extensions) + systemPrompt: event.systemPrompt + "\n\nExtra instructions for this turn...", + }; +}); +``` + +A returned `message` is added to either kind of run; check `messageType` to skip runs started by custom messages. + +The `systemPromptOptions` field gives extensions access to the same structured data Pi uses to build the system prompt. Collections are mutable. Prefer changing `sections`, `selectedTools`, or `promptGuidelines`: Pi diffs the resulting prompt sections against what the model already has and appends one system message patching only the changed sections. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt for the run: every provider receives the forced text as its leading system prompt (a cache miss when it changes), and the session transcript keeps recording the structured sections. Tool selection changes update both the prompt contributions and executable provider tools; calling `pi.setActiveTools()` inside the handler has the same effect as editing `selectedTools`. Models that accept system messages mid-conversation receive the patch in place and keep their cached prefix; other models get the replayed prompt as their system prompt, which is a cache miss once per change. + +Inside `before_agent_start`, `event.systemPrompt` and `ctx.getSystemPrompt()` both reflect the chained system prompt as of the current handler. Later `before_agent_start` handlers can still modify it again. + + + +#### agent_start / agent_end / agent_before_settle / agent_settled + +`agent_start` fires when a low-level agent run begins. `agent_end` fires when that run ends, but Pi may still auto-retry, auto-compact and retry, or continue with queued follow-up messages. `agent_before_settle` is the final actionable boundary: it can append session entries and request one continuation. `agent_settled` is final and notification-only; use it for status integrations that need to know Pi will not continue running automatically. + +```typescript +pi.on("agent_start", async (_event, ctx) => {}); + +pi.on("agent_end", async (event, ctx) => { + // event.messages - messages from this low-level run +}); + +let addedReviewReminder = false; +pi.on("agent_before_settle", async (event, ctx) => { + if (addedReviewReminder) return; + addedReviewReminder = true; + return { + entries: [...event.entries, { + type: "custom_message", + customType: "review-reminder", + content: "Review the final diff before replying.", + display: false, + }], + continue: true, + }; +}); + +pi.on("agent_settled", async (event, ctx) => { + // event.aborted - true when the run ended because it was aborted, for example with Escape + // ctx.isIdle() is true; runs requested here start after all settled handlers finish. +}); +``` + +If the run is aborted while `agent_before_settle` handlers are running, valid returned entries are still committed, but requested continuation is suppressed. Work requested from `agent_settled` is deferred until every settled handler completes, so notification dispatch is non-reentrant. + +#### ui_prompt_start / ui_prompt_end + +Notification-only lifecycle events for blocking user-facing extension UI prompts. They fire around `ctx.ui.select()`, `ctx.ui.confirm()`, `ctx.ui.input()`, `ctx.ui.editor()`, and `ctx.ui.custom()` so host/status integrations can report "waiting for user" instead of just "running". + +Nested or overlapping prompts are coalesced into one outer waiting span. Handlers are invoked best-effort and are not awaited before showing or closing the prompt. + +```typescript +pi.on("ui_prompt_start", async (event, ctx) => { + // event.reason === "ui_prompt" + // event.kind: "select" | "confirm" | "input" | "editor" | "custom" + // event.title: prompt title when available +}); + +pi.on("ui_prompt_end", async (event, ctx) => { + // Pi is no longer waiting on that UI prompt span. +}); +``` + +#### turn_start / turn_end + +Fired for each turn (one LLM response + tool calls). + +```typescript +pi.on("turn_start", async (event, ctx) => { + // event.turnIndex, event.timestamp +}); + +let replacedResponse = false; +pi.on("turn_end", async (event, ctx) => { + // event.turnIndex, event.message, event.toolResults + // event.entries contains the structural entries proposed so far. + if (replacedResponse || event.outcome !== "completed" || event.toolResults.length > 0) return; + replacedResponse = true; + return { + entries: [ + ...event.entries, + { type: "context_edit", targetId: event.messageEntryId, replacement: null }, + { + type: "custom_message", + customType: "replacement-instruction", + content: "Answer again using the persisted user request.", + display: false, + }, + ], + continue: true, + }; +}); +``` + +`turn_end` runs after the assistant and tool-result messages have been persisted and before the low-level `turn_end` event. Retry backoff and final-attempt recovery still happen after `agent_end`, preserving their existing lifecycle and queue ordering; `agent_before_settle` sees the repaired projection after that work completes. Boundary handlers run in extension load and registration order. Each handler sees prior proposals in `event.entries` and sees `event.context` rebuilt from them. Returning `entries` or `continue` replaces only that field; omitted fields preserve the current proposal. Allowed draft entry types are `custom`, `custom_message`, `context_edit`, and `compaction`. The complete proposal is validated before it is appended in list order after all handlers finish; a handler error is reported and later handlers still run. Validation prevents partially applied semantic errors, but persistence is not transactional. + +`continue: true` ensures one next provider request for that boundary invocation. If tool results, steering, or a follow-up already cause that request, they satisfy the decision and no additional request is made; otherwise Pi makes one context-only request. Error and aborted responses remain hard exits. `continue: false` never suppresses natural work. Guard continuation conditions: an unconditional `continue: true` is evaluated again after the next response and can create an endless loop. A `custom_message` draft contributes a user-role model message but is extension-authored: it does not run human input hooks, slash commands, skills, or prompt templates. + +Host integrations that construct `TurnEndEvent` values must provide `messageEntryId`, `toolResultEntryIds`, `outcome`, `entries`, `continue`, and `context`. `ExtensionEvent` exhaustive switches must also handle `agent_before_settle`. `ExtensionRunner.emit()` excludes actionable turn boundaries; dispatch `turn_end` and `agent_before_settle` through `emitBoundary(baseEvent, buildContext)` so handlers receive chained previews. Other dedicated runner methods still return results for events such as `session_before_*`. + +#### message_start / message_update / message_end + +Fired for message lifecycle updates. + +- `message_start` and `message_end` fire for every message, including user, assistant, tool-result, and custom messages. +- `message_update` fires for assistant streaming updates. +- `message_end` handlers can return `{ message }` to replace the finalized message. The replacement must keep the same `role`. + +```typescript +pi.on("message_start", async (event, ctx) => { + // event.message +}); + +pi.on("message_update", async (event, ctx) => { + // event.message + // event.assistantMessageEvent (token-by-token stream event) +}); + +pi.on("message_end", async (event, ctx) => { + if (event.message.role !== "assistant") return; + + return { + message: { + ...event.message, + usage: { + ...event.message.usage, + cost: { + ...event.message.usage.cost, + total: 0.123, + }, + }, + }, + }; +}); +``` + +#### tool_execution_start / tool_execution_update / tool_execution_end + +Fired for tool execution lifecycle updates. + +In parallel tool mode: +- `tool_execution_start` is emitted in assistant source order during the preflight phase +- `tool_execution_update` events may interleave across tools +- `tool_execution_end` is emitted in tool completion order after each tool is finalized +- final `toolResult` message events are still emitted later in assistant source order + +```typescript +pi.on("tool_execution_start", async (event, ctx) => { + // event.toolCallId, event.toolName, event.args +}); + +pi.on("tool_execution_update", async (event, ctx) => { + // event.toolCallId, event.toolName, event.args, event.partialResult +}); + +pi.on("tool_execution_end", async (event, ctx) => { + // event.toolCallId, event.toolName, event.result, event.isError + // event.durationMs - milliseconds execute() took; absent when the tool did not run +}); +``` + +All three events, `tool_call`, and `tool_result` carry `parentToolCallId` when another tool made the call; see [Calling Other Tools](#calling-other-tools). + +#### context + +Fired before each LLM call. Modify messages non-destructively. See [Session Format](session-format.md) for message types. + +```typescript +pi.on("context", async (event, ctx) => { + // event.messages - deep copy without system messages, safe to modify + const filtered = event.messages.filter(m => !shouldPrune(m)); + return { messages: filtered }; +}); +``` + +`event.messages` holds the conversation without system messages. The prompt and tool declarations belong to Pi and are not part of this hook: when the handler returns a changed list, Pi replays the current prompt sections and tool declarations into one leading system message ahead of the returned messages. Filtering, windowing, or slicing from a compaction summary therefore cannot drop the prompt or the tools. An unchanged list keeps mid-conversation system messages in place, so models that accept them retain their cached prefix. System messages a handler adds are kept after Pi's head. To change the prompt or the tool set durably, use [`before_agent_start`](#before_agent_start) or `pi.setActiveTools()`; to edit system messages for one request, use [`context_with_system`](#context_with_system). + +#### context_with_system + +Fired before each LLM call, after every `context` handler has run and Pi has restored the prompt and tool state. `event.messages` is the full transcript, including the leading system message and any mid-conversation prompt or tool patches (see [Session Format](session-format.md#sessionmessageentry)). The returned messages are sent as they are: this hook owns the prompt and tool declarations for the request. + +```typescript +import { getCurrentSystemMessage } from "@earendil-works/pi-ai"; + +pi.on("context_with_system", async (event, ctx) => { + const cut = findCutIndex(event.messages); + // Fold the dropped prefix so its prompt and tool state survives as the new head. + const head = getCurrentSystemMessage(event.messages.slice(0, cut)); + return { messages: head ? [head, ...event.messages.slice(cut)] : event.messages.slice(cut) }; +}); +``` + +Rules: keep a system message at index 0 (providers read the prompt and initial tool declarations there; Pi reports an error if a handler drops it). Removing a system message removes the tool declarations and section patches it carries. Check your output with `getCurrentSystemPrompt()` and `getCurrentTools()` from `@earendil-works/pi-ai`. Handlers run in extension load order; a `systemPrompt` forced from `before_agent_start` is still projected onto the request afterwards. + +#### before_provider_headers + +Fired after the outgoing HTTP headers are assembled. Use it to add, override, or remove request headers. + +Handlers mutate `event.headers` in place. Set a key to a string to add or override it, or to `null` to delete it. + +```typescript +pi.on("before_provider_headers", (event, ctx) => { + // Add or override — e.g. a session id for gateway tracing/attribution + event.headers["x-session-id"] = ctx.sessionManager.getSessionId(); + + // Drop a tracking header pi adds for this call + event.headers["X-OpenRouter-Title"] = null; +}); +``` + +Runs once per provider request; retries reuse the same headers rather than re-firing the hook. + +#### before_provider_request + +Fired after the provider-specific payload is built, right before the request is sent. Handlers run in extension load order. Returning `undefined` keeps the payload unchanged. Returning any other value replaces the payload for later handlers and for the actual request. + +This hook can rewrite provider-level system instructions or remove them entirely. Those payload-level changes are not reflected by `ctx.getSystemPrompt()`, which reports Pi's system prompt string rather than the final serialized provider payload. + +```typescript +pi.on("before_provider_request", (event, ctx) => { + console.log(JSON.stringify(event.payload, null, 2)); + + // Optional: replace payload + // return { ...event.payload, temperature: 0 }; +}); +``` + +This is mainly useful for debugging provider serialization and cache behavior. + +#### after_provider_response + +Fired after an HTTP response is received and before its stream body is consumed. Handlers run in extension load order. + +```typescript +pi.on("after_provider_response", (event, ctx) => { + // event.status - HTTP status code + // event.headers - normalized response headers + if (event.status === 429) { + console.log("rate limited", event.headers["retry-after"]); + } +}); +``` + +Header availability depends on provider and transport. Providers that abstract HTTP responses may not expose headers. + +#### provider_stream_event + +Fired for each parsed provider stream event before Pi normalizes it. Notification-only and not persisted. + +```typescript +pi.on("provider_stream_event", (event, ctx) => { + // event.provider, event.api, event.model + // event.data - earliest structured value available to Pi (read-only) +}); +``` + +`event.data` is not necessarily the original HTTP bytes or SSE frame. Treat it as read-only because mutation can affect normalization. Handlers are awaited in stream order, so slow handlers delay stream consumption. Handler errors are reported without changing the provider response. See [`debug-provider.ts`](../examples/extensions/debug-provider.ts) for an opt-in viewer that groups raw events by assistant message. + +#### cache_warming_decision + +Fired before each prompt-cache refresh with Pi's decision filled in. The event carries only Pi's cost estimates; use `ctx.model`, `ctx.isIdle()`, and `ctx.getContextUsage()` for everything else. + +```typescript +pi.on("cache_warming_decision", (event, ctx) => { + // event.warmCost: price of this refresh + // event.missCost: extra price of the next request if the entry is lost + // event.continuationProbability: pi's estimate that a request arrives in time + // event.action: "warm" | "stop", pi's decision + + if (ctx.model?.provider === "my-provider") { + return { action: "stop" }; + } +}); +``` + +Return `{ action: "warm" }` or `{ action: "stop" }` to override; the last handler that returns an action wins. `"stop"` ends warming until the next real request. + +### Model Events + +#### model_select + +Fired when the model changes via `/model` command, model cycling (`Ctrl+P`), or `pi.setModel()`. Restoring a session does not fire it; read `ctx.model` in `session_start` to initialize model-dependent state. + +```typescript +pi.on("model_select", async (event, ctx) => { + // event.model - newly selected model + // event.previousModel - previous model (undefined if first selection) + // event.source - "set" | "cycle" + + const prev = event.previousModel + ? `${event.previousModel.provider}/${event.previousModel.id}` + : "none"; + const next = `${event.model.provider}/${event.model.id}`; + + ctx.ui.notify(`Model changed (${event.source}): ${prev} -> ${next}`, "info"); +}); +``` + +Use this to update UI elements (status bars, footers) or perform model-specific initialization when the active model changes. + +#### thinking_level_select + +Fired when the thinking level changes. This is notification-only; handler return values are ignored. + +```typescript +pi.on("thinking_level_select", async (event, ctx) => { + // event.level - newly selected thinking level + // event.previousLevel - previous thinking level + + ctx.ui.setStatus("thinking", `thinking: ${event.level}`); +}); +``` + +Use this to update extension UI when `pi.setThinkingLevel()`, model changes, or built-in thinking-level controls change the active thinking level. + + + +### Tool Events + +#### tool_call + +Fired after `tool_execution_start`, before the tool executes. **Can block.** Use `isToolCallEventType` to narrow and get typed inputs. + +Before `tool_call` runs, Pi waits for previously emitted Agent events to finish draining through `AgentSession`. This means `ctx.sessionManager` is up to date through the current assistant tool-calling message. + +In the default parallel tool execution mode, sibling tool calls from the same assistant message are preflighted sequentially, then executed concurrently. `tool_call` is not guaranteed to see sibling tool results from that same assistant message in `ctx.sessionManager`. + +`event.input` is mutable. Mutate it in place to patch tool arguments before execution. + +Behavior guarantees: +- Mutations to `event.input` affect the actual tool execution +- Later `tool_call` handlers see mutations made by earlier handlers +- No re-validation is performed after your mutation +- Return values from `tool_call` control blocking via `{ block: true, reason?: string, terminate?: boolean }` +- `terminate` only applies to a blocked call; the agent stops early only when every finalized result in the batch is terminating + +```typescript +import { isToolCallEventType } from "@earendil-works/pi-coding-agent"; + +pi.on("tool_call", async (event, ctx) => { + // event.toolName - "bash", "read", "write", "edit", etc. + // event.toolCallId + // event.input - tool parameters (mutable) + + // Built-in tools: no type params needed + if (isToolCallEventType("bash", event)) { + // event.input is { command: string; timeout?: number } + event.input.command = `source ~/.profile\n${event.input.command}`; + + if (event.input.command.includes("rm -rf")) { + return { block: true, reason: "Dangerous command", terminate: true }; + } + } + + if (isToolCallEventType("read", event)) { + // event.input is { path: string; offset?: number; limit?: number } + console.log(`Reading: ${event.input.path}`); + } +}); +``` + +#### Typing custom tool input + +Custom tools should export their input type: + +```typescript +// my-tool.ts +import { type Static, Type } from "typebox"; + +export const myToolSchema = Type.Object({ action: Type.String() }); +export type MyToolInput = Static; +``` + +Use `isToolCallEventType` with explicit type parameters: + +```typescript +import { isToolCallEventType } from "@earendil-works/pi-coding-agent"; +import type { MyToolInput } from "./my-tool.ts"; + +pi.on("tool_call", (event) => { + if (isToolCallEventType<"my_tool", MyToolInput>("my_tool", event)) { + event.input.action; // typed + } +}); +``` + +#### tool_result + +Fired after tool execution finishes and before `tool_execution_end` plus the final tool result message events are emitted. **Can modify result.** + +In parallel tool mode, `tool_result` and `tool_execution_end` may interleave in tool completion order, while final `toolResult` message events are still emitted later in assistant source order. + +`tool_result` handlers chain like middleware: +- Handlers run in extension load order +- Each handler sees the latest result after previous handler changes +- Handlers can return partial patches (`content`, `details`, `structuredContent`, `isError`, or `usage`); omitted fields keep their current values +- Replacing `content` without returning `structuredContent` drops the structured content, because it may no longer match + +Use `ctx.signal` for nested async work inside the handler. This lets Esc cancel model calls, `fetch()`, and other abort-aware operations started by the extension. + +```typescript +import { isBashToolResult } from "@earendil-works/pi-coding-agent"; + +pi.on("tool_result", async (event, ctx) => { + // event.toolName, event.toolCallId, event.input, event.parentToolCallId + // event.content, event.details, event.structuredContent, event.isError, event.usage + + if (isBashToolResult(event)) { + // event.details is typed as BashToolDetails + } + + const response = await fetch("https://example.com/summarize", { + method: "POST", + body: JSON.stringify({ content: event.content }), + signal: ctx.signal, + }); + + // Modify result: + return { content: [...], details: {...}, isError: false, usage: nestedModelUsage }; +}); +``` + +### User Bash Events + +#### user_bash + +Fired when user executes `!` or `!!` commands. **Can intercept.** + +```typescript +import { createLocalBashOperations } from "@earendil-works/pi-coding-agent"; + +pi.on("user_bash", (event, ctx) => { + // event.command - the bash command + // event.excludeFromContext - true if !! prefix + // event.cwd - working directory + + // Option 1: Provide custom operations (e.g., SSH) + return { operations: remoteBashOps }; + + // Option 2: Wrap pi's built-in local bash backend + const local = createLocalBashOperations(); + return { + operations: { + exec(command, cwd, options) { + return local.exec(`source ~/.profile\n${command}`, cwd, options); + } + } + }; + + // Option 3: Full replacement - return result directly + return { result: { output: "...", exitCode: 0, cancelled: false, truncated: false } }; +}); +``` + +Returning `undefined` continues to the next handler, then local execution if none handles the event. A valid result stops propagation: `operations` executes the command through the supplied backend, while `result` records the completed command without executing it. + +### Input Events + +#### input + +Fired when user input is received, after extension commands are checked but before skill and template expansion. The event sees the raw input text, so `/skill:foo` and `/template` are not yet expanded. + +**Processing order:** +1. Extension commands (`/cmd`) checked first - if found, handler runs and input event is skipped. Messages from `pi.sendUserMessage()` are checked only with `expandPromptTemplates: true`. +2. `input` event fires - can intercept, transform, or handle +3. If not handled: skill commands (`/skill:name`) expanded to skill content +4. If not handled: prompt templates (`/template`) expanded to template content +5. Agent processing begins (`before_agent_start`, etc.) + +```typescript +pi.on("input", async (event, ctx) => { + // event.text - raw input (before skill/template expansion) + // event.images - attached images, if any + // event.source - "interactive" (typed), "rpc" (API), or "extension" (via sendUserMessage) + // event.streamingBehavior - "steer" | "followUp" | undefined + // undefined when idle, "steer" for mid-stream interrupts, + // "followUp" for messages queued until the agent finishes + + // Transform: rewrite input before expansion + if (event.text.startsWith("?quick ")) + return { action: "transform", text: `Respond briefly: ${event.text.slice(7)}` }; + + // Handle: respond without LLM (extension shows its own feedback) + if (event.text === "ping") { + ctx.ui.notify("pong", "info"); + return { action: "handled" }; + } + + // Route by source: skip processing for extension-injected messages + if (event.source === "extension") return { action: "continue" }; + + // Intercept skill commands before expansion + if (event.text.startsWith("/skill:")) { + // Could transform, block, or let pass through + } + + return { action: "continue" }; // Default: pass through to expansion +}); +``` + +**Results:** +- `continue` - pass through unchanged (default if handler returns nothing) +- `transform` - modify text/images, then continue to expansion +- `handled` - skip agent entirely (first handler to return this wins) + +Transforms chain across handlers. See [input-transform.ts](../examples/extensions/input-transform.ts) and [input-transform-streaming.ts](../examples/extensions/input-transform-streaming.ts) for `streamingBehavior`-aware routing. + + + +## ExtensionContext + +All handlers receive `ctx: ExtensionContext`. Tool `execute()` receives `ExtensionToolContext`, which adds `ctx.tools` and `ctx.executeTool()` (see [Calling Other Tools](#calling-other-tools)). Command handlers receive [`ExtensionCommandContext`](#extensioncommandcontext). + +### ctx.ui + +UI methods for user interaction. See [Custom UI](#custom-ui) for full details. + +### ctx.mode + +Current run mode: `"tui"`, `"rpc"`, `"json"`, or `"print"`. Use `ctx.mode === "tui"` to guard terminal-only features such as `custom()`, component factories, terminal input, and direct TUI rendering. + +### ctx.hasUI + +`true` in TUI and RPC modes. `false` in print mode (`-p`) and JSON mode. Use this to guard dialog methods (`select`, `confirm`, `input`, `editor`) and fire-and-forget methods (`notify`, `setStatus`, `setWidget`, `setTitle`, `setEditorText`) that work in both TUI and RPC modes. In RPC mode, some TUI-specific methods are no-ops or return defaults (see [RPC Extension UI](rpc-extension-ui.md)). + +### ctx.cwd + +Current working directory. + +Use `CONFIG_DIR_NAME` instead of hardcoding `.pi` when constructing project-local config paths. Rebranded distributions can use a different config directory name. + +```typescript +import { CONFIG_DIR_NAME, type ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { join } from "node:path"; + +export default function (pi: ExtensionAPI) { + pi.on("session_start", (_event, ctx) => { + const projectConfigPath = join(ctx.cwd, CONFIG_DIR_NAME, "my-extension.json"); + // ... + }); +} +``` + +### ctx.isProjectTrusted() + +Returns whether project-local trust is active for the current session context. This includes temporary trust decisions and CLI trust overrides, not just saved decisions in the global trust store. + +Use this before reading project-local extension configuration that should only be honored for trusted projects. + +### ctx.sessionManager + +Read-only access to session state: `getCwd()`, `getSessionDir()`, `getSessionId()`, `getSessionFile()`, `getSessionName()`, `getHeader()`, `getEntries()`, `getEntry()`, `getBranch()`, `getTree()`, `getLeafId()`, `getLeafEntry()`, `getLabel()`, `buildContextEntries()`, and `buildSessionProjection()`. See [Session Format](session-format.md) for entry types. + +For `tool_call`, this state is synchronized through the current assistant message before handlers run. In parallel tool execution mode it is still not guaranteed to include sibling tool results from the same assistant message. + +```typescript +ctx.sessionManager.getEntries() // All entries +ctx.sessionManager.getBranch() // Current branch +ctx.sessionManager.buildContextEntries() // Active branch entries with compaction applied +ctx.sessionManager.getLeafId() // Current leaf entry ID +``` + +### ctx.modelRegistry / ctx.model / ctx.thinkingLevel / ctx.scopedModels + +Access to models, providers, and resolved authentication. `ctx.modelRegistry.getProvider(id)` returns the effective pi-ai provider, while `getProviderAuth(id)` resolves its current API key, headers, base URL, and provider-scoped environment without requiring a loaded model. `ctx.model` is the active model, and `ctx.thinkingLevel` is its current effective thinking level. + +`ctx.scopedModels` is the read-only list of models scoped to the current session — the same set the `/scoped-models` command shows. It is resolved at session start from the `--models` CLI flag and the `enabledModels` setting (matched against the available catalogue with minimatch on `provider/modelId` or a bare `modelId`). It is empty when no scoping is configured, meaning every available model is usable. Each entry is `{ model, thinkingLevel? }`, where `thinkingLevel` is set only when a pattern pinned it (e.g. `anthropic/*:high`). Use it to populate a model picker that mirrors the built-in one instead of enumerating the whole catalogue via `ctx.modelRegistry.getAvailable()`. + +#### Streaming model calls + +Use `ctx.modelRegistry.streamSimple(model, context, options)` for provider-neutral options such as `reasoning`, or `complete(model, context, options)` to await the final assistant message with API-specific options. Both use configured providers and resolve authentication, including for providers registered with `pi.registerProvider()`. Use these instead of `pi-ai/compat` streaming functions, which cannot see extension provider registrations. + +`streamSimple()` returns an `AssistantMessageEventStream`. Iterate it for response events and await `.result()` for the final message. Setup failures produce error events and error results. + +For non-chat models, `findOfType()`, `getModelsOfType()`, `getAvailableOfType()`, and `getModelOfType()` look up image and classifier models, and `classify()` and `generateImages()` call them. See [Use classifier models](models.md#use-classifier-models) and [Use image models](models.md#use-image-models). + +### ctx.signal + +The current agent abort signal, or `undefined` when no agent turn is active. + +Use this for abort-aware nested work started by extension handlers, for example: +- `fetch(..., { signal: ctx.signal })` +- model calls that accept `signal` +- file or process helpers that accept `AbortSignal` + +`ctx.signal` is typically defined during active turn events such as `tool_call`, `tool_result`, `message_update`, and `turn_end`. +It is usually `undefined` in idle or non-turn contexts such as session events, extension commands, and shortcuts fired while Pi is idle. + +```typescript +pi.on("tool_result", async (event, ctx) => { + const response = await fetch("https://example.com/api", { + method: "POST", + body: JSON.stringify(event), + signal: ctx.signal, + }); + + const data = await response.json(); + return { details: data }; +}); +``` + +### ctx.isIdle() / ctx.abort() / ctx.hasPendingMessages() + +Control flow helpers. `ctx.isIdle()` is false while Pi is processing an agent run, automatic retry, auto-compaction retry, or queued continuation. + +### ctx.shutdown() + +Request a graceful shutdown of pi. + +- **Interactive mode:** Deferred until the agent becomes idle (after processing all queued steering and follow-up messages). +- **RPC mode:** Deferred until the next idle state (after completing the current command response, when waiting for the next command). +- **Print mode:** No-op. The process exits automatically when all prompts are processed. + +Emits `session_shutdown` event to all extensions before exiting. Available in all contexts (event handlers, tools, commands, shortcuts). + +```typescript +pi.on("tool_call", (event, ctx) => { + if (isFatal(event.input)) { + ctx.shutdown(); + } +}); +``` + +### ctx.getContextUsage() + +Returns current context usage for the active model. Uses last assistant usage when available, then estimates tokens for trailing messages. + +```typescript +const usage = ctx.getContextUsage(); +if (usage?.tokens != null && usage.tokens > 100_000) { + // ... +} +``` + +### ctx.compact() + +Trigger compaction without awaiting completion. Use `onComplete` and `onError` for follow-up actions. + +```typescript +ctx.compact({ + customInstructions: "Focus on recent changes", + onComplete: (result) => { + ctx.ui.notify("Compaction completed", "info"); + }, + onError: (error) => { + ctx.ui.notify(`Compaction failed: ${error.message}`, "error"); + }, +}); +``` + +### ctx.getSystemPrompt() + +Returns Pi's current system prompt string. + +- During `before_agent_start`, this reflects chained system-prompt changes made so far for the current turn. +- It does not include later `context` message mutations. +- It does not include `before_provider_request` payload rewrites. +- If later-loaded extensions run after yours, they can still change what is ultimately sent. + +```typescript +pi.on("before_agent_start", (event, ctx) => { + const prompt = ctx.getSystemPrompt(); + console.log(`System prompt length: ${prompt.length}`); +}); +``` + +## ExtensionCommandContext + +Command handlers receive `ExtensionCommandContext`, which extends `ExtensionContext` with session control methods. These are only available in commands because they can deadlock if called from event handlers. + +### ctx.getSystemPromptOptions() + +Returns the base inputs Pi currently uses to build the system prompt. + +```typescript +const options = ctx.getSystemPromptOptions(); +const contextPaths = options.contextFiles?.map((file) => file.path) ?? []; +``` + +This has the same shape and mutability as `before_agent_start` `event.systemPromptOptions`: custom or forced prompt, active tools, tool snippets, per-tool and custom rules, custom sections, appended prompt text, cwd, loaded context files, and loaded skills. It may include full context file contents, so treat it as sensitive extension-local data and avoid exposing it through command lists, logs, or autocomplete metadata. + +This reports the current base prompt inputs. It does not include per-turn `before_agent_start` chained system-prompt changes, later `context` event message mutations, or `before_provider_request` payload rewrites. + +### ctx.waitForIdle() + +Wait for the agent to fully settle, including automatic retries, auto-compaction retries, and queued continuations: + +```typescript +pi.registerCommand("my-cmd", { + handler: async (args, ctx) => { + await ctx.waitForIdle(); + // Agent is now idle, safe to modify session + }, +}); +``` + +### ctx.newSession(options?) + +Create a new session: + +```typescript +const parentSession = ctx.sessionManager.getSessionFile(); +const kickoff = "Continue in the replacement session"; + +const result = await ctx.newSession({ + parentSession, + setup: async (sm) => { + sm.appendMessage({ + role: "user", + content: [{ type: "text", text: "Context from previous session..." }], + timestamp: Date.now(), + }); + }, + withSession: async (ctx) => { + // Use only the replacement-session ctx here. + await ctx.sendUserMessage(kickoff); + }, +}); + +if (result.cancelled) { + // An extension cancelled the new session +} +``` + +Options: +- `parentSession`: parent session file to record in the new session header +- `setup`: mutate the new session's `SessionManager` before `withSession` runs +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +### ctx.fork(entryId, options?) + +Fork from a specific entry, creating a new session file: + +```typescript +const result = await ctx.fork("entry-id-123", { + withSession: async (ctx) => { + // Use only the replacement-session ctx here. + ctx.ui.notify("Now in the forked session", "info"); + }, +}); +if (result.cancelled) { + // An extension cancelled the fork +} + +const cloneResult = await ctx.fork("entry-id-456", { position: "at" }); +if (cloneResult.cancelled) { + // An extension cancelled the clone +} +``` + +Options: +- `position`: `"before"` (default) forks before the selected user message, restoring that prompt into the editor +- `position`: `"at"` duplicates the active path through the selected entry without restoring editor text +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +### ctx.navigateTree(targetId, options?) + +Navigate to a different point in the session tree. Rejects while an agent response, manual or automatic compaction, or another tree navigation is active, even with `summarize: false`. These conflicts leave the active branch unchanged and reject the promise rather than returning `{ cancelled: true }`. Wait for the active operation to finish (for example, with `await ctx.waitForIdle()` in a command handler) and retry. It also rejects when `summarize` is set without an available model, or when the target entry does not exist: + +```typescript +const result = await ctx.navigateTree("entry-id-456", { + summarize: true, + customInstructions: "Focus on error handling changes", + replaceInstructions: false, // true = replace default prompt entirely + label: "review-checkpoint", +}); +``` + +Options: +- `summarize`: Whether to generate a summary of the abandoned branch +- `customInstructions`: Custom instructions for the summarizer +- `replaceInstructions`: If true, `customInstructions` replaces the default prompt instead of being appended +- `label`: Label to attach to the branch summary entry (or target entry if not summarizing) + +### ctx.switchSession(sessionPath, options?) + +Switch to a different session file: + +```typescript +const result = await ctx.switchSession("/path/to/session.jsonl", { + withSession: async (ctx) => { + await ctx.sendUserMessage("Resume work in the replacement session"); + }, +}); +if (result.cancelled) { + // An extension cancelled the switch via session_before_switch +} +``` + +Options: +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +To discover available sessions, use the static `SessionManager.list()` or `SessionManager.listAll()` methods: + +```typescript +import { SessionManager } from "@earendil-works/pi-coding-agent"; + +pi.registerCommand("switch", { + description: "Switch to another session", + handler: async (args, ctx) => { + const sessions = await SessionManager.list(ctx.cwd); + if (sessions.length === 0) return; + const choice = await ctx.ui.select( + "Pick session:", + sessions.map((s) => s.path), + ); + if (choice) { + await ctx.switchSession(choice, { + withSession: async (ctx) => { + ctx.ui.notify("Switched session", "info"); + }, + }); + } + }, +}); +``` + +### Session replacement lifecycle and footguns + +`withSession` receives a fresh `ReplacedSessionContext`, which extends `ExtensionCommandContext` with async `sendMessage()` and `sendUserMessage()` helpers bound to the replacement session. + +Lifecycle and footguns: +- `withSession` runs only after the old session has emitted `session_shutdown`, the old runtime has been torn down, the replacement session has been rebound, and the new extension instance has already received `session_start`. +- The callback still executes in the original closure, not inside the new extension instance. That means your old extension instance may already have run its shutdown cleanup before `withSession` starts. +- Captured old `pi` / old command `ctx` session-bound objects are stale after replacement and will throw if used. Use only the `ctx` passed to `withSession` for session-bound work. +- Previously extracted raw objects are still your responsibility. For example, if you capture `const sm = ctx.sessionManager` before replacement, `sm` is still the old `SessionManager` object. Do not reuse it after replacement. +- Code in `withSession` should assume any state invalidated by your `session_shutdown` handler is already gone. Only capture plain data that survives shutdown cleanly, such as strings, ids, and serialized config. + +Safe pattern: + +```typescript +pi.registerCommand("handoff", { + handler: async (_args, ctx) => { + const kickoff = "Continue from the replacement session"; + await ctx.newSession({ + withSession: async (ctx) => { + await ctx.sendUserMessage(kickoff); + }, + }); + }, +}); +``` + +Unsafe pattern: + +```typescript +pi.registerCommand("handoff", { + handler: async (_args, ctx) => { + const oldSessionManager = ctx.sessionManager; + await ctx.newSession({ + withSession: async (_ctx) => { + // stale old objects: do not do this + oldSessionManager.getSessionFile(); + pi.sendUserMessage("wrong"); + }, + }); + }, +}); +``` + +### ctx.reload() + +Run the same reload flow as `/reload`. + +```typescript +pi.registerCommand("reload-runtime", { + description: "Reload extensions, skills, prompts, themes, and context files", + handler: async (_args, ctx) => { + await ctx.reload(); + return; + }, +}); +``` + +Important behavior: +- `await ctx.reload()` emits `session_shutdown` for the current extension runtime +- It then reloads resources and emits `session_start` with `reason: "reload"` and `resources_discover` with reason `"reload"` +- The currently running command handler still continues in the old call frame +- Code after `await ctx.reload()` still runs from the pre-reload version +- Code after `await ctx.reload()` must not assume old in-memory extension state is still valid +- After the handler returns, future commands/events/tool calls use the new extension version + +For predictable behavior, treat reload as terminal for that handler (`await ctx.reload(); return;`). + +Tools run with `ExtensionToolContext`, not `ExtensionCommandContext`, so they cannot call `ctx.reload()`, and Pi cannot reload while the agent is running. To let the LLM trigger a reload, have the tool record the request and run the command from `agent_settled`, where messages sent by handlers are processed after all settled handlers finish: + +```typescript +let reloadRequested = false; + +pi.registerTool({ + name: "reload_runtime", + label: "Reload Runtime", + description: "Reload extensions, skills, prompts, themes, and context files", + parameters: Type.Object({}), + async execute() { + reloadRequested = true; + return { + content: [{ type: "text", text: "Reload requested. Pi reloads after this response." }], + details: {}, + terminate: true, + }; + }, +}); + +pi.on("agent_settled", () => { + if (!reloadRequested) return; + reloadRequested = false; + pi.sendUserMessage("/reload-runtime", { expandPromptTemplates: true }); +}); +``` + +`expandPromptTemplates: true` dispatches the text as a command. Without it, or when queued with `deliverAs`, the text reaches the model as a plain user message. See [reload-runtime.ts](../examples/extensions/reload-runtime.ts). + +## ExtensionAPI Methods + +### pi.on(event, handler) + +Subscribe to events. Returns an unsubscribe function that removes only that registration. See [Events](#events) for event types and return values. + +```typescript +const unsubscribe = pi.on("agent_end", async (event) => { + unsubscribe(); + await updateIntegration(event.messages); +}); +``` + +Handlers run in extension load order, then registration order within each extension. Adding or removing a handler does not affect a dispatch already in progress. + +### pi.registerTool(definition) + +Register a custom tool callable by the LLM. See [Custom Tools](#custom-tools) for full details. + +`pi.registerTool()` works both during extension load and after startup. You can call it inside `session_start`, command handlers, or other event handlers. New tools are refreshed immediately in the same session, so they appear in `pi.getAllTools()` without `/reload`. Registering a tool activates it unless it sets `defaultActive: false` or an [exposure](#tool-exposure) other than `direct` or `model-only`. + +Use `pi.setActiveTools()` to enable or disable tools (including dynamically added tools) at runtime. + +Use `promptSnippet` to opt a custom tool into a one-line entry in the `` section of the default system prompt, and `promptGuidelines` to append tool-specific bullets to its `` section when the tool is active. + +**Important:** `promptGuidelines` bullets are appended flat to the `` section with no tool name prefix. Each guideline must name the tool it refers to — avoid "Use this tool when..." because the LLM cannot tell which tool "this" means. Write "Use my_tool when..." instead. + +See [dynamic-tools.ts](../examples/extensions/dynamic-tools.ts) for a full example. + +```typescript +import { Type } from "typebox"; +import { StringEnum } from "@earendil-works/pi-ai"; + +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "What this tool does", + promptSnippet: "Summarize or transform text according to action", + promptGuidelines: ["Use my_tool when the user asks to summarize previously generated text."], + parameters: Type.Object({ + action: StringEnum(["list", "add"] as const), + text: Type.Optional(Type.String()), + }), + + async execute(toolCallId, params, signal, onUpdate, ctx) { + // Stream progress; partial results also need details + onUpdate?.({ content: [{ type: "text", text: "Working..." }], details: { result: "pending" } }); + + return { + content: [{ type: "text", text: "Done" }], + details: { result: "..." }, + }; + }, + + // Optional: Custom rendering + renderCall(args, theme, context) { ... }, + renderResult(result, options, theme, context) { ... }, +}); +``` + +### pi.sendMessage(message, options?) + +Inject a custom message into the session. Custom messages participate in LLM context. For durable TUI-only content that should not be sent to the LLM, use [`pi.appendEntry()`](#piappendentrycustomtype-data) with [`pi.registerEntryRenderer()`](#piregisterentryrenderercustomtype-renderer). + +```typescript +pi.sendMessage({ + customType: "my-extension", + content: "Message text", + display: true, + details: { ... }, +}, { + triggerTurn: true, + deliverAs: "steer", +}); +``` + +**Options:** +- `deliverAs` - Delivery mode: + - `"steer"` (default) - Queues the message while streaming. Delivered after the current assistant turn finishes executing its tool calls, before the next LLM call. + - `"followUp"` - Waits for agent to finish. Delivered only when agent has no more tool calls. + - `"nextTurn"` - Queued for next user prompt. Does not interrupt or trigger anything. +- `triggerTurn: true` - If agent is idle, trigger an LLM response immediately. Only applies to `"steer"` and `"followUp"` modes (ignored for `"nextTurn"`). Without it, an idle session stores the message without starting a turn. +- `triggerTurn: false` while streaming - The message is not steered or queued as a follow-up; it is appended after the current turn ends. + +### pi.sendUserMessage(content, options?) + +Send a user message to the agent. Unlike `sendMessage()` which sends custom messages, this sends an actual user message that appears as if typed by the user. It starts or queues a turn unless an `input` handler handles it or, with `expandPromptTemplates: true`, it runs an extension command. + +```typescript +// Simple text message +pi.sendUserMessage("What is 2+2?"); + +// With content array (text + images) +pi.sendUserMessage([ + { type: "text", text: "Describe this image:" }, + { type: "image", data: "...", mimeType: "image/png" }, +]); + +// During streaming - must specify delivery mode +pi.sendUserMessage("Focus on error handling", { deliverAs: "steer" }); +pi.sendUserMessage("And then summarize", { deliverAs: "followUp" }); + +// Opt in to extension command dispatch and skill/prompt template expansion +pi.sendUserMessage("/review src/index.ts", { expandPromptTemplates: true }); +``` + +**Options:** +- `deliverAs` - Required when agent is streaming: + - `"steer"` - Queues the message for delivery after the current assistant turn finishes executing its tool calls + - `"followUp"` - Waits for agent to finish all tools +- `expandPromptTemplates` - Dispatch extension commands and expand skill commands and prompt templates. Defaults to `false`. + +When not streaming, the message is sent immediately and triggers a new turn. When streaming without `deliverAs`, the message is rejected and Pi reports an extension error. + +See [send-user-message.ts](../examples/extensions/send-user-message.ts) for a complete example. + +### pi.appendEntry(customType, data?) + +Persist extension data. Custom entries do NOT participate in LLM context. In interactive mode, they can also render inside the chat transcript when paired with `pi.registerEntryRenderer()`. + +```typescript +pi.appendEntry("my-state", { count: 42 }); +pi.appendEntry("status-card", { title: "Indexed files", count: 17 }); + +// Restore on reload +pi.on("session_start", async (_event, ctx) => { + for (const entry of ctx.sessionManager.getEntries()) { + if (entry.type === "custom" && entry.customType === "my-state") { + // Reconstruct from entry.data + } + } +}); +``` + +### pi.setSessionName(name) + +Set the session display name (shown in session selector instead of first message). + +```typescript +pi.setSessionName("Refactor auth module"); +``` + +### pi.getSessionName() + +Get the current session name, if set. + +```typescript +const name = pi.getSessionName(); +if (name) { + console.log(`Session: ${name}`); +} +``` + +### pi.setLabel(entryId, label) + +Set or clear a label on an entry. Labels are user-defined markers for bookmarking and navigation (shown in `/tree` selector). + +```typescript +// Set a label +pi.setLabel(entryId, "checkpoint-before-refactor"); + +// Clear a label +pi.setLabel(entryId, undefined); + +// Read labels via sessionManager +const label = ctx.sessionManager.getLabel(entryId); +``` + +Labels persist in the session and survive restarts. Use them to mark important points (turns, checkpoints) in the conversation tree. + +### pi.registerCommand(name, options) + +Register a command. + +If multiple extensions register the same command name, Pi keeps them all and assigns numeric invocation suffixes in load order, for example `/review:1` and `/review:2`. + +```typescript +pi.registerCommand("stats", { + description: "Show session statistics", + handler: async (args, ctx) => { + const count = ctx.sessionManager.getEntries().length; + ctx.ui.notify(`${count} entries`, "info"); + } +}); +``` + +Optional: add argument auto-completion for `/command ...`. `getArgumentCompletions` can also return a `Promise`: + +```typescript +import type { AutocompleteItem } from "@earendil-works/pi-tui"; + +pi.registerCommand("deploy", { + description: "Deploy to an environment", + getArgumentCompletions: (prefix: string): AutocompleteItem[] | null => { + const envs = ["dev", "staging", "prod"]; + const items = envs.map((e) => ({ value: e, label: e })); + const filtered = items.filter((i) => i.value.startsWith(prefix)); + return filtered.length > 0 ? filtered : null; + }, + handler: async (args, ctx) => { + ctx.ui.notify(`Deploying: ${args}`, "info"); + }, +}); +``` + +### pi.getCommands() + +Get the slash commands available for invocation via `prompt` in the current session. Includes extension commands, prompt templates, and skill commands. +The list matches the RPC `get_commands` ordering: extensions first, then templates, then skills. Skill commands are named `skill:`. + +```typescript +const commands = pi.getCommands(); +const bySource = commands.filter((command) => command.source === "extension"); +const userScoped = commands.filter((command) => command.sourceInfo.scope === "user"); +``` + +Each entry has this shape: + +```typescript +{ + name: string; // Invokable command name without the leading slash. May be suffixed like "review:1" + description?: string; + source: "extension" | "prompt" | "skill"; + sourceInfo: { + path: string; + source: string; + scope: "user" | "project" | "temporary"; + origin: "package" | "top-level"; + baseDir?: string; + }; +} +``` + +Use `sourceInfo` as the canonical provenance field. Do not infer ownership from command names or from ad hoc path parsing. + +Built-in interactive commands (like `/model` and `/settings`) are not included here. They are handled only in interactive +mode and would not execute if sent via `prompt`. + +### pi.registerMessageRenderer(customType, renderer) + +Register a custom TUI renderer for custom messages with your `customType`. Custom messages are created with `pi.sendMessage()` and participate in LLM context. See [Custom UI](#custom-ui). + +### pi.registerMarkdownTransformer(transformer) + +Register a transformer for the Markdown in normal user text, assistant text, and thinking blocks. Transformers run in extension load order, and each transformer receives the Markdown returned by the previous transformer. After the chain finishes, Pi renders the transformed content with its built-in renderer. + +The transformer receives the Markdown string and a context with: + +- `messageType` — `"user"`, `"assistant"`, or `"assistant-thinking"` +- `isStreaming` — `true` for partial assistant updates; `false` for user, finalized assistant, and restored messages +- `availableWidth` — exact terminal columns available for the transformed Markdown content +- `messageId`, `timestamp` — session-entry ID and ISO timestamp when rendering from persisted session entries; `undefined` for live messages + +Return the transformed Markdown: + +```typescript +pi.registerMarkdownTransformer((markdown, { messageType, isStreaming }) => { + if (isStreaming || messageType === "assistant-thinking") return markdown; + return markdown.replaceAll("-->", "→"); +}); +``` + +If a transformer throws, Pi keeps the Markdown produced so far and continues with the next transformer. The hook is display-only: the original message remains unchanged in the session and model context. It runs for new user messages, assistant streaming updates, restored session messages, and terminal width changes, so transformers should remain synchronous and inexpensive. + +### pi.registerEntryRenderer(customType, renderer) + +Register a custom TUI renderer for custom entries with your `customType`. Custom entries are created with `pi.appendEntry()` and do not participate in LLM context. + +```typescript +import { Box, Text } from "@earendil-works/pi-tui"; + +pi.registerEntryRenderer("status-card", (entry, { expanded }, theme) => { + const data = entry.data as { title: string; count: number }; + const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text)); + box.addChild(new Text(`${theme.bold(data.title)}: ${data.count}`)); + if (expanded) { + box.addChild(new Text(theme.fg("dim", JSON.stringify(data, null, 2)))); + } + return box; +}); + +pi.appendEntry("status-card", { title: "Indexed files", count: 17 }); +``` + +### pi.registerShortcut(shortcut, options) + +Register a keyboard shortcut. See [keybindings.md](keybindings.md) for the shortcut format and built-in keybindings. + +```typescript +pi.registerShortcut("ctrl+shift+p", { + description: "Toggle plan mode", + handler: async (ctx) => { + ctx.ui.notify("Toggled!"); + }, +}); +``` + +### pi.registerFlag(name, options) + +Register a CLI flag. + +```typescript +pi.registerFlag("plan", { + description: "Start in plan mode", + type: "boolean", + default: false, +}); + +// Check the value after loading; in the factory, getFlag() returns the default +pi.on("session_start", () => { + if (pi.getFlag("plan")) { + // Plan mode enabled + } +}); +``` + +### pi.exec(command, args, options?) + +Execute a command without a shell. + +```typescript +const result = await pi.exec("git", ["status"], { signal, timeout: 5000 }); +// result.stdout, result.stderr, result.code, result.killed, result.truncated +``` + +Options: `signal` and `timeout` terminate the command, `cwd` sets the working directory, and `maxBuffer` limits stdout and stderr combined (default 256 MiB). When output exceeds `maxBuffer`, pi terminates the command and returns the output up to the limit with `truncated: true`, `killed: true`, and a nonzero `code`. `killed` is true whenever pi terminated the command: timeout, abort, or `maxBuffer`. A command that ignores SIGTERM gets SIGKILL after 5 seconds. + +### pi.getActiveTools() / pi.getAllTools() / pi.setActiveTools(names) + +Manage active tools. This works for both built-in tools and dynamically registered tools. `pi.getActiveTools()` returns the active tool names as `string[]`; `pi.getAllTools()` returns metadata for all configured tools. + +```typescript +const active = pi.getActiveTools(); // ["read", "bash", ...] +const all = pi.getAllTools(); +// all = [{ +// name: "read", +// description: "Read file contents...", +// parameters: ..., +// promptGuidelines: ["Use read to examine files instead of cat or sed."], +// exposure: "direct", +// sourceInfo: { path: "builtin:read", source: "builtin", scope: "temporary", origin: "top-level" } +// }, ...] +// Tools shipped with Pi: core tools and tools from built-in extensions such as builtin:mcp +const shippedTools = all.filter((t) => t.sourceInfo.source === "builtin"); +// Tools from installed or local extensions +const extensionTools = all.filter((t) => t.sourceInfo.source !== "builtin" && t.sourceInfo.source !== "sdk"); +pi.setActiveTools([...new Set([...active, "my_custom_tool"])]); // Keep current tools and enable my_custom_tool +pi.setActiveTools(["read", "bash"]); // Only read and bash +``` + +`pi.getAllTools()` returns `name`, `description`, `parameters`, `promptGuidelines`, `exposure`, `sourceInfo`, and, when set, `namespace`, `annotations`, and `reachesTools`. + +Typical `sourceInfo` values: +- `source: "builtin"` for tools shipped with Pi: core tools (`path: "builtin:"`) and tools registered by built-in extensions such as `builtin:mcp` (`path: "builtin:"`) +- `source: "sdk"` for tools passed via `createAgentSession({ customTools })` +- extension source metadata for tools registered by extensions + +`pi.setActiveTools()` ignores unknown and `hidden` tools. Tools with `codemode` or `deferred` exposure stay callable from codemode scripts whether active or not. + +### pi.setModel(model) + +Set the model for the current session. The change is recorded in session history and restored when that session is resumed, but it does not change the configured `defaultProvider` or `defaultModel` used by new sessions. Returns `false` if authentication is not configured for the model's provider. See [models.md](models.md) for configuring custom models. + +```typescript +const model = ctx.modelRegistry.find("anthropic", "claude-sonnet-4-5"); +if (model) { + const success = await pi.setModel(model); + if (!success) { + ctx.ui.notify("No API key for this model", "error"); + } +} +``` + +### pi.getThinkingLevel() / pi.setThinkingLevel(level) + +Get the current thinking level. Level is clamped to model capabilities (non-reasoning models always use "off"). Changes emit `thinking_level_select`. + +`pi.setThinkingLevel()` changes the thinking level for the current session. The change is recorded in session history and restored when that session is resumed, but it does not change the configured default used by new sessions. + +```typescript +const current = pi.getThinkingLevel(); // "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" +pi.setThinkingLevel("high"); +``` + +### pi.events + +Shared event bus for communication between extensions: + +```typescript +pi.events.on("my:event", (data) => { ... }); +pi.events.emit("my:event", { ... }); +``` + +### pi.registerProvider(name, config) + +Register or override a model provider dynamically. Useful for proxies, custom endpoints, or team-wide model configurations. + +Calls made during the extension factory function are queued and applied once the runner initialises. Calls made after that — for example from a command handler following a user setup flow — take effect immediately without requiring a `/reload`. + +Dynamic providers can implement `refreshModels`. Pi calls it during model refresh, publishes the returned list synchronously through the provider, and passes the canonical credential/stored-catalog/network/signal context. The extension decides whether to persist catalog metadata through generation-checked `context.publish({ persist: entry })`; live servers such as llama.cpp can return models without persisting them. + +`context.signal` is always a concrete signal and provider callbacks must pass it to blocking I/O. Public `ModelRuntime.refresh()` and `ModelRegistry.refresh()` calls accept an optional signal and are unbounded when it is omitted; extensions and applications choose their own deadlines. Cancellation stops the caller waiting even if a provider ignores the signal, but cooperation is still required to stop the underlying work. + +Extensions that need native provider auth, filtering, refresh, or stream behavior can register a complete `Provider` from `@earendil-works/pi-ai`. The provider becomes the composition base and `models.json` overrides still apply above it. + +```typescript +import { createProvider, openAICompletionsApi } from "@earendil-works/pi-ai/compat"; + +const provider = createProvider({ + id: "local-server", + name: "Local Server", + baseUrl: "http://localhost:8080/v1", + auth: { + apiKey: { + name: "Local server setup", + async login(interaction) { + return { + type: "api_key", + key: await interaction.prompt({ type: "secret", message: "API key" }), + }; + }, + async resolve({ credential }) { + return credential?.key + ? { auth: { apiKey: credential.key }, source: "stored API key" } + : undefined; + }, + }, + }, + models: [], + api: openAICompletionsApi(), +}); + +pi.registerProvider(provider); + +// Register a new provider with custom models +pi.registerProvider("my-proxy", { + name: "My Proxy", + baseUrl: "https://proxy.example.com", + apiKey: "$PROXY_API_KEY", // env var reference + api: "anthropic-messages", + models: [ + { + id: "claude-sonnet-4-20250514", + name: "Claude 4 Sonnet (proxy)", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 16384 + } + ] +}); + +// Register a live llama.cpp catalog without persisting discovered models +pi.registerProvider("llama.cpp", { + baseUrl: "http://localhost:8080/v1", + apiKey: "local", + api: "openai-completions", + async refreshModels({ signal }) { + const response = await fetch("http://localhost:8080/v1/models", { signal }); + const { data } = (await response.json()) as { data: Array<{ id: string }> }; + return data.map(({ id }) => ({ + id, + name: id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384 + })); + } +}); + +// Override baseUrl for an existing provider (keeps all models) +pi.registerProvider("anthropic", { + baseUrl: "https://proxy.example.com" +}); + +// Register provider with OAuth support for /login +pi.registerProvider("corporate-ai", { + baseUrl: "https://ai.corp.com", + api: "openai-responses", + models: [...], + oauth: { + name: "Corporate AI (SSO)", + async login(callbacks) { + // Custom OAuth flow + callbacks.onAuth({ url: "https://sso.corp.com/..." }); + const code = await callbacks.onPrompt({ message: "Enter code:" }); + return { refresh: code, access: code, expires: Date.now() + 3600000 }; + }, + async refreshToken(credentials, signal) { + signal.throwIfAborted(); + // Refresh logic + return credentials; + }, + getApiKey(credentials) { + return credentials.access; + } + } +}); +``` + +The object form accepts a complete pi-ai `Provider`, including native `auth`, `getModels`, `refreshModels`, `filterModels`, `stream`, and `streamSimple` behavior. + +**Legacy config options:** +- `name` - Display name for the provider in UI such as `/login`. +- `baseUrl` - API endpoint URL. Each defined model needs one, from the model definition or the provider. +- `apiKey` - API key literal, environment interpolation (`$ENV_VAR` or `${ENV_VAR}`), or leading `!command`. Optional: without it, users store a key with `/login`. `$$` escapes `$`, and `$!` escapes a literal `!` without triggering command execution. +- `api` - API type: `"anthropic-messages"`, `"openai-completions"`, `"openai-responses"`, etc. +- `headers` - Custom headers to include in requests. +- `authHeader` - If true, adds `Authorization: Bearer` header automatically. +- `models` - Array of model definitions. If provided, replaces all existing models for this provider. Model definitions can set `baseUrl` to override the provider endpoint for that model. +- `refreshModels` - Async dynamic discovery callback. Its returned models replace extension-provided models. `context.stored` contains the persisted provider snapshot; use generation-checked `context.publish({ persist: entry })` only when updated catalog data should persist. Use `persist: null` to delete that snapshot. +- `oauth` - OAuth provider config for `/login` support. When provided, the provider appears in the login menu. +- `streamSimple` - Custom streaming implementation for non-standard APIs. + +See [custom-provider.md](custom-provider.md) for advanced topics: custom streaming APIs, OAuth details, model definition reference. + +### pi.unregisterProvider(name) + +Remove a previously registered provider and its models. Built-in models that were overridden by the provider are restored. Has no effect if the provider was not registered. + +Like `registerProvider`, this takes effect immediately when called after the initial load phase, so a `/reload` is not required. + +```typescript +pi.registerCommand("my-setup-teardown", { + description: "Remove the custom proxy provider", + handler: async (_args, _ctx) => { + pi.unregisterProvider("my-proxy"); + }, +}); +``` + +### pi.registerVirtualModel(model) / pi.unregisterVirtualModel(provider, id) + +Register a selectable catalog entry that routes each request to a physical model. `ctx.model` and the session's model selection name the virtual model; assistant messages record the physical model and thinking level the router picked. Registering the same provider and ID again replaces the virtual model. See [Virtual Models](virtual-models.md). + +### pi.registerMcpServer(name, config) / pi.unregisterMcpServer(name) / pi.getMcpServers() + +Register an MCP server for the current session. See [MCP Servers](#mcp-servers). + +### pi.registerToolRenderer(resolver) + +Choose how calls to any tool are drawn, including tools that are not registered yet, such as MCP tools in a resumed session before their server connected. The resolver receives the tool name and `next()`, which returns what the remaining resolvers (in extension load order), then the registered tool, would use, so `next() ?? mine` only fills in. + +```typescript +pi.registerToolRenderer((toolName, next) => { + if (!toolName.startsWith("mcp__")) return next(); + return next() ?? { renderCall: (args, theme) => new Text(theme.fg("toolTitle", toolName), 0, 0) }; +}); +``` + +### pi.getSettings() + +Return a copy of the effective settings, with global and project settings merged and overrides applied. + + + +## State Management + +Extensions with state should store it in tool result `details` for proper branching support: + +```typescript +export default function (pi: ExtensionAPI) { + let items: string[] = []; + + // Reconstruct state from the active branch + const reconstruct = (ctx: ExtensionContext) => { + items = []; + for (const entry of ctx.sessionManager.getBranch()) { + if (entry.type === "message" && entry.message.role === "toolResult") { + if (entry.message.toolName === "my_tool") { + items = (entry.message.details as { items?: string[] } | undefined)?.items ?? []; + } + } + } + }; + pi.on("session_start", async (_event, ctx) => reconstruct(ctx)); + pi.on("session_tree", async (_event, ctx) => reconstruct(ctx)); // /tree changes the branch + + pi.registerTool({ + name: "my_tool", + // ... + async execute(toolCallId, params, signal, onUpdate, ctx) { + items.push("new item"); + return { + content: [{ type: "text", text: "Added" }], + details: { items: [...items] }, // Store for reconstruction + }; + }, + }); +} +``` + +Rebuild from `getBranch()`, not from every entry: abandoned branches represent alternative histories. Use [`pi.appendEntry()`](#piappendentrycustomtype-data) for durable data that should not reach the model. - -### Tools +## Custom Tools -A custom tool defines a name, model-facing description, TypeBox parameter schema, and `execute()` function. -Its result requires model-facing `content` and a `details` field for rendering or state reconstruction. -Use `details: undefined` when there are no structured details. If the tool makes nested model calls, include their `usage` in the result so session totals remain accurate. +Register tools the LLM can call via `pi.registerTool()`. Tools appear in the system prompt and can have custom rendering. -Throw from `execute()` to produce a failed tool result. -Returning an object does not mark it as an error. -Return `terminate: true` only when the agent should skip its automatic follow-up after every completed tool in that batch agrees to terminate. +Use `promptSnippet` for a short one-line entry in the `` section of the default system prompt. If omitted, custom tools are left out of that section. -Use sequential execution when tools share mutable in-memory state. -File-mutating tools should wrap the complete read-modify-write operation with `withFileMutationQueue()`. -Truncate large model-facing results and tell the model where to read the complete output. +Use `promptGuidelines` to add tool-specific bullets to the `` section of the default system prompt. These bullets are included only while the tool is active (for example, after `pi.setActiveTools([...])`). -Declare `outputSchema` and return a matching `structuredContent` when the result is data. The model still receives `content`; programmatic callers such as codemode scripts receive `structuredContent` instead of the text. Tools without `outputSchema` are passed to scripts as their text content. To report a failure that still carries data, return the result with `isError: true` instead of throwing: the model sees an error, and scripts still receive `structuredContent`. +**Important:** `promptGuidelines` bullets are appended flat to the `` section with no tool name prefix or grouping. Each guideline must name the tool it refers to — avoid "Use this tool when..." because the LLM cannot tell which tool "this" means. Write "Use my_tool when..." instead. -A tool can run other tools with `ctx.executeTool(name, args, { signal, onUpdate })`. Nested calls go through argument validation and the `tool_call` and `tool_result` handlers like model-issued calls, and emit `tool_execution_start`, `tool_execution_update`, and `tool_execution_end`; all of these events carry `parentToolCallId`, and their `toolCallId` is assigned by pi as `/`. These ids do not appear as tool calls or tool results in the transcript. Nested calls do not add transcript entries: their results only reach the calling tool, which reports them itself, for example through `onUpdate` and `details`. The session keeps a bounded record of them (name, arguments, status, duration, error; never results) as `nestedCalls` on the calling tool's result message. It is used for compaction file lists and shown in HTML exports. Arguments over 8 KiB per call or 32 KiB per tool result are omitted, at most 256 calls are kept, and `complete: false` marks a record that lost anything. The `usage` of nested results, at every depth, is added to the calling tool's result `usage`, so a tool reports only its own usage, not that of the tools it called. `ctx.tools` lists the tools `ctx.executeTool()` can call. `tool_result` handlers that redact `content` should also replace `structuredContent`; replacing only `content` drops it. +Note: Some models include the @ prefix in tool path arguments. Built-in tools strip a leading @ before resolving paths. If your custom tool accepts a path, normalize a leading @ as well. -See [`hello.ts`](../examples/extensions/hello.ts), [`todo.ts`](../examples/extensions/todo.ts), [`dynamic-tools.ts`](../examples/extensions/dynamic-tools.ts), and [`truncated-tool.ts`](../examples/extensions/truncated-tool.ts). +If your custom tool mutates files, use `withFileMutationQueue()` so it participates in the same per-file queue as built-in `edit` and `write`. This matters because tool calls run in parallel by default. Without the queue, two tools can read the same old file contents, compute different updates, and then whichever write lands last overwrites the other. -### Tool exposure +Example failure case: your custom tool edits `foo.ts` while built-in `edit` also changes `foo.ts` in the same assistant turn. If your tool does not participate in the queue, both can read the original `foo.ts`, apply separate changes, and one of those changes is lost. + +Pass the real target file path to `withFileMutationQueue()`, not the raw user argument. Resolve it to an absolute path first, relative to `ctx.cwd` or your tool's working directory. For existing files, the helper canonicalizes through `realpath()`, so symlink aliases for the same file share one queue. For new files, it falls back to the resolved absolute path because there is nothing to `realpath()` yet. + +Queue the entire mutation window on that target path. That includes read-modify-write logic, not just the final write. + +```typescript +import { withFileMutationQueue } from "@earendil-works/pi-coding-agent"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +async execute(_toolCallId, params, _signal, _onUpdate, ctx) { + const absolutePath = resolve(ctx.cwd, params.path); + + return withFileMutationQueue(absolutePath, async () => { + await mkdir(dirname(absolutePath), { recursive: true }); + const current = await readFile(absolutePath, "utf8"); + const next = current.replace(params.oldText, params.newText); + await writeFile(absolutePath, next, "utf8"); + + return { + content: [{ type: "text", text: `Updated ${params.path}` }], + details: {}, + }; + }); +} +``` + +### Tool Definition + +```typescript +import { type Static, Type } from "typebox"; +import { StringEnum } from "@earendil-works/pi-ai"; + +const myToolSchema = Type.Object({ + action: StringEnum(["list", "add"] as const), + text: Type.Optional(Type.String()), +}); + +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "What this tool does (shown to LLM)", + promptSnippet: "List or add items in the project todo list", + promptGuidelines: [ + "Use my_tool for todo planning instead of direct file edits when the user asks for a task list." + ], + parameters: myToolSchema, + prepareArguments(args) { + // Fold a legacy `oldAction` field into `action`. Schema validation still runs afterwards. + const input = (args ?? {}) as { action?: unknown; oldAction?: unknown }; + if (input.action === undefined && input.oldAction !== undefined) { + return { ...input, action: input.oldAction } as Static; + } + return args as Static; + }, + + async execute(toolCallId, params, signal, onUpdate, ctx) { + // Check for cancellation + if (signal?.aborted) { + return { content: [{ type: "text", text: "Cancelled" }], details: {} }; + } + + // Stream progress updates; partial results also need details + onUpdate?.({ + content: [{ type: "text", text: "Working..." }], + details: {}, + }); + + // Run commands via pi.exec (captured from extension closure) + const result = await pi.exec("some-command", [], { signal }); + + // Return result + return { + content: [{ type: "text", text: "Done" }], // Sent to LLM + details: { data: result }, // For rendering & state + // usage: nestedModelResponse.usage, // Optional nested LLM usage + // Optional: stop after this tool batch when every finalized tool result + // in the batch also returns terminate: true. + terminate: true, + }; + }, + + // Optional: Custom rendering + renderCall(args, theme, context) { ... }, + renderResult(result, options, theme, context) { ... }, +}); +``` + +**Result fields:** Every result, including partial results passed to `onUpdate`, needs model-facing `content` and a `details` value. Use `details: undefined` when there is nothing to store; the type of `details` must match across all results of the tool. + +**Usage accounting:** If a tool makes nested LLM calls, return their combined `Usage` as `usage`. Pi persists it on the tool result and includes it in footer, `/session`, and RPC session totals. `tool_result` handlers can inspect or replace this value. + +**Signaling errors:** To mark a tool execution as failed, throw an error from `execute`, or return a result with `isError: true`. Both report an error result to the LLM; a returned result also keeps `details` and `structuredContent` for rendering and scripts. Other returned values never set the error flag. + +**Early termination:** Return `terminate: true` from `execute()` to hint that the automatic follow-up LLM call should be skipped after the current tool batch. This only takes effect when every finalized tool result in that batch is terminating. See [examples/extensions/structured-output.ts](../examples/extensions/structured-output.ts) for a minimal example where the agent ends on a final structured-output tool call. + +```typescript +// Correct: throw to signal an error +async execute(toolCallId, params) { + if (!isValid(params.input)) { + throw new Error(`Invalid input: ${params.input}`); + } + return { content: [{ type: "text", text: "OK" }], details: {} }; +} +``` + +**String enums:** Prefer `StringEnum` from `@earendil-works/pi-ai`. It produces a plain `enum` schema that every provider accepts. + +**Structured output:** Declare `outputSchema` and return a matching `structuredContent` when the result is data. The model still receives `content`; programmatic callers such as codemode scripts receive `structuredContent` instead of the text. Tools without `outputSchema` are passed to scripts as their text content. To report a failure that still carries data, return the result with `isError: true` instead of throwing: the model sees an error, and scripts still receive `structuredContent`. + +**Execution mode:** Tool calls from one assistant message run in parallel by default. Set `executionMode: "sequential"` when a tool must not run concurrently with other calls, for example because it shares mutable in-memory state. + +**Other definition fields:** + +| Field | Purpose | +|---|---| +| `defaultActive` | Set `false` to register the tool inactive; `--tools`, the `defaultTools` setting, or `pi.setActiveTools()` activates it | +| `exposure`, `namespace`, `annotations`, `reachesTools`, `prepareLoadout` | Control how the model and other tools reach the tool; see [Tool Exposure](#tool-exposure) | +| `renderShell`, `renderCall`, `renderResult` | Custom rendering; see [Custom Rendering](#custom-rendering) | +| `constrainedSampling` | Optional provider-side constrained sampling request for the tool's arguments | + +Use `defineTool()` to keep parameter type inference when you assign a tool definition to a variable before registering it. + +**Argument preparation:** `prepareArguments(args)` is optional. If defined, it runs before schema validation and before `execute()`. Use it to mimic an older accepted input shape when Pi resumes an older session whose stored tool call arguments no longer match the current schema. Return the object you want validated against `parameters`. Keep the public schema strict. Do not add deprecated compatibility fields to `parameters` just to keep old resumed sessions working. + +Example: an older session may contain an `edit` tool call with top-level `oldText` and `newText`, while the current schema only accepts `edits: [{ oldText, newText }]`. + +```typescript +const editSchema = Type.Object({ + path: Type.String(), + edits: Type.Array( + Type.Object({ + oldText: Type.String(), + newText: Type.String(), + }), + ), +}); +type EditInput = Static; + +pi.registerTool({ + name: "edit", + label: "Edit", + description: "Edit a single file using exact text replacement", + parameters: editSchema, + prepareArguments(args) { + const input = (args ?? {}) as Partial & { oldText?: unknown; newText?: unknown }; + if (typeof input.oldText !== "string" || typeof input.newText !== "string") { + return args as EditInput; // validation reports anything still invalid + } + + const { oldText, newText, ...rest } = input; + return { ...rest, edits: [...(input.edits ?? []), { oldText, newText }] } as EditInput; + }, + async execute(toolCallId, params, signal, onUpdate, ctx) { + // params now matches the current schema + return { + content: [{ type: "text", text: `Applying ${params.edits.length} edit block(s)` }], + details: {}, + }; + }, +}); +``` + +### Overriding Built-in Tools + +Extensions can override built-in tools (`read`, `bash`, `powershell`, `edit`, `write`, `grep`, `find`, `ls`) by registering a tool with the same name. The extension tool replaces the built-in without a warning. + +```bash +# Extension's read tool replaces built-in read +pi -e ./tool-override.ts +``` + +Alternatively, use `--no-builtin-tools` to start without any built-in tools while keeping extension tools enabled: +```bash +# No built-in tools, only extension tools +pi --no-builtin-tools -e ./my-extension.ts +``` + +See [examples/extensions/tool-override.ts](../examples/extensions/tool-override.ts) for a complete example that overrides `read` with logging and access control. + +**Rendering:** Built-in renderer inheritance is resolved per slot. Execution override and rendering override are independent. If your override omits `renderCall`, the built-in `renderCall` is used. If your override omits `renderResult`, the built-in `renderResult` is used. If your override omits both, the built-in renderer is used automatically (syntax highlighting, diffs, etc.). This lets you wrap built-in tools for logging or access control without reimplementing the UI. + +**Prompt metadata:** `promptSnippet` and `promptGuidelines` are not inherited from the built-in tool. If your override should keep those prompt instructions, define them on the override explicitly. + +**Your implementation must match the exact result shape**, including the `details` type. The UI and session logic depend on these shapes for rendering and state tracking. + +Built-in tool implementations: +- [read.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/read.ts) - `ReadToolDetails` +- [bash.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/bash.ts) - `BashToolDetails` +- [powershell.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/powershell.ts) - `PowerShellToolDetails` +- [edit.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/edit.ts) +- [write.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/write.ts) +- [grep.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/grep.ts) - `GrepToolDetails` +- [find.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/find.ts) - `FindToolDetails` +- [ls.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/ls.ts) - `LsToolDetails` + +### Remote Execution + +Built-in tools support pluggable operations for delegating to remote systems (SSH, containers, etc.): + +```typescript +import { createReadTool, type ReadOperations } from "@earendil-works/pi-coding-agent"; + +const localRead = createReadTool(cwd); + +// Register, checking the remote configuration at execution time +pi.registerTool({ + ...localRead, + async execute(id, params, signal, onUpdate, _ctx) { + const ssh = getSshConfig(); + if (ssh) { + const operations: ReadOperations = { + readFile: (path) => sshReadBuffer(ssh, path), // must resolve to a Buffer + access: (path) => sshExec(ssh, `test -r ${path}`).then(() => {}), + }; + return createReadTool(cwd, { operations }).execute(id, params, signal, onUpdate); + } + return localRead.execute(id, params, signal, onUpdate); + }, +}); +``` + +The `create*ToolDefinition()` factories, such as `createReadToolDefinition()`, return complete tool definitions, including the built-in prompt snippet and guidelines, for the same purpose. Rendering does not need copying: a tool registered under a built-in name inherits the built-in renderer for each slot it leaves out (see [Overriding Built-in Tools](#overriding-built-in-tools)). + +**Operations interfaces:** `ReadOperations`, `WriteOperations`, `EditOperations`, `BashOperations`, `PowerShellOperations`, `LsOperations`, `GrepOperations`, `FindOperations` + +For `user_bash`, extensions can reuse Pi's local shell backend via `createLocalBashOperations()` (or `createLocalPowerShellOperations()`) instead of reimplementing local process spawning, shell resolution, and process-tree termination. + +The `bash` and `powershell` tools also support a spawn hook to adjust the command, cwd, or env before execution: + +```typescript +import { createBashTool } from "@earendil-works/pi-coding-agent"; + +const bashTool = createBashTool(cwd, { + spawnHook: ({ command, cwd, env }) => ({ + command: `source ~/.profile\n${command}`, + cwd: `/mnt/sandbox${cwd}`, + env: { ...env, CI: "1" }, + }), +}); +``` + +`createBashTool()` and `createPowerShellTool()` expose the current session to commands through `PI_SESSION_ID`, `PI_SESSION_FILE`, `PI_PROVIDER`, `PI_MODEL`, and `PI_REASONING_LEVEL`. Injection happens before `spawnHook`, so hooks receive these values in `env` and preserve them when they spread the existing environment as above. Set `exposeSessionEnvironment: false` to disable them: + +```typescript +const bashTool = createBashTool(cwd, { + exposeSessionEnvironment: false, +}); +``` + +See [Shell tool session environment](environment-variables.md#shell-tool-session-environment) for variable semantics. See [examples/extensions/ssh.ts](../examples/extensions/ssh.ts) for a complete SSH example with `--ssh` flag. + +### Output Truncation + +**Tools MUST truncate their output** to avoid overwhelming the LLM context. Large outputs can cause: +- Context overflow errors (prompt too long) +- Compaction failures +- Degraded model performance + +The built-in limit is **50KB** (~10k tokens) and **2000 lines**, whichever is hit first. Use the exported truncation utilities: + +```typescript +import { + truncateHead, // Keep first N lines/bytes (good for file reads, search results) + truncateTail, // Keep last N lines/bytes (good for logs, command output) + truncateLine, // Truncate a single line to a character count; returns { text, wasTruncated } + formatSize, // Human-readable size (e.g., "50KB", "1.5MB") + DEFAULT_MAX_BYTES, // 50KB + DEFAULT_MAX_LINES, // 2000 +} from "@earendil-works/pi-coding-agent"; + +async execute(toolCallId, params, signal, onUpdate, ctx) { + const output = await runCommand(); + + // Apply truncation + const truncation = truncateHead(output, { + maxLines: DEFAULT_MAX_LINES, + maxBytes: DEFAULT_MAX_BYTES, + }); + + let result = truncation.content; + + if (truncation.truncated) { + // Write full output to temp file + const tempFile = writeTempFile(output); + + // Inform the LLM where to find complete output + result += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines`; + result += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}).`; + result += ` Full output saved to: ${tempFile}]`; + } + + return { content: [{ type: "text", text: result }], details: { truncation } }; +} +``` + +**Key points:** +- Use `truncateHead` for content where the beginning matters (search results, file reads) +- Use `truncateTail` for content where the end matters (logs, command output) +- Always inform the LLM when output is truncated and where to find the full version +- Document the truncation limits in your tool's description + +See [examples/extensions/truncated-tool.ts](../examples/extensions/truncated-tool.ts) for a complete example wrapping `rg` (ripgrep) with proper truncation. + +### Multiple Tools + +One extension can register multiple tools with shared state: + +```typescript +export default function (pi: ExtensionAPI) { + let connection = null; + + pi.registerTool({ name: "db_connect", ... }); + pi.registerTool({ name: "db_query", ... }); + pi.registerTool({ name: "db_close", ... }); + + pi.on("session_shutdown", async () => { + connection?.close(); + }); +} +``` + + + +### Custom Rendering + +Tools can provide `renderCall` and `renderResult` for custom TUI display. See [tui.md](tui.md) for the full component API and [tool-execution.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/modes/interactive/components/tool-execution.ts) for how tool rows are composed. + +By default, tool output is wrapped in a `Box` that handles background and horizontal padding from the `outputPad` setting. Tool renderers are also used in HTML exports. A defined `renderCall` or `renderResult` must return a `Component`. If a slot renderer is not defined, `tool-execution.ts` uses fallback rendering for that slot. + +Set `renderShell: "self"` when the tool should render its own shell instead of using the default `Box`. This is useful for tools that need complete control over framing or background behavior, for example large previews that must stay visually stable after the tool settles. + +```typescript +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "Custom shell example", + parameters: Type.Object({}), + renderShell: "self", + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + renderCall(args, theme, context) { + return new Text(theme.fg("accent", "my custom shell"), 0, 0); + }, +}); +``` + +`renderCall` and `renderResult` each receive a `context` object with: +- `args` - the current tool call arguments +- `state` - shared row-local state across `renderCall` and `renderResult` +- `lastComponent` - the previously returned component for that slot, if any +- `invalidate()` - request a rerender of this tool row +- `toolCallId`, `cwd`, `executionStarted`, `argsComplete`, `isPartial`, `expanded`, `showImages`, `isError` +- `durationMs` - how long execution took, from the final result; `undefined` while running +- `outputPad` - horizontal padding from the `outputPad` setting; renderers with `renderShell: "self"` apply it themselves +- `keyText()`, `keyHint()` - format the user's configured keybindings, see [Keybinding Hints](#keybinding-hints) + +Use `context.state` for cross-slot shared state. Keep slot-local caches on the returned component instance when you want to reuse and mutate the same component across renders. + +#### renderCall + +Renders the tool call or header: + +```typescript +import { Text } from "@earendil-works/pi-tui"; + +renderCall(args, theme, context) { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + let content = theme.fg("toolTitle", theme.bold("my_tool ")); + content += theme.fg("muted", args.action); + if (args.text) { + content += " " + theme.fg("dim", `"${args.text}"`); + } + text.setText(content); + return text; +} +``` + +#### renderResult + +Renders the tool result or output: + +```typescript +renderResult(result, { expanded, isPartial }, theme, context) { + if (isPartial) { + return new Text(theme.fg("warning", "Processing..."), 0, 0); + } + + if (result.details?.error) { + return new Text(theme.fg("error", `Error: ${result.details.error}`), 0, 0); + } + + let text = theme.fg("success", "✓ Done"); + if (expanded && result.details?.items) { + for (const item of result.details.items) { + text += "\n " + theme.fg("dim", item); + } + } + return new Text(text, 0, 0); +} +``` + +If a slot intentionally has no visible content, return an empty `Component` such as an empty `Container`. + +#### Keybinding Hints + +Use `keyHint()` and `keyText()` from the render context to display keybinding hints that respect the user's keybinding configuration: + +```typescript +renderResult(result, { expanded }, theme, context) { + let text = theme.fg("success", "✓ Done"); + if (!expanded) { + text += ` (${context.keyHint("app.tools.expand", "to expand")})`; + } + return new Text(text, 0, 0); +} +``` + +The same functions are available as `options.keyHint()`/`options.keyText()` in message and entry renderers: + +- `keyHint(keybinding, description)` - Formats a configured keybinding id such as `"app.tools.expand"` or `"tui.select.confirm"`, followed by the description +- `keyText(keybinding)` - Returns the raw configured key text for a keybinding id, or `""` if the id has no keys + +In renderers, use these instead of importing `keyHint()` or `keyText()` from `@earendil-works/pi-coding-agent`. An extension can resolve its own copy of the package, for example from its own `node_modules`, and that copy does not see pi's configured keybindings: `app.*` ids then format as `""`. `rawKeyHint(key, description)` formats a literal key string and is safe to import. + +Use namespaced keybinding ids: +- Coding-agent ids use the `app.*` namespace, for example `app.tools.expand`, `app.editor.external`, `app.session.rename` +- Shared TUI ids use the `tui.*` namespace, for example `tui.select.confirm`, `tui.select.cancel`, `tui.input.tab` + +For the exhaustive list of keybinding ids and defaults, see [keybindings.md](keybindings.md). `keybindings.json` uses those same namespaced ids. + +Custom editors and `ctx.ui.custom()` components receive `keybindings: KeybindingsManager` as an injected argument. They should use that injected manager directly instead of calling `getKeybindings()` or `setKeybindings()`. + +#### Best Practices + +- Use `Text` with padding `(0, 0)`. The default Box handles padding. +- Use `\n` for multi-line content. +- Handle `isPartial` for streaming progress. +- Support `expanded` for detail on demand. +- Keep default view compact. +- Read `context.args` in `renderResult` instead of copying args into `context.state`. +- Use `context.state` only for data that must be shared across call and result slots. +- Reuse `context.lastComponent` when the same component instance can be updated in place. +- Use `renderShell: "self"` only when the default boxed shell gets in the way. In self-shell mode the tool is responsible for its own framing, padding, and background. + +#### Fallback + +If a slot renderer is not defined or throws: +- `renderCall`: Shows the tool name +- `renderResult`: Shows raw text from `content` + +### Dynamic Tool Loading + +Extensions can register many tools while keeping only a small initial set active. A tool can then change the active set with `pi.setActiveTools()` during execution. Pi stores the initial prompt and tool loadout in the transcript's first system message, then appends tool and prompt deltas before the next model request. Providers that cannot represent a transition receive a complete transcript checkpoint, which may invalidate the cached prefix. + +The lifecycle is: + +1. Register every tool with `pi.registerTool()` so it appears in `pi.getAllTools()`. Register searchable tools with `defaultActive: false` so they start inactive. +2. Keep loader tools, such as `search_tools`, active. +3. During loader execution, call `pi.setActiveTools()` with the desired active tool names. Names must already be registered; unknown and `hidden` names are ignored. + +Pi also ships a built-in `tool_search` tool. Register tools with `exposure: "deferred"` to make them findable by it instead of writing your own loader; see [Tool Exposure](#tool-exposure). + +#### Search tool example + +The following extension registers two inactive searchable tools and keeps `search_tools` active as their loader. The example uses simple keyword matching, but the search implementation could use BM25, embeddings, a remote catalog, or project-specific routing. + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; + +const SEARCHABLE_TOOL_NAMES = new Set(["lookup_weather", "search_issues"]); + +export default function (pi: ExtensionAPI) { + pi.registerTool({ + name: "lookup_weather", + label: "Lookup Weather", + defaultActive: false, + description: "Look up the current weather for a city", + parameters: Type.Object({ city: Type.String() }), + async execute(_toolCallId, params) { + return { + content: [{ type: "text", text: `Weather for ${params.city}: sunny` }], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "search_issues", + label: "Search Issues", + defaultActive: false, + description: "Search project issues by keyword", + parameters: Type.Object({ query: Type.String() }), + async execute(_toolCallId, params) { + return { + content: [{ type: "text", text: `No open issues matching ${params.query}` }], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "search_tools", + label: "Search Tools", + description: "Search for and enable tools relevant to a task", + promptSnippet: "Search for additional tools when the active tools cannot perform the task", + promptGuidelines: [ + "Use search_tools when a task requires a capability that is not currently available.", + ], + parameters: Type.Object({ + query: Type.String({ description: "Capability or task to search for" }), + limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })), + }), + async execute(_toolCallId, params) { + const terms = params.query.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); + const matches = pi.getAllTools() + .filter((tool) => SEARCHABLE_TOOL_NAMES.has(tool.name)) + .map((tool) => ({ + tool, + score: terms.reduce( + (score, term) => + score + (`${tool.name} ${tool.description}`.toLowerCase().includes(term) ? 1 : 0), + 0, + ), + })) + .filter((match) => match.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, params.limit ?? 3) + .map((match) => match.tool.name); + + if (matches.length === 0) { + return { + content: [{ type: "text", text: `No tools found for: ${params.query}` }], + details: { matches: [] as string[], added: [] as string[] }, + }; + } + + const active = pi.getActiveTools(); + const added = matches.filter((name) => !active.includes(name)); + pi.setActiveTools([...new Set([...active, ...added])]); + + return { + content: [{ + type: "text", + text: added.length > 0 + ? `Loaded tools: ${added.join(", ")}` + : `Matching tools already active: ${matches.join(", ")}`, + }], + details: { matches, added }, + }; + }, + }); +} +``` + +When `search_tools` adds a match, the model receives the complete updated tool list on the immediately following request. + +### Tool Exposure `exposure` controls how the model reaches a tool. "Callable" means callable from other tools through `ctx.executeTool()` (`ctx.tools`), as the `codemode` tool's scripts do: @@ -274,7 +2747,7 @@ See [`hello.ts`](../examples/extensions/hello.ts), [`todo.ts`](../examples/exten `namespace: { name, aliases, description, instructions }` groups related tools, as MCP servers do. Codemode tools list a namespace under one heading with its `description`. `instructions` holds longer usage guidance; it is not listed, and codemode scripts read it with `describeNamespace(name)`. `aliases` are other names scripts may look the namespace up by, for example the server name `docs` for `mcp__docs`; a name and the same name as a script identifier (`my-docs`, `my_docs`) match alike. -Registering a `direct` or `model-only` tool activates it; the other exposures are not activated on registration. The active set (`pi.getActiveTools()`, `pi.setActiveTools()`) is the set of tools declared to the model. `pi.getAllTools()` reports each tool's `exposure`, `namespace`, `annotations`, and `reachesTools`. +Registering a `direct` or `model-only` tool activates it unless it sets `defaultActive: false`; the other exposures are not activated on registration. The active set (`pi.getActiveTools()`, `pi.setActiveTools()`) is the set of tools declared to the model, except active tools whose declarations `prepareLoadout` hides. `pi.getAllTools()` reports each tool's `exposure`, `namespace`, `annotations`, and `reachesTools`. `annotations` are hints about what a tool does, with the meaning of MCP tool annotations: `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint`. MCP tools carry the hints their server declares. Missing hints take the MCP defaults: a tool is not read-only, and may be destructive and reach an open world. The hints are not verified, but a permission extension can use them to decide which calls to confirm. This confirms the calls Codex asks approval for: @@ -294,17 +2767,17 @@ A tool that orchestrates other tools can adjust what the model sees while it is A tool that reaches other tools declares which ones a call may reach with `reachesTools(input)`: `"all"`, or the words of the input that may name them, where a word refers to every tool or namespace whose name it contains. `codemode` returns `"all"` for scripts that search or list tools and the words of the script otherwise; `tool_search` always returns `"all"`. Extensions that register tools late use it from their `tool_call` handler to finish registering the tools a call needs before it runs, as the MCP extension does for servers that are still connecting. The MCP extension also activates `codemode` or `tool_search` for MCP tools only when the tool of that name declares `reachesTools`. -### Activate tools dynamically +### Calling Other Tools -Register every tool first, keep optional tools inactive, and use `pi.setActiveTools()` from a loader tool to select the desired active tools. Names must already be registered; unknown names are ignored. +A tool can run other tools with `ctx.executeTool(name, args, { signal, onUpdate })`. `ctx.tools` lists the tools it can call. The call never rejects for tool failures: unknown tools, validation errors, blocked calls, and thrown errors come back as results with `isError: true`. -Pi records the initial prompt and tool set in the transcript's first system message, then appends tool and prompt changes before the next model request. Providers that cannot represent the transition receive a complete transcript checkpoint, which can invalidate the cached prefix. +Nested calls go through argument validation and the `tool_call` and `tool_result` handlers like model-issued calls, and emit `tool_execution_start`, `tool_execution_update`, and `tool_execution_end`. All of these events carry `parentToolCallId`, and their `toolCallId` is assigned by Pi as `/`. These IDs do not appear as tool calls or tool results in the transcript. -### Tool rendering +Nested calls do not add transcript entries: their results only reach the calling tool, which reports them itself, for example through `onUpdate` and `details`. The session keeps a bounded record of them (name, arguments, status, duration, error; never results) as `nestedCalls` on the calling tool's result message. It is used for compaction file lists and shown in HTML exports. Arguments over 8 KiB per call or 32 KiB per tool result are omitted, at most 256 calls are kept, and `complete: false` marks a record that lost anything. -A tool's `renderCall` and `renderResult` draw its calls in the interactive transcript and in HTML exports. `pi.registerToolRenderer((toolName, next) => renderers)` chooses renderers for calls to any tool, including tools that are not registered yet, such as MCP tools in a resumed session before their server connected. `next()` returns what the remaining resolvers (in extension load order), then the registered tool, would use, so `next() ?? mine` only fills in. +The `usage` of nested results, at every depth, is added to the calling tool's result `usage`, so a tool reports only its own usage, not that of the tools it called. -### MCP servers +## MCP Servers `pi.registerMcpServer(name, config)` adds an MCP server for the current session. `config` has the shape of an `mcpServers` entry in [`mcp.json`](mcp.md): `command`, `args`, `env`, and `cwd` for stdio servers, `url`, `headers`, and `oauth` for HTTP servers, plus `exposure`, `toolExposure`, `description`, `enabled`, and `timeout`. @@ -317,84 +2790,558 @@ Servers registered while the extension loads connect when the session starts, to The built-in MCP support connects registered servers. When nothing does, because another extension replaced it (see [MCP](mcp.md#replace-the-built-in-mcp-support)), each registration is reported as an extension error. Other MCP extensions can connect registered servers too: read them with `pi.getMcpServers()` on `session_start` and handle the `mcp_servers_change` event for later changes. - - - - -### Context and session changes - -`ExtensionContext` provides the working directory, mode, UI, session manager, model runtime, abort signal, context usage, and controls for compaction and shutdown. -Use `ctx.modelRegistry.streamSimple()` for provider-neutral nested model calls. - -Command handlers receive `ExtensionCommandContext`, which adds operations for waiting until idle, reloading, tree navigation, and session replacement. -These operations are command-only because calling them from lifecycle handlers can deadlock the runtime. - -Session replacement invalidates the old context. Capture only plain data before switching, then use the fresh context supplied to `withSession` for session-bound work. - - - - -### State - -Choose storage based on how state participates in the conversation: - -| State | Storage | -|---|---| -| Tool state that follows the active branch | Tool-result `details` | -| Durable data excluded from model context | `pi.appendEntry()` | -| Custom content stored and sent to the model | `pi.sendMessage()` | -| Data outside one session | External storage | - -Reconstruct branch-sensitive state from `ctx.sessionManager.getBranch()` during `session_start`. -Do not rebuild it from every file entry because abandoned branches represent alternative histories. -Register an entry or message renderer when custom stored content should appear in the transcript. - - - - -### UI and modes +## Custom UI -`ctx.ui` provides dialogs, notifications, status text, widgets, titles, editor access, and custom components. -Use `ctx.ui.custom()` only when the interaction needs its own rendering and input. -See [Terminal UI](tui.md) for component, focus, overlay, theme, and performance guidance. +Extensions can interact with users via `ctx.ui` methods and customize how messages/tools render. -Extensions load in interactive, RPC, JSON, and print modes. -Interactive mode provides the complete terminal UI. -RPC can forward supported dialogs and notifications through the [RPC Extension UI protocol](rpc-extension-ui.md), but not custom terminal components; JSON and print modes have no UI. -Guard terminal-only behavior with `ctx.mode === "tui"` and use `ctx.hasUI` for interactions supported by interactive and RPC clients. +For custom components, focus, overlays, themes, and rendering performance, see [Terminal UI](tui.md). The examples it lists show selection lists, settings toggles, cancellable loaders, footers, and widgets. -Keep tool and event behavior independent from rendering so non-interactive modes remain functional. +### Dialogs + +```typescript +// Select from options +const choice = await ctx.ui.select("Pick one:", ["A", "B", "C"]); + +// Confirm dialog +const ok = await ctx.ui.confirm("Delete?", "This cannot be undone"); + +// Text input +const name = await ctx.ui.input("Name:", "placeholder"); + +// Multi-line editor +const text = await ctx.ui.editor("Edit:", "prefilled text"); + +// Notification (non-blocking) +ctx.ui.notify("Done!", "info"); // "info" | "warning" | "error" +``` + +#### Timed Dialogs with Countdown + +Dialogs support a `timeout` option that auto-dismisses with a live countdown display: + +```typescript +// Dialog shows "Title (5s)" → "Title (4s)" → ... → auto-dismisses at 0 +const confirmed = await ctx.ui.confirm( + "Timed Confirmation", + "This dialog will auto-cancel in 5 seconds. Confirm?", + { timeout: 5000 } +); + +if (confirmed) { + // User confirmed +} else { + // User cancelled or timed out +} +``` + +**Return values on timeout:** +- `select()` returns `undefined` +- `confirm()` returns `false` +- `input()` returns `undefined` + +#### Manual Dismissal with AbortSignal + +For more control (e.g., to distinguish timeout from user cancel), use `AbortSignal`: + +```typescript +const controller = new AbortController(); +const timeoutId = setTimeout(() => controller.abort(), 5000); + +const confirmed = await ctx.ui.confirm( + "Timed Confirmation", + "This dialog will auto-cancel in 5 seconds. Confirm?", + { signal: controller.signal } +); + +clearTimeout(timeoutId); + +if (confirmed) { + // User confirmed +} else if (controller.signal.aborted) { + // Dialog timed out +} else { + // User cancelled (pressed Escape or selected "No") +} +``` + +See [examples/extensions/timed-confirm.ts](../examples/extensions/timed-confirm.ts) for complete examples. + +### Widgets, Status, and Footer + +```typescript +// Status in footer (persistent until cleared) +ctx.ui.setStatus("my-ext", "Processing..."); +ctx.ui.setStatus("my-ext", undefined); // Clear + +// Working loader (shown during streaming) +ctx.ui.setWorkingMessage("Thinking deeply..."); +ctx.ui.setWorkingMessage(); // Restore default +ctx.ui.setWorkingVisible(false); // Hide the built-in working loader row entirely +ctx.ui.setWorkingVisible(true); // Show the built-in working loader row + +// Working indicator (shown during streaming) +ctx.ui.setWorkingIndicator({ frames: [ctx.ui.theme.fg("accent", "●")] }); // Static dot +ctx.ui.setWorkingIndicator({ + frames: [ + ctx.ui.theme.fg("dim", "·"), + ctx.ui.theme.fg("muted", "•"), + ctx.ui.theme.fg("accent", "●"), + ctx.ui.theme.fg("muted", "•"), + ], + intervalMs: 120, +}); +ctx.ui.setWorkingIndicator({ frames: [] }); // Hide indicator +ctx.ui.setWorkingIndicator(); // Restore default spinner + +// Widget above editor (default) +ctx.ui.setWidget("my-widget", ["Line 1", "Line 2"]); +// Widget below editor +ctx.ui.setWidget("my-widget", ["Line 1", "Line 2"], { placement: "belowEditor" }); +ctx.ui.setWidget("my-widget", (tui, theme) => new Text(theme.fg("accent", "Custom"), 0, 0)); +ctx.ui.setWidget("my-widget", undefined); // Clear + +// Custom footer (replaces built-in footer entirely) +// footerData provides the git branch and extension statuses from setStatus() +ctx.ui.setFooter((tui, theme, footerData) => ({ + render(width) { return [theme.fg("dim", "Custom footer")]; }, + invalidate() {}, +})); +ctx.ui.setFooter(undefined); // Restore built-in footer + +// Custom header shown at startup above the chat +ctx.ui.setHeader((tui, theme) => new Text(theme.fg("accent", "my-project"), 0, 0)); +ctx.ui.setHeader(undefined); // Restore built-in header + +// Label shown for hidden thinking blocks +ctx.ui.setHiddenThinkingLabel("Reasoning..."); +ctx.ui.setHiddenThinkingLabel(); // Restore default + +// Raw terminal input (interactive mode only); return { consume: true } to swallow it +const unsubscribe = ctx.ui.onTerminalInput((data) => undefined); + +// Terminal title +ctx.ui.setTitle("pi - my-project"); + +// Editor text +ctx.ui.setEditorText("Prefill text"); +const current = ctx.ui.getEditorText(); + +// Paste into editor (triggers paste handling, including collapse for large content) +ctx.ui.pasteToEditor("pasted content"); + +// Stack custom autocomplete behavior on top of the built-in provider +ctx.ui.addAutocompleteProvider((current) => ({ + triggerCharacters: ["#"], + async getSuggestions(lines, line, col, options) { + const beforeCursor = (lines[line] ?? "").slice(0, col); + const match = beforeCursor.match(/(?:^|[ \t])#([^\s#]*)$/); + if (!match) { + return current.getSuggestions(lines, line, col, options); + } + + return { + prefix: `#${match[1] ?? ""}`, + items: [{ value: "#2983", label: "#2983", description: "Extension API for autocomplete" }], + }; + }, + applyCompletion(lines, line, col, item, prefix) { + return current.applyCompletion(lines, line, col, item, prefix); + }, + shouldTriggerFileCompletion(lines, line, col) { + return current.shouldTriggerFileCompletion?.(lines, line, col) ?? true; + }, +})); + +// Tool output expansion +const wasExpanded = ctx.ui.getToolsExpanded(); +ctx.ui.setToolsExpanded(true); +ctx.ui.setToolsExpanded(wasExpanded); + +// Custom editor (vim mode, emacs mode, etc.) +ctx.ui.setEditorComponent((tui, theme, keybindings) => new VimEditor(tui, theme, keybindings)); +const currentEditor = ctx.ui.getEditorComponent(); +ctx.ui.setEditorComponent((tui, theme, keybindings) => + new WrappedEditor(tui, theme, keybindings, currentEditor?.(tui, theme, keybindings)) +); +ctx.ui.setEditorComponent(undefined); // Restore default editor + +// Theme management (see themes.md for creating themes) +const themes = ctx.ui.getAllThemes(); // [{ name: "dark", path: "/..." | undefined }, ...] +const lightTheme = ctx.ui.getTheme("light"); // Load without switching +const result = ctx.ui.setTheme("light"); // Switch by name +if (!result.success) { + ctx.ui.notify(`Failed: ${result.error}`, "error"); +} +ctx.ui.setTheme(lightTheme!); // Or switch by Theme object +ctx.ui.theme.fg("accent", "styled text"); // Access current theme +``` + +Custom working-indicator frames are rendered verbatim. If you want colors, add them to the frame strings yourself, for example with `ctx.ui.theme.fg(...)`. + +### Autocomplete Providers + +Use `ctx.ui.addAutocompleteProvider()` to stack custom autocomplete logic on top of the built-in slash-command and path provider. Set `triggerCharacters` for custom natural triggers such as `$`. + +Typical pattern: + +- inspect the text before the cursor +- return your own suggestions when your extension-specific syntax matches +- otherwise delegate to `current.getSuggestions(...)` +- delegate `applyCompletion(...)` unless you need custom insertion behavior + +```typescript +pi.on("session_start", (_event, ctx) => { + ctx.ui.addAutocompleteProvider((current) => ({ + triggerCharacters: ["#"], + async getSuggestions(lines, cursorLine, cursorCol, options) { + const line = lines[cursorLine] ?? ""; + const beforeCursor = line.slice(0, cursorCol); + const match = beforeCursor.match(/(?:^|[ \t])#([^\s#]*)$/); + if (!match) { + return current.getSuggestions(lines, cursorLine, cursorCol, options); + } + + return { + prefix: `#${match[1] ?? ""}`, + items: [ + { value: "#2983", label: "#2983", description: "Extension API for registering custom @ autocomplete providers" }, + { value: "#2753", label: "#2753", description: "Reload stale resource settings" }, + ], + }; + }, + + applyCompletion(lines, cursorLine, cursorCol, item, prefix) { + return current.applyCompletion(lines, cursorLine, cursorCol, item, prefix); + }, + + shouldTriggerFileCompletion(lines, cursorLine, cursorCol) { + return current.shouldTriggerFileCompletion?.(lines, cursorLine, cursorCol) ?? true; + }, + })); +}); +``` + +See [github-issue-autocomplete.ts](../examples/extensions/github-issue-autocomplete.ts) for a complete example that preloads the latest open GitHub issues with `gh issue list` and filters them locally for fast `#...` completion. It requires GitHub CLI (`gh`) and a GitHub repository checkout. + +### Custom Components + +For complex UI, use `ctx.ui.custom()`. This temporarily replaces the editor with your component until `done()` is called: + +```typescript +import { truncateToWidth } from "@earendil-works/pi-tui"; + +const result = await ctx.ui.custom((tui, theme, keybindings, done) => ({ + render(width: number): string[] { + return [truncateToWidth(theme.fg("accent", "Press Enter to confirm, Escape to cancel"), width)]; + }, + handleInput(data: string): void { + if (keybindings.matches(data, "tui.select.confirm")) done(true); + else if (keybindings.matches(data, "tui.select.cancel")) done(false); + }, + invalidate() {}, +})); + +if (result) { + // User pressed Enter +} +``` + +The callback receives: +- `tui` - TUI instance (for screen dimensions, focus management) +- `theme` - Current theme for styling +- `keybindings` - App keybinding manager (for checking shortcuts) +- `done(value)` - Call to close component and return value + +The factory can also return a `Promise` of a component. See [Terminal UI](tui.md#build-a-minimal-custom-screen) for a complete custom screen with cached rendering. + +#### Overlay Mode (Experimental) + +Pass `{ overlay: true }` to render the component as a floating modal on top of existing content, without clearing the screen: + +```typescript +const result = await ctx.ui.custom( + (tui, theme, keybindings, done) => new MyOverlayComponent({ onClose: done }), + { overlay: true } +); +``` + +For advanced positioning (anchors, margins, percentages, responsive visibility), pass `overlayOptions`. Use `onHandle` to control focus or visibility programmatically: + +```typescript +const result = await ctx.ui.custom( + (tui, theme, keybindings, done) => new MyOverlayComponent({ onClose: done }), + { + overlay: true, + overlayOptions: { anchor: "top-right", width: "50%", margin: 2 }, + onHandle: (handle) => { + handle.focus(); // focus this overlay and bring it to the visual front + // handle.unfocus({ target: editorComponent }); // release input to a specific component + // handle.setHidden(true/false); // toggle visibility + // handle.isHidden(), handle.isFocused(), handle.getBounds() + } + } +); +``` + +`overlayOptions` can also be a function, re-evaluated for dynamic positioning. Finish the interaction with `done()`; do not call `handle.hide()` on an overlay created by `ctx.ui.custom()`. + +A focused visible overlay can reclaim input after temporary non-overlay custom UI closes. If you intentionally want another component to keep input while the overlay stays visible, call `handle.unfocus({ target })`. Passing `{ target: null }` releases the overlay without focusing another component. + +See [`OverlayOptions` and `OverlayHandle`](https://github.com/earendil-works/pi/blob/main/packages/tui/src/tui.ts) for the full API and [overlay-qa-tests.ts](../examples/extensions/overlay-qa-tests.ts) for examples. + +### Custom Editor + +Replace the main input editor with a custom implementation (vim mode, emacs mode, etc.): + +```typescript +import { CustomEditor, type ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { matchesKey } from "@earendil-works/pi-tui"; + +class VimEditor extends CustomEditor { + private mode: "normal" | "insert" = "insert"; + + handleInput(data: string): void { + if (matchesKey(data, "escape") && this.mode === "insert") { + this.mode = "normal"; + return; + } + if (this.mode === "normal" && data === "i") { + this.mode = "insert"; + return; + } + super.handleInput(data); // App keybindings + text editing + } +} + +export default function (pi: ExtensionAPI) { + pi.on("session_start", (_event, ctx) => { + ctx.ui.setEditorComponent((tui, theme, keybindings) => + new VimEditor(tui, theme, keybindings) + ); + }); +} +``` + +**Key points:** +- Extend `CustomEditor` (not base `Editor`) to get app keybindings (escape to abort, ctrl+d, model switching) +- Call `super.handleInput(data)` for keys you don't handle +- Custom editors keep the standalone working row by default. Pass `{ embedWorkingStatus: true }` as the fourth `CustomEditor` constructor argument to use the built-in editor-border spinner instead. +- Factory receives `tui`, `theme`, and `keybindings` from the app +- Use `ctx.ui.getEditorComponent()` before `setEditorComponent()` to wrap the previously configured custom editor +- Pass `undefined` to restore default: `ctx.ui.setEditorComponent(undefined)` + +To compose with another extension that already replaced the editor, capture the previous factory before setting yours: + +```typescript +const previous = ctx.ui.getEditorComponent(); +ctx.ui.setEditorComponent((tui, theme, keybindings) => + new MyEditor(tui, theme, keybindings, { base: previous?.(tui, theme, keybindings) }) +); +``` + +See [modal-editor.ts](../examples/extensions/modal-editor.ts) for a complete example with a mode indicator. + +### Message and Entry Rendering + +Register a custom renderer for messages with your `customType`. Use message renderers for content that should participate in LLM context: + +```typescript +import { Text } from "@earendil-works/pi-tui"; + +pi.registerMessageRenderer("my-extension", (message, options, theme) => { + const { expanded, outputPad } = options; + let text = theme.fg("accent", `[${message.customType}] `); + text += message.content; + + if (expanded && message.details) { + text += "\n" + theme.fg("dim", JSON.stringify(message.details, null, 2)); + } else if (message.details) { + text += ` (${options.keyHint("app.tools.expand", "to expand")})`; + } + + return new Text(text, outputPad, 0); +}); +``` + +Messages are sent via `pi.sendMessage()`: + +```typescript +pi.sendMessage({ + customType: "my-extension", // Matches registerMessageRenderer + content: "Status update", + display: true, // Show in TUI + details: { ... }, // Available in renderer +}); +``` + +For TUI-only content that should not be sent to the LLM, render custom entries instead: + +```typescript +pi.registerEntryRenderer("my-card", (entry, options, theme) => { + return new Text(theme.fg("accent", JSON.stringify(entry.data))); +}); + +pi.appendEntry("my-card", { status: "done" }); +``` + +### Theme Colors + +All render functions receive a `theme` object. See [themes.md](themes.md) for creating custom themes and the full color palette. + +```typescript +// Foreground colors +theme.fg("toolTitle", text) // Tool names +theme.fg("accent", text) // Highlights +theme.fg("success", text) // Success (green) +theme.fg("error", text) // Errors (red) +theme.fg("warning", text) // Warnings (yellow) +theme.fg("muted", text) // Secondary text +theme.fg("dim", text) // Tertiary text + +// Text styles +theme.bold(text) +theme.italic(text) +theme.underline(text) +theme.strikethrough(text) + +// Combined foreground, background, and attributes +theme.style(text, { fg: "success", bg: "toolSuccessBg", bold: true }) +``` + +`theme.colors` exposes concrete colors and `theme.appearance` is `"dark"` or `"light"`. See [Apply themes correctly](tui.md#apply-themes-correctly). + +For syntax highlighting in custom tool renderers: + +```typescript +import { highlightCode, getLanguageFromPath } from "@earendil-works/pi-coding-agent"; + +// Highlight code with an explicit language; returns one string per line +const lines = highlightCode("const x = 1;", "typescript"); + +// Auto-detect language from file path +const lang = getLanguageFromPath("/path/to/file.rs"); // "rust", or undefined +const highlighted = highlightCode(code, lang); +``` - -### Errors and cleanup +## Error Handling -Pi reports handler errors and continues where possible. A `tool_call` handler failure blocks the tool as a fail-safe; a tool execution failure becomes an error result for the model. +- Extension errors are logged, agent continues +- `tool_call` errors block the tool (fail-safe) +- `user_bash` errors block the command instead of falling through to local execution +- Tool `execute` errors are signaled by throwing or by returning `isError: true`; a thrown error is caught, reported to the LLM as an error result, and execution continues -Release resources in `session_shutdown` even when normal operation attempted cleanup. -Keep cleanup idempotent because cancellation, reload, session replacement, and process exit can converge on the same path. -Use `ctx.shutdown()` to request an orderly process shutdown. + + + +## Mode Behavior + +| Mode | `ctx.mode` | `ctx.hasUI` | Notes | +|------|------------|-------------|-------| +| Interactive | `"tui"` | `true` | Full TUI with terminal rendering | +| RPC (`--mode rpc`) | `"rpc"` | `true` | Dialogs and notifications via JSON protocol; `custom()` returns `undefined`, component factories are ignored. See [RPC Extension UI](rpc-extension-ui.md) | +| JSON (`--mode json`) | `"json"` | `false` | Event stream to stdout; UI methods are no-ops | +| Print (`-p`) | `"print"` | `false` | Extensions run but can't prompt | + +Use `ctx.mode === "tui"` before TUI-specific features (`custom()`, component factories, terminal input). Use `ctx.hasUI` before dialog and notification methods that work in both TUI and RPC modes. - -## Examples and reference +## Examples Reference -The checked [extension examples](../examples/extensions/) are the implementation reference. Start with the smallest example matching the behavior you need: +All examples in [examples/extensions/](../examples/extensions/). -| Goal | Start with | Why | -|---|---|---| -| Confirm or block tool calls | [`permission-gate.ts`](../examples/extensions/permission-gate.ts) | Combines `tool_call`, mode-aware UI, and fail-closed behavior. | -| Modify prompt sections for a run | [`prompt-customizer.ts`](../examples/extensions/prompt-customizer.ts) | Uses structured prompt options instead of replacing the complete prompt. | -| Keep branch-aware tool state | [`todo.ts`](../examples/extensions/todo.ts) | Reconstructs state from persisted entries and renders it in the TUI. | -| Register tools after startup | [`dynamic-tools.ts`](../examples/extensions/dynamic-tools.ts) | Shares registration logic between lifecycle and command handlers. | -| Run cancellable model work from a command | [`qna.ts`](../examples/extensions/qna.ts) | Combines nested model calls, cancellation, and custom UI. | -| Replace the main editor | [`modal-editor.ts`](../examples/extensions/modal-editor.ts) | Extends `CustomEditor` while preserving application controls. | -| Run finalization at shutdown | [`auto-commit-on-exit.ts`](../examples/extensions/auto-commit-on-exit.ts) | Uses the session shutdown hook for finalization work. | -| Route built-in tools to another machine | [`ssh.ts`](../examples/extensions/ssh.ts) | Reuses tool operation interfaces instead of recreating tool contracts. | -| Add OAuth and provider-specific streaming | [`custom-provider-gitlab-duo/`](../examples/extensions/custom-provider-gitlab-duo/) | Shows a complete provider integration while delegating supported protocols. | - -Use [Custom Providers](custom-provider.md) for model-service integrations, [Terminal UI](tui.md) for custom components, and [Pi Packages](packages.md) to install or distribute extensions with other resources. Use the exported declarations for exact signatures, payloads, and result schemas; this guide focuses on lifecycle choices, composition, and durable implementation patterns. +| Example | Description | Key APIs | +|---------|-------------|----------| +| **Tools** ||| +| `hello.ts` | Minimal tool registration | `registerTool` | +| `question.ts` | Tool with user interaction | `registerTool`, `ui.custom`, `renderResult` | +| `questionnaire.ts` | Multi-step wizard tool | `registerTool`, `ui.custom` | +| `todo.ts` | Stateful tool with branch-aware state in `details` | `registerTool`, `renderResult`, `registerCommand`, `on("session_start")`, `on("session_tree")` | +| `dynamic-tools.ts` | Register tools after startup and during commands | `registerTool`, `session_start`, `registerCommand` | +| `structured-output.ts` | Final structured-output tool with `terminate: true` | `registerTool`, terminating tool results | +| `truncated-tool.ts` | Output truncation example | `registerTool`, `truncateHead` | +| `tic-tac-toe.ts` | Tool that must run sequentially | `registerTool`, `executionMode: "sequential"`, `ui.custom` | +| `tool-override.ts` | Override built-in read tool | `registerTool` (same name as built-in) | +| `built-in-tool-renderer.ts` | Custom rendering for built-in tools | `create*ToolDefinition`, `renderCall`, `renderResult`, `renderShell` | +| `minimal-mode.ts` | Collapsed display for built-in tools | Built-in tool overrides, `renderResult` | +| **Commands** ||| +| `pirate.ts` | Modify system prompt per-turn | `registerCommand`, `before_agent_start` | +| `summarize.ts` | Conversation summary command | `registerCommand`, `ui.custom` | +| `handoff.ts` | Cross-provider model handoff | `registerCommand`, `ui.editor`, `ui.custom` | +| `qna.ts` | Q&A with custom UI | `registerCommand`, `ui.custom`, `setEditorText` | +| `send-user-message.ts` | Inject user messages | `registerCommand`, `sendUserMessage` | +| `reload-runtime.ts` | Reload command and LLM tool handoff | `registerCommand`, `ctx.reload()`, `on("agent_settled")`, `sendUserMessage` | +| `shutdown-command.ts` | Graceful shutdown command | `registerCommand`, `shutdown()` | +| **Events & Gates** ||| +| `permission-gate.ts` | Block dangerous commands | `on("tool_call")`, `ui.select` | +| `project-trust.ts` | Decide or defer project trust from a user/global or CLI extension | `on("project_trust")`, trust UI, required trust result | +| `protected-paths.ts` | Block writes to specific paths | `on("tool_call")` | +| `confirm-destructive.ts` | Confirm session changes | `on("session_before_switch")`, `on("session_before_fork")` | +| `dirty-repo-guard.ts` | Warn on dirty git repo | `on("session_before_*")`, `exec` | +| `input-transform.ts` | Transform user input | `on("input")` | +| `input-transform-streaming.ts` | Streaming-aware input transform | `on("input")`, `streamingBehavior` | +| `model-status.ts` | React to model changes | `on("model_select")`, `setStatus` | +| `provider-payload.ts` | Inspect payloads and provider response headers | `on("before_provider_request")`, `on("after_provider_response")` | +| `debug-provider.ts` | View raw provider stream events | `on("provider_stream_event")`, `appendEntry`, `registerEntryRenderer` | +| `system-prompt-header.ts` | Display system prompt info | `on("agent_start")`, `getSystemPrompt` | +| `claude-rules.ts` | Load rules from files | `on("session_start")`, `on("before_agent_start")` | +| `prompt-customizer.ts` | Add context-aware tool guidance using `systemPromptOptions` | `on("before_agent_start")`, `BuildSystemPromptOptions` | +| `file-trigger.ts` | File watcher triggers messages | `sendMessage` | +| **Compaction & Sessions** ||| +| `custom-compaction.ts` | Custom compaction summary | `on("session_before_compact")` | +| `trigger-compact.ts` | Trigger compaction manually | `compact()` | +| `git-checkpoint.ts` | Git stash on turns | `on("turn_start")`, `on("tool_result")`, `on("agent_settled")`, `on("session_before_fork")`, `exec` | +| `git-merge-and-resolve.ts` | Fetch, merge, and resolve conflicts | `on("agent_end")`, `exec`, `sendUserMessage` | +| `auto-commit-on-exit.ts` | Commit on shutdown | `on("session_shutdown")`, `exec` | +| **UI Components** ||| +| `status-line.ts` | Footer status indicator | `setStatus`, session events | +| `working-indicator.ts` | Customize the streaming working indicator | `setWorkingIndicator`, `registerCommand` | +| `github-issue-autocomplete.ts` | Add `#1234` issue completions on top of built-in autocomplete by preloading recent open issues from `gh issue list` | `addAutocompleteProvider`, `on("session_start")`, `exec` | +| `custom-footer.ts` | Replace footer entirely | `registerCommand`, `setFooter` | +| `custom-header.ts` | Replace startup header | `on("session_start")`, `setHeader` | +| `modal-editor.ts` | Vim-style modal editor | `setEditorComponent`, `CustomEditor` | +| `rainbow-editor.ts` | Custom editor styling | `setEditorComponent` | +| `widget-placement.ts` | Widget above/below editor | `setWidget` | +| `overlay-test.ts` | Overlay components | `ui.custom` with overlay options | +| `overlay-qa-tests.ts` | Comprehensive overlay tests | `ui.custom`, all overlay options | +| `notify.ts` | Native terminal notification when the agent is done | `on("agent_settled")` | +| `titlebar-spinner.ts` | Spinner in the terminal title while working | `setTitle` | +| `hidden-thinking-label.ts` | Label for hidden thinking blocks | `setHiddenThinkingLabel` | +| `border-status-editor.ts` | Status in the editor border | `setEditorComponent`, `CustomEditor` | +| `rpc-demo.ts` | Every RPC-supported UI method | `ui.*` in RPC mode | +| `timed-confirm.ts` | Dialogs with timeout | `ui.confirm` with timeout/signal | +| `mac-system-theme.ts` | Auto-switch theme | `setTheme`, `exec` | +| **Complex Extensions** ||| +| `plan-mode/` | Full plan mode implementation | All event types, `registerCommand`, `registerShortcut`, `registerFlag`, `setStatus`, `setWidget`, `sendMessage`, `setActiveTools` | +| `preset.ts` | Saveable presets (model, tools, thinking) | `registerCommand`, `registerShortcut`, `registerFlag`, `setModel`, `setActiveTools`, `setThinkingLevel`, `appendEntry` | +| `tools.ts` | Toggle tools on/off UI | `registerCommand`, `setActiveTools`, `SettingsList`, session events | +| **Remote & Sandbox** ||| +| `ssh.ts` | SSH remote execution | `registerFlag`, `on("user_bash")`, `on("before_agent_start")`, tool operations | +| `interactive-shell.ts` | Persistent shell session | `on("user_bash")` | +| `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | Tool operations, built-in tool overrides, `on("user_bash")` | +| `subagent/` | Spawn sub-agents | `registerTool`, `exec` | +| **Games** ||| +| `snake.ts` | Snake game | `registerCommand`, `ui.custom`, keyboard handling | +| `space-invaders.ts` | Space Invaders game | `registerCommand`, `ui.custom` | +| `doom-overlay/` | Doom in overlay | `ui.custom` with overlay | +| **Providers** ||| +| `custom-provider-anthropic/` | Custom Anthropic proxy | `registerProvider` | +| `custom-provider-gitlab-duo/` | GitLab Duo integration | `registerProvider` with OAuth | +| `jev-router.ts` | Virtual model that routes planning and implementation to different models | `registerVirtualModel` | +| **Messages & Communication** ||| +| `message-renderer.ts` | Custom message rendering | `registerMessageRenderer`, `sendMessage` | +| `entry-renderer.ts` | TUI-only custom entry rendering | `registerEntryRenderer`, `appendEntry` | +| `event-bus.ts` | Inter-extension events | `pi.events` | +| **Session Metadata** ||| +| `session-name.ts` | Name sessions for selector | `setSessionName`, `getSessionName` | +| `bookmark.ts` | Bookmark entries for /tree | `setLabel` | +| **Misc** ||| +| `commands.ts` | List available slash commands | `getCommands` | +| `dynamic-resources/` | Add skills, prompts, and themes at runtime | `on("resources_discover")` | +| `inline-bash.ts` | Expand inline `!{command}` in user prompts | `on("input")`, `exec` | +| `bash-spawn-hook.ts` | Adjust bash command, cwd, and env before execution | `createBashTool`, `spawnHook` | +| `with-deps/` | Extension with npm dependencies | Package structure with `package.json` | diff --git a/packages/coding-agent/docs/keybindings.md b/packages/coding-agent/docs/keybindings.md index 77838b629..34e4d68b4 100644 --- a/packages/coding-agent/docs/keybindings.md +++ b/packages/coding-agent/docs/keybindings.md @@ -98,7 +98,7 @@ The dedicated history actions browse prompt history regardless of cursor positio #### Fullscreen -In fullscreen mode, these actions control the transcript and take precedence over editor actions using the same key. +In fullscreen mode, these actions control the transcript and take precedence over editor actions using the same key. While an overlay or a scrollable list such as autocomplete, `/settings`, or the model selector has focus, the scrolling and prompt-jump actions go to it instead of the transcript, so `tui.select.pageUp` and `tui.select.pageDown` move the list selection. Transcript search actions keep working. | Keybinding id | Default | Description | |---|---|---| diff --git a/packages/coding-agent/docs/llama-cpp.md b/packages/coding-agent/docs/llama-cpp.md index d4c41ab2e..6684bb4df 100644 --- a/packages/coding-agent/docs/llama-cpp.md +++ b/packages/coding-agent/docs/llama-cpp.md @@ -1,31 +1,42 @@ # Local Models with llama.cpp -Pi supports the [llama.cpp](https://github.com/ggml-org/llama.cpp) router server. The router discovers multiple GGUF models and loads or unloads them on demand. +Pi supports the [llama.cpp](https://llama.app) router server. The router discovers multiple GGUF models and loads or unloads them on demand. -Use a current llama.cpp build with router support. Follow the [build instructions](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) or install a [prebuilt release](https://github.com/ggml-org/llama.cpp/releases) for your platform. +## Install llama.cpp + +Install llama.cpp with the [official installer](https://llama.app/docs/installation): + +```bash +curl -LsSf https://llama.app/install.sh | sh +``` + +Verify the installation: + +```bash +llama cli --version +``` + +See [llama.app](https://llama.app) for other installation methods, including package managers and building from source. ## Start the router -Start `llama-server` without `--model` or `-m`. Passing a model starts single-model mode instead of router mode. +Start `llama serve` without `--model`, `-m`, or `-hf`. Passing a model starts single-model mode instead of router mode. See the [llama.cpp server guide](https://llama.app/docs/serve#serving-multiple-models-router-mode) for router configuration. ```bash -llama-server \ - --models-dir ~/models \ +llama serve \ --no-models-autoload \ - --jinja \ --host 127.0.0.1 \ --port 8080 \ - -ngl 999 \ + -ngl all \ -c 32768 ``` Important options: -- `--models-dir ~/models` discovers local GGUF files. - `--no-models-autoload` keeps loading explicit through `/llama`. - `--jinja` enables compatible chat templates and tool calling. -- `-ngl 999` offloads as many layers as possible to the GPU. -- `-c 32768` sets the context window for each loaded model. Omit it to use the model's native context, which may require substantially more memory. +- `-ngl all` offloads all layers to the GPU. Omit it to use automatic GPU offloading. +- `-c 32768` sets the context size in tokens. Use `-c 0` for the model's full context window, which may require substantially more memory. Context is shared across parallel server slots. A single-file model can sit directly in the model directory. Put multimodal and multi-shard models in separate subdirectories: @@ -41,7 +52,7 @@ A single-file model can sit directly in the model directory. Put multimodal and └── large-model-Q4_K_M-00003-of-00003.gguf ``` -Restart the router after manually adding files. For per-model context sizes and other options, use [llama.cpp model presets](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#model-presets). For settings tuned to a specific NVIDIA GPU and model, see [Run Pi with NVIDIA RTX and llama.cpp](nvidia-rtx-llama-cpp.md). +Restart the router after manually adding files. The router also supports cached models and preset files via `--models-preset`, as described in the [server guide](https://llama.app/docs/serve#serving-multiple-models-router-mode). ## Configure Pi @@ -63,7 +74,7 @@ export LLAMA_API_KEY=optional-secret pi ``` -If the server uses an API key, start `llama-server` with the matching `--api-key` value. Keep `--host 127.0.0.1` for local-only access. +If the server uses an API key, start `llama serve` with the matching `--api-key` value. Keep `--host 127.0.0.1` for local-only access. ## Manage models diff --git a/packages/coding-agent/docs/mcp.md b/packages/coding-agent/docs/mcp.md index 5d5eb338e..5c6aa7d86 100644 --- a/packages/coding-agent/docs/mcp.md +++ b/packages/coding-agent/docs/mcp.md @@ -126,7 +126,7 @@ Remote servers that use OAuth, such as Sentry, need no credentials in `mcp.json` When the server rejects an unauthenticated connection, `/mcp` shows that it needs sign-in. Select "Sign in", run `/mcp login sentry`, or run `pi mcp login sentry`. Pi opens the authorization page and waits for approval. If the browser runs on another machine, such as over SSH, paste its redirected URL into the sign-in screen. A running session uses the new credentials on its next turn. -Pi registers itself with the authorization server, stores tokens in `~/.pi/agent/mcp-auth.json`, and refreshes access tokens when they expire or the server rejects them. If a server later requests additional scope, Pi asks for sign-in again. Signing out deletes the stored credentials. +Pi identifies itself to the authorization server with its Client ID Metadata Document or by registering a client, stores tokens in `~/.pi/agent/mcp-auth.json`, and refreshes access tokens when they expire or the server rejects them. If a server later requests additional scope, Pi asks for sign-in again. Signing out deletes the stored credentials. Credentials belong to a server name and URL. Servers with the same URL under different names, such as one per account, sign in separately; servers with the same name and URL in different `mcp.json` files share one sign-in. @@ -159,17 +159,19 @@ Pi registers as `pi`. Some servers only accept registrations from known clients. The name is only sent when Pi registers a client. To register again under a new name, sign out first. -Some authorization servers allow clients by their Client ID Metadata Document URL instead of registering them. Set `clientRegistration` to `cimd` to identify as Pi's document on pi.dev instead of registering: +Without `clientId`, Pi identifies itself with its Client ID Metadata Document on pi.dev when the authorization server advertises support for documents and public clients. Otherwise it registers a client dynamically. A client registered before stays in use while its redirect URI matches, until you sign out. With the document, the client ID is `https://pi.dev/oauth/client.json` with the redirect URI `http://127.0.0.1:/callback`. If the authorization server does not send the `iss` parameter in authorization responses (RFC 9207), Pi uses a document and redirect path specific to the MCP server instead: `https://pi.dev/oauth//client.json` with `http://127.0.0.1:/callback/`. + +Pi registers instead of using the document when `clientName` is set or `callbackUrl` is not `/callback` on `localhost` or `127.0.0.1`. Set `clientRegistration` to choose the method: `auto` (default) as described, `dcr` to always register, or `cimd` to always use the document. With `cimd`, sign-in fails if the authorization server does not support documents for public clients. Use `dcr` when the authorization page reports an invalid client: ```json { "mcpServers": { - "example": { "url": "https://mcp.example.com/mcp", "oauth": { "clientRegistration": "cimd" } } + "example": { "url": "https://mcp.example.com/mcp", "oauth": { "clientRegistration": "dcr" } } } } ``` -The client ID is `https://pi.dev/oauth/client.json` with the redirect URI `http://127.0.0.1:/callback`. If the authorization server does not send the `iss` parameter in authorization responses (RFC 9207), Pi uses a document and redirect path specific to the MCP server instead: `https://pi.dev/oauth//client.json` with `http://127.0.0.1:/callback/`. The authorization server must advertise Client ID Metadata Document support and public clients, or sign-in fails. `cimd` cannot be combined with `clientId` or `clientName`, and a `callbackUrl` must use `localhost` or `127.0.0.1` with the path `/callback`. +`pi mcp add --oauth-client-registration ` sets it when adding a server. `cimd` cannot be combined with `clientId` or `clientName`, and a `callbackUrl` must use `localhost` or `127.0.0.1` with the path `/callback`. Pi finds the authorization server through the server's protected resource metadata (RFC 9728) and checks that the authorization server's metadata names the expected issuer (RFC 8414). Some servers advertise the wrong authorization server or none, so sign-in opens a page that does not exist. Set `authServerMetadataUrl` to the metadata document of the right authorization server: diff --git a/packages/coding-agent/docs/packages.md b/packages/coding-agent/docs/packages.md index a4f609db8..6aa1d8afc 100644 --- a/packages/coding-agent/docs/packages.md +++ b/packages/coding-agent/docs/packages.md @@ -28,6 +28,8 @@ Use `--extension` or `-e` to try a package for one invocation without adding it pi -e npm:@example/pi-tools ``` +Pi caches `-e` packages under `~/.pi/agent/tmp/extensions/` and checks them on every start and `/reload`. An npm package is reinstalled when the registry has a newer version that matches the specification. A version pinned to an exact number, such as `@1.2.3`, is not checked. A git package is fetched and reset to the remote branch or ref. A ref that is a commit hash already checked out is not fetched. If the check fails or offline mode is on, Pi loads the cached copy. + ## Choose a source | Source | Example | Behavior | @@ -75,6 +77,26 @@ Paths are relative to the package root. Arrays accept glob patterns and exclusio The `pi-package` keyword makes an npm package eligible for discovery in the [Pi package gallery](https://pi.dev/packages). Optional `pi.image` and `pi.video` fields add gallery previews. +## Show a changelog + +After a package update, Pi can show what changed. On the next fresh launch, Pi reads the package changelog and shows entries newer than the last seen version under "Extension Updates", the same way it shows its own changelog. The first time Pi sees a package, it records the version without showing anything. + +This applies to packages that load at least one extension. Packages that only provide skills, prompt templates, or themes do not show a changelog. + +By default Pi reads `CHANGELOG.md` from the package root. Point `pi.changelogPath` at another file inside the package, or set it to `""` to disable the changelog: + +```json +{ + "name": "my-pi-package", + "version": "1.2.0", + "pi": { + "changelogPath": "./docs/CHANGELOG.md" + } +} +``` + +Use version headings in the same format as Pi's changelog, such as `## [1.2.0]` or `## 1.2.0 - 2026-01-01`. Entries are matched against the `version` field in `package.json`, so a git package only shows a changelog when that field increases. The `collapseChangelog` setting condenses the section to one line per package, such as `Updated my-pi-package to v1.2.0.` Unlike Pi's own changelog, `/changelog` does not show package changelogs, so read the package's changelog file or repository for details. + ## Declare dependencies Put runtime packages imported by extensions in `dependencies`. Pi installs package dependencies when it installs an npm or git source. diff --git a/packages/coding-agent/docs/quickstart.md b/packages/coding-agent/docs/quickstart.md index 89b56f07e..44054e949 100644 --- a/packages/coding-agent/docs/quickstart.md +++ b/packages/coding-agent/docs/quickstart.md @@ -6,13 +6,17 @@ For native Windows setup, read [Windows Setup](windows.md). For Android, read [T ## 1. Install Pi -On macOS or Linux, you can use the installer: +Install Pi with the installer on macOS, Linux, or Windows: -```bash +```bash tab="macOS / Linux" curl -fsSL https://pi.dev/install.sh | sh ``` -The installer pins all dependencies and updates Pi with `pi update`. Alternatively, install Pi from npm, which does not pin transitive dependencies. This requires Node.js 22.19 or newer: +```powershell tab="Windows" +powershell -c "irm https://pi.dev/install.ps1 | iex" +``` + +Both installers pin all dependencies and update Pi with `pi update`. Alternatively, install Pi from npm, which does not pin transitive dependencies. This requires Node.js 22.19 or newer: ```bash npm install -g --ignore-scripts @earendil-works/pi-coding-agent diff --git a/packages/coding-agent/docs/sessions.md b/packages/coding-agent/docs/sessions.md index 1a8be36c5..aeb59a308 100644 --- a/packages/coding-agent/docs/sessions.md +++ b/packages/coding-agent/docs/sessions.md @@ -11,7 +11,17 @@ pi --continue pi --resume ``` -`--continue` opens the most recent session for the current working directory. `--resume` opens the session picker. In interactive mode, `/resume` opens the same picker and `/new` starts a new session. +`--continue` opens the most recent session for the current directory. `--resume` opens the session picker. In interactive mode, `/resume` opens the same picker and `/new` starts a new session. + +Each session records the working directory it was created in and never changes it. Pi runs a session in its working directory, so tools, `@path` references, project configuration, and context files resolve from there. + +| Entry point | Working directory | +|---|---| +| `--continue` | The current directory | +| `--resume`, `/resume`, `--session`, `/import` | The session's working directory, which can differ from the current directory | +| `--fork`, or a `--session` ID from another directory that you choose to fork | The current directory, in a new session | + +If a session's working directory no longer exists, interactive mode offers to run the session in the current directory instead. Pi does not store that choice, so it asks again the next time you open the session. Other modes exit with an error. Use `/name` or `--name` to assign a recognizable session name. Run `/session` to verify the current session file, ID, message count, token usage, and cost. @@ -51,8 +61,6 @@ By default, Pi stores sessions under `~/.pi/agent/sessions/`, grouped by working Use `--no-session` for an ephemeral run. An ephemeral session cannot be resumed after Pi exits. -Use `--session` when you already know the session path or ID. Use `--fork` to create a new session from an existing session before interactive mode starts. - ## Export or share a session Use `/export` to write the current session as HTML or JSONL. Use `/share` to upload it and get a viewer link. By default, Pi uses a Radius artifact when Radius authentication is configured; otherwise, it uses a private GitHub gist. Set [`sessionShareDestination`](settings.md#sessions-and-context) to `gist` or `radius` to force one destination. diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 150eaf4d7..3de528c44 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -142,6 +142,7 @@ Keep `retry.provider.maxRetries` at `0` unless provider-level retries are requir | `shellPath` | string | Platform default | Custom shell executable path. Supports a leading `~`. | | `shellCommandPrefix` | string | None | Prefix prepended to every shell command. | | `npmCommand` | `string[]` | `npm` | Command and arguments used for npm package lookup and installation. | +| `selfUpdateCommand` | `string[]` | None | Command and arguments `pi update` runs to update Pi itself, with `{packageName}`, `{version}`, and `{installSpec}` placeholders. `PI_SELF_UPDATE_COMMAND` takes precedence. **Can only be set in agent-directory settings.** See [Custom update command](cli.md#custom-update-command). | See [Shell aliases](shell-aliases.md) for shell setup and [Pi Packages](packages.md) for package-manager behavior. @@ -166,7 +167,7 @@ The built-in extensions are named `builtin:mcp`, `builtin:llama.cpp`, `builtin:c | Setting | Type | Default | Description | |---|---|---|---| -| `collapseChangelog` | boolean | `false` | Show a condensed changelog after an update. | +| `collapseChangelog` | boolean | `false` | Show a condensed changelog after an update. Also condenses extension package changelogs to one line per package. | | `enableInstallTelemetry` | boolean | `true` | Enable anonymous install/update reporting and selected provider attribution headers. Does not control update checks. | | `enableAnalytics` | boolean | `false` | Opt in to analytics data sharing. Currently used only by the experimental first-run setup. | | `warnings.anthropicExtraUsage` | boolean | `true` | Warn when Anthropic subscription authentication may use paid extra usage. | diff --git a/packages/coding-agent/docs/slash-commands.md b/packages/coding-agent/docs/slash-commands.md index f2b5f3bac..cb534a363 100644 --- a/packages/coding-agent/docs/slash-commands.md +++ b/packages/coding-agent/docs/slash-commands.md @@ -21,14 +21,14 @@ Extensions, prompt templates, and skills can add commands. The command menu in P | Command | Description | |---|---| | `/new` | Start a new session | -| `/resume` | Switch to another saved session | +| `/resume` | Resume another session | | `/name [name]` | Set the session display name, or show the current name when omitted | | `/session` | Show current session information and statistics | | `/tree` | Navigate the session tree | | `/fork` | Create a new session from an earlier user message | | `/clone` | Duplicate the current session at its current position | | `/compact [instructions]` | Compact the current context, optionally with custom instructions | -| `/import ` | Import and resume a JSONL session | +| `/import ` | Import a JSONL session, preserving its ID and working directory | ## Export and share diff --git a/packages/coding-agent/docs/terminal-setup.md b/packages/coding-agent/docs/terminal-setup.md index a3c67af07..ab8e1dc0e 100644 --- a/packages/coding-agent/docs/terminal-setup.md +++ b/packages/coding-agent/docs/terminal-setup.md @@ -40,7 +40,7 @@ This is an iTerm2-wide setting and can also change native trackpad scrolling. Th ## Apple Terminal -Pi enables enhanced key reporting when available. If Terminal.app still sends plain Return for `Shift+Enter`, Pi uses a local macOS modifier fallback and treats it as `Shift+Enter`. +Pi enables enhanced key reporting when available. If Terminal.app still sends plain Return or `ESC` + Return for `Shift+Enter`, Pi uses a local macOS modifier fallback and treats it as `Shift+Enter`. `ESC` + Return without Shift held remains `Alt+Enter`. The fallback works only when Pi runs on the same Mac as Terminal.app. It cannot inspect the local modifier state when Pi runs on another machine over SSH. @@ -170,6 +170,8 @@ Windows Terminal uses Pi's Windows and WSL shortcut defaults. See [Keybindings]( ### Forward Shift+Enter +When Pi runs natively on Windows, it checks the local Shift key state and treats plain Return or `ESC` + Return as `Shift+Enter` while Shift is held. This fallback does not work in WSL or over SSH. If `Shift+Enter` still submits, forward it explicitly. + Open Windows Terminal's `settings.json` with `Ctrl+Shift+,` or **Settings > Open JSON file**. Add this object to its `actions` array: ```json diff --git a/packages/coding-agent/docs/tmux.md b/packages/coding-agent/docs/tmux.md index 091d61647..1a640d174 100644 --- a/packages/coding-agent/docs/tmux.md +++ b/packages/coding-agent/docs/tmux.md @@ -53,3 +53,9 @@ set -g extended-keys on Pi supports the xterm `modifyOtherKeys` format used by these versions. Restart tmux and repeat the verification steps. For older versions, upgrade tmux or use Pi outside tmux rather than relying on modified Enter shortcuts. + +## Known issues + +| Issue | Affects | Remedy | +|---|---|---| +| Text such as `c0c/0c0c/0c0c` appears in the editor at startup ([tmux#4749](https://github.com/tmux/tmux/issues/4749)) | tmux 3.6 | Upgrade tmux. Changing Pi's theme does not help. | diff --git a/packages/coding-agent/docs/tui.md b/packages/coding-agent/docs/tui.md index 704a78633..c3fb30939 100644 --- a/packages/coding-agent/docs/tui.md +++ b/packages/coding-agent/docs/tui.md @@ -135,6 +135,8 @@ export default function (pi: ExtensionAPI) { This component does not display a text cursor, so it does not need `Focusable`. A component containing `Input` or `Editor` must implement `Focusable` and propagate its `focused` value to that child. +Custom components do not report pi as waiting for the user, because `ctx.ui.custom()` also hosts loaders and other non-blocking UI. When an inline component waits for an answer, pass `blocked: { kind: "question", message: "Pick a color" }` so terminals that support [program status](terminal-setup.md#program-status) show pi as blocked while the component is visible. `kind` is `"question"`, `"permission"`, or `"auth"`. Overlays ignore this option. + Pass `overlay: true` to draw above existing content. Overlay options control size, anchors, offsets, margins, and responsive visibility. An overlay handle can change focus or temporarily hide and show the overlay with `setHidden()` while the interaction remains active. Focused overlays retain input ownership across ordinary renders. If another component should receive input while an overlay remains visible, explicitly release or redirect focus through the handle. diff --git a/packages/coding-agent/docs/windows.md b/packages/coding-agent/docs/windows.md index 26f2f961c..6e68979b8 100644 --- a/packages/coding-agent/docs/windows.md +++ b/packages/coding-agent/docs/windows.md @@ -76,6 +76,33 @@ JSON uses backslashes for escape sequences. When you write a Windows path with b See [Configure shell commands](shell-aliases.md) for command prefixes, aliases, and the complete shell-resolution behavior. +## Choose a terminal + +Windows terminals run Pi through a console host (conhost). Some bundle an up-to-date copy; others use the older one built into Windows, which has bugs that affect Pi. + +We recommend [Windows Terminal](https://aka.ms/terminal). It always ships an up-to-date conhost, and Pi is best tested with it. + +Last updated: October 2026. + +| Terminal | Conhost | Status | +|---|---|---| +| [Windows Terminal](https://aka.ms/terminal) | Bundled, current | Recommended | +| [VS Code](https://code.visualstudio.com), [Tabby](https://tabby.sh), [WezTerm](https://wezterm.org) nightly | Bundled, current | Works | +| [Alacritty](https://alacritty.org) | Built-in, unless `conpty.dll` and `OpenConsole.exe` from Windows Terminal are next to `alacritty.exe` | Works only with those files | +| WezTerm 20240203 and older | Bundled, old | [Known issues](#known-issues) | +| Classic console (`cmd.exe` or PowerShell without Windows Terminal), mintty (Git Bash), ConEmu, Cmder, [Hyper](https://hyper.is), Windows OpenSSH | Built-in | [Known issues](#known-issues) | + ## Configure Windows Terminal Windows Terminal reserves or rewrites some modified keys. See [Windows Terminal](terminal-setup.md#windows-terminal) to configure `Shift+Enter` and `Alt+Enter`, and [Keybindings](keybindings.md) for Pi's Windows and WSL shortcut defaults. + +## Known issues + +These issues come from conhost. Microsoft has fixed them in its source, but has not said when the built-in conhost will get the fixes. + +| Issue | Affects | Remedy | +|---|---|---| +| Switching from fullscreen to regular mode, or `Ctrl+G` from fullscreen, crashes the session ([#9186](https://github.com/earendil-works/pi/issues/9186), [microsoft/terminal#17515](https://github.com/microsoft/terminal/issues/17515)) | Built-in conhost | Use a terminal with a current conhost, or start Pi with `--tui-mode regular` | +| Text such as `;0;rgb:0000/0000/0000` appears in the prompt, and BEL opens the external editor ([#10256](https://github.com/earendil-works/pi/issues/10256)) | Built-in conhost behind another terminal: mintty, Windows OpenSSH | Use a terminal with a current conhost | +| Fragments such as `79` or `55` appear in the prompt ([#10717](https://github.com/earendil-works/pi/issues/10717)) | WezTerm 20240203 | Use WezTerm nightly | +| In regular mode, each keystroke redraws the input line on a new row ([#6300](https://github.com/earendil-works/pi/issues/6300)) | Built-in conhost, classic console | Use fullscreen mode (the default), or a terminal with a current conhost | diff --git a/packages/coding-agent/examples/extensions/README.md b/packages/coding-agent/examples/extensions/README.md index 7667ffe03..de24f7af4 100644 --- a/packages/coding-agent/examples/extensions/README.md +++ b/packages/coding-agent/examples/extensions/README.md @@ -23,9 +23,10 @@ cp permission-gate.ts ~/.pi/agent/extensions/ | `protected-paths.ts` | Blocks writes to protected paths (.env, .git/, node_modules/) | | `confirm-destructive.ts` | Confirms before destructive session actions (clear, switch, fork) | | `dirty-repo-guard.ts` | Prevents session changes with uncommitted git changes | -| `sandbox/` | OS-level sandboxing using `@anthropic-ai/sandbox-runtime` with per-project config | | `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | +For isolation setup and limitations, see [Run Pi in an isolated environment](../../docs/containerization.md). + ### Custom Tools | Extension | Description | diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json index b7d2ebab0..c6ed32b21 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "^0.52.0" } diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json index b22368863..b4900f974 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-anthropic", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json index dbdc63c72..54e070434 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-gitlab-duo", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/debug-provider.ts b/packages/coding-agent/examples/extensions/debug-provider.ts index 14aae7027..2f13e0d24 100644 --- a/packages/coding-agent/examples/extensions/debug-provider.ts +++ b/packages/coding-agent/examples/extensions/debug-provider.ts @@ -6,7 +6,7 @@ * as expandable custom entries. */ -import { type ExtensionAPI, keyHint } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Box, Text } from "@earendil-works/pi-tui"; const ENTRY_TYPE = "debug-provider-events"; @@ -24,7 +24,7 @@ export default function (pi: ExtensionAPI) { let activeEvents: unknown[] | undefined; let completedEntry: ProviderDebugEntry | undefined; - pi.registerEntryRenderer(ENTRY_TYPE, (entry, { expanded }, theme) => { + pi.registerEntryRenderer(ENTRY_TYPE, (entry, { expanded, keyHint }, theme) => { const data = entry.data; if (!data) return new Text(theme.fg("warning", "[provider debug] Missing event data"), 0, 0); diff --git a/packages/coding-agent/examples/extensions/gondolin/package-lock.json b/packages/coding-agent/examples/extensions/gondolin/package-lock.json index 99a06ff69..0887734dd 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package-lock.json +++ b/packages/coding-agent/examples/extensions/gondolin/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } diff --git a/packages/coding-agent/examples/extensions/gondolin/package.json b/packages/coding-agent/examples/extensions/gondolin/package.json index d19d7190f..762d1d230 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package.json +++ b/packages/coding-agent/examples/extensions/gondolin/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-gondolin", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/preset.ts b/packages/coding-agent/examples/extensions/preset.ts index 39f55642c..3b5f98764 100644 --- a/packages/coding-agent/examples/extensions/preset.ts +++ b/packages/coding-agent/examples/extensions/preset.ts @@ -252,6 +252,8 @@ export default function presetExtension(pi: ExtensionAPI) { container.addChild(new DynamicBorder((str) => theme.fg("accent", str))); return { + // Let PageUp/PageDown page the list instead of scrolling the fullscreen transcript. + capturesViewportInput: selectList.capturesViewportInput, render(width: number) { return container.render(width); }, diff --git a/packages/coding-agent/examples/extensions/reload-runtime.ts b/packages/coding-agent/examples/extensions/reload-runtime.ts index d0e9abd47..938e31dfe 100644 --- a/packages/coding-agent/examples/extensions/reload-runtime.ts +++ b/packages/coding-agent/examples/extensions/reload-runtime.ts @@ -2,7 +2,7 @@ * Reload Runtime Extension * * Demonstrates ctx.reload() from ExtensionCommandContext and an LLM-callable - * tool that queues a follow-up command to trigger reload. + * tool that requests a reload once the agent has settled. */ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; @@ -19,19 +19,31 @@ export default function (pi: ExtensionAPI) { }, }); - // LLM-callable tool. Tools get ExtensionContext, so they cannot call ctx.reload() directly. - // Instead, queue a follow-up user command that executes the command above. + // LLM-callable tool. Tools do not receive ExtensionCommandContext, so they cannot call + // ctx.reload(), and Pi cannot reload while the agent is running. The tool records the + // request and ends the run; the agent_settled handler then runs the command above. + let reloadRequested = false; + pi.registerTool({ name: "reload_runtime", label: "Reload Runtime", description: "Reload extensions, skills, prompts, themes, and context files", parameters: Type.Object({}), async execute() { - pi.sendUserMessage("/reload-runtime", { deliverAs: "followUp" }); + reloadRequested = true; return { - content: [{ type: "text", text: "Queued /reload-runtime as a follow-up command." }], + content: [{ type: "text", text: "Reload requested. Pi reloads after this response." }], details: {}, + terminate: true, }; }, }); + + pi.on("agent_settled", () => { + if (!reloadRequested) return; + reloadRequested = false; + // Sent from agent_settled, the command runs after every settled handler finishes, while + // the agent is idle. expandPromptTemplates dispatches it as a command, not as model input. + pi.sendUserMessage("/reload-runtime", { expandPromptTemplates: true }); + }); } diff --git a/packages/coding-agent/examples/extensions/sandbox/.gitignore b/packages/coding-agent/examples/extensions/sandbox/.gitignore deleted file mode 100644 index 3c3629e64..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/packages/coding-agent/examples/extensions/sandbox/index.ts b/packages/coding-agent/examples/extensions/sandbox/index.ts deleted file mode 100644 index b54d75d1c..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/index.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Sandbox Extension - OS-level sandboxing for bash commands - * - * Uses @anthropic-ai/sandbox-runtime to enforce filesystem and network - * restrictions on bash commands at the OS level (sandbox-exec on macOS, - * bubblewrap on Linux). - * - * Note: this example intentionally overrides the built-in `bash` tool to show - * how built-in tools can be replaced. Alternatively, you could sandbox `bash` - * via `tool_call` input mutation without replacing the tool. - * - * Config files (merged, project takes precedence): - * - ~/.pi/agent/extensions/sandbox.json (global) - * - /.pi/sandbox.json (project-local) - * - * Example .pi/sandbox.json: - * ```json - * { - * "enabled": true, - * "network": { - * "allowedDomains": ["github.com", "*.github.com"], - * "deniedDomains": [] - * }, - * "filesystem": { - * "denyRead": ["~/.ssh", "~/.aws"], - * "allowWrite": [".", "/tmp"], - * "denyWrite": [".env"] - * } - * } - * ``` - * - * Usage: - * - `pi -e ./sandbox` - sandbox enabled with default/config settings - * - `pi -e ./sandbox --no-sandbox` - disable sandboxing - * - `/sandbox` - show current sandbox configuration - * - * Setup: - * 1. Copy sandbox/ directory to ~/.pi/agent/extensions/ - * 2. Run `npm install` in ~/.pi/agent/extensions/sandbox/ - * - * Linux also requires: bubblewrap, socat, ripgrep - */ - -import { spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { SandboxManager, type SandboxRuntimeConfig } from "@anthropic-ai/sandbox-runtime"; -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { type BashOperations, CONFIG_DIR_NAME, createBashTool, getAgentDir } from "@earendil-works/pi-coding-agent"; - -interface SandboxConfig extends SandboxRuntimeConfig { - enabled?: boolean; -} - -const DEFAULT_CONFIG: SandboxConfig = { - enabled: true, - network: { - allowedDomains: [ - "npmjs.org", - "*.npmjs.org", - "registry.npmjs.org", - "registry.yarnpkg.com", - "pypi.org", - "*.pypi.org", - "github.com", - "*.github.com", - "api.github.com", - "raw.githubusercontent.com", - ], - deniedDomains: [], - }, - filesystem: { - denyRead: ["~/.ssh", "~/.aws", "~/.gnupg"], - allowWrite: [".", "/tmp"], - denyWrite: [".env", ".env.*", "*.pem", "*.key"], - }, -}; - -function loadConfig(cwd: string): SandboxConfig { - const projectConfigPath = join(cwd, CONFIG_DIR_NAME, "sandbox.json"); - const globalConfigPath = join(getAgentDir(), "extensions", "sandbox.json"); - - let globalConfig: Partial = {}; - let projectConfig: Partial = {}; - - if (existsSync(globalConfigPath)) { - try { - globalConfig = JSON.parse(readFileSync(globalConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${globalConfigPath}: ${e}`); - } - } - - if (existsSync(projectConfigPath)) { - try { - projectConfig = JSON.parse(readFileSync(projectConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${projectConfigPath}: ${e}`); - } - } - - return deepMerge(deepMerge(DEFAULT_CONFIG, globalConfig), projectConfig); -} - -function deepMerge(base: SandboxConfig, overrides: Partial): SandboxConfig { - const result: SandboxConfig = { ...base }; - - if (overrides.enabled !== undefined) result.enabled = overrides.enabled; - if (overrides.network) { - result.network = { ...base.network, ...overrides.network }; - } - if (overrides.filesystem) { - result.filesystem = { ...base.filesystem, ...overrides.filesystem }; - } - - const extOverrides = overrides as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - const extResult = result as { ignoreViolations?: Record; enableWeakerNestedSandbox?: boolean }; - - if (extOverrides.ignoreViolations) { - extResult.ignoreViolations = extOverrides.ignoreViolations; - } - if (extOverrides.enableWeakerNestedSandbox !== undefined) { - extResult.enableWeakerNestedSandbox = extOverrides.enableWeakerNestedSandbox; - } - - return result; -} - -function createSandboxedBashOps(): BashOperations { - return { - async exec(command, cwd, { onData, signal, timeout }) { - if (!existsSync(cwd)) { - throw new Error(`Working directory does not exist: ${cwd}`); - } - - const wrappedCommand = await SandboxManager.wrapWithSandbox(command); - - return new Promise((resolve, reject) => { - const child = spawn("bash", ["-c", wrappedCommand], { - cwd, - detached: true, - stdio: ["ignore", "pipe", "pipe"], - }); - - let timedOut = false; - let timeoutHandle: NodeJS.Timeout | undefined; - - if (timeout !== undefined && timeout > 0) { - timeoutHandle = setTimeout(() => { - timedOut = true; - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }, timeout * 1000); - } - - child.stdout?.on("data", onData); - child.stderr?.on("data", onData); - - child.on("error", (err) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - reject(err); - }); - - const onAbort = () => { - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }; - - signal?.addEventListener("abort", onAbort, { once: true }); - - child.on("close", (code) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - signal?.removeEventListener("abort", onAbort); - - if (signal?.aborted) { - reject(new Error("aborted")); - } else if (timedOut) { - reject(new Error(`timeout:${timeout}`)); - } else { - resolve({ exitCode: code }); - } - }); - }); - }, - }; -} - -export default function (pi: ExtensionAPI) { - pi.registerFlag("no-sandbox", { - description: "Disable OS-level sandboxing for bash commands", - type: "boolean", - default: false, - }); - - const localCwd = process.cwd(); - const localBash = createBashTool(localCwd); - - let sandboxEnabled = false; - let sandboxInitialized = false; - - pi.registerTool({ - ...localBash, - label: "bash (sandboxed)", - async execute(id, params, signal, onUpdate, _ctx) { - if (!sandboxEnabled || !sandboxInitialized) { - return localBash.execute(id, params, signal, onUpdate); - } - - const sandboxedBash = createBashTool(localCwd, { - operations: createSandboxedBashOps(), - }); - return sandboxedBash.execute(id, params, signal, onUpdate); - }, - }); - - pi.on("user_bash", () => { - if (!sandboxEnabled || !sandboxInitialized) return; - return { operations: createSandboxedBashOps() }; - }); - - pi.on("session_start", async (_event, ctx) => { - const noSandbox = pi.getFlag("no-sandbox") as boolean; - - if (noSandbox) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via --no-sandbox", "warning"); - return; - } - - const config = loadConfig(ctx.cwd); - - if (!config.enabled) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via config", "info"); - return; - } - - const platform = process.platform; - if (platform !== "darwin" && platform !== "linux") { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox not supported on ${platform}`, "warning"); - return; - } - - try { - const configExt = config as unknown as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - - await SandboxManager.initialize({ - network: config.network, - filesystem: config.filesystem, - ignoreViolations: configExt.ignoreViolations, - enableWeakerNestedSandbox: configExt.enableWeakerNestedSandbox, - }); - - sandboxEnabled = true; - sandboxInitialized = true; - - const networkCount = config.network?.allowedDomains?.length ?? 0; - const writeCount = config.filesystem?.allowWrite?.length ?? 0; - ctx.ui.setStatus( - "sandbox", - ctx.ui.theme.fg("accent", `🔒 Sandbox: ${networkCount} domains, ${writeCount} write paths`), - ); - ctx.ui.notify("Sandbox initialized", "info"); - } catch (err) { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox initialization failed: ${err instanceof Error ? err.message : err}`, "error"); - } - }); - - pi.on("session_shutdown", async () => { - if (sandboxInitialized) { - try { - await SandboxManager.reset(); - } catch { - // Ignore cleanup errors - } - } - }); - - pi.registerCommand("sandbox", { - description: "Show sandbox configuration", - handler: async (_args, ctx) => { - if (!sandboxEnabled) { - ctx.ui.notify("Sandbox is disabled", "info"); - return; - } - - const config = loadConfig(ctx.cwd); - const lines = [ - "Sandbox Configuration:", - "", - "Network:", - ` Allowed: ${config.network?.allowedDomains?.join(", ") || "(none)"}`, - ` Denied: ${config.network?.deniedDomains?.join(", ") || "(none)"}`, - "", - "Filesystem:", - ` Deny Read: ${config.filesystem?.denyRead?.join(", ") || "(none)"}`, - ` Allow Write: ${config.filesystem?.allowWrite?.join(", ") || "(none)"}`, - ` Deny Write: ${config.filesystem?.denyWrite?.join(", ") || "(none)"}`, - ]; - ctx.ui.notify(lines.join("\n"), "info"); - }, - }); -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package-lock.json b/packages/coding-agent/examples/extensions/sandbox/package-lock.json deleted file mode 100644 index 90ef1c2f5..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package-lock.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "version": "1.4.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "^0.0.26" - } - }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, - "node_modules/@types/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-RDvF6wTulMPjrNdCoYRC8gNR880JNGT8uB+REUpC2Ns4pRqQJhGz90wh7rgdXDPpCczF3VGktDuFGVnz8zP7HA==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, - "node_modules/shell-quote": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", - "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package.json b/packages/coding-agent/examples/extensions/sandbox/package.json deleted file mode 100644 index 8059d577d..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "private": true, - "version": "1.4.0", - "type": "module", - "scripts": { - "clean": "echo 'nothing to clean'", - "build": "echo 'nothing to build'", - "check": "echo 'nothing to check'" - }, - "pi": { - "extensions": [ - "./index.ts" - ] - }, - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } -} diff --git a/packages/coding-agent/examples/extensions/tools.ts b/packages/coding-agent/examples/extensions/tools.ts index 8db496a51..7f65b1796 100644 --- a/packages/coding-agent/examples/extensions/tools.ts +++ b/packages/coding-agent/examples/extensions/tools.ts @@ -117,6 +117,8 @@ export default function toolsExtension(pi: ExtensionAPI) { container.addChild(settingsList); const component = { + // Let PageUp/PageDown page the list instead of scrolling the fullscreen transcript. + capturesViewportInput: settingsList.capturesViewportInput, render(width: number) { return container.render(width); }, diff --git a/packages/coding-agent/examples/extensions/with-deps/package-lock.json b/packages/coding-agent/examples/extensions/with-deps/package-lock.json index 90c4e6339..2d84832c4 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package-lock.json +++ b/packages/coding-agent/examples/extensions/with-deps/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "^2.1.3" }, diff --git a/packages/coding-agent/examples/extensions/with-deps/package.json b/packages/coding-agent/examples/extensions/with-deps/package.json index f22d436c7..c71ad77f2 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package.json +++ b/packages/coding-agent/examples/extensions/with-deps/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-with-deps", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/tsconfig.json b/packages/coding-agent/examples/tsconfig.json new file mode 100644 index 000000000..03e1a36e1 --- /dev/null +++ b/packages/coding-agent/examples/tsconfig.json @@ -0,0 +1,180 @@ +// Type-checks the example extensions against the modules extensions see at run time (core/extensions/virtual-modules.ts): +// "@earendil-works/pi-ai" is the extension view, which adds TypeBox's Type, Static and TSchema to the compat entry (the +// view imports "@earendil-works/pi-ai/compat", so that path stays the package's own). +// The other paths repeat the root tsconfig.json; keep them in sync. +// Also the test that imports the subagent example: it type-checks only against the extension view. +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "paths": { + "@earendil-works/chord": [ + "../../../packages/chord/src/index.ts" + ], + "@earendil-works/chord/context": [ + "../../../packages/chord/src/context/index.ts" + ], + "@earendil-works/chord/delta": [ + "../../../packages/chord/src/delta/index.ts" + ], + "@earendil-works/chord/next": [ + "../../../packages/chord/src/next/index.ts" + ], + "@earendil-works/chord/next/plugins": [ + "../../../packages/chord/src/next-node/plugins/index.ts" + ], + "@earendil-works/chord/next/plugins/runtime": [ + "../../../packages/chord/src/next-node/plugins/runtime.ts" + ], + "@test/contracts/*": [ + "../../../packages/chord/test/next-node/fixtures/plugins/contracts/*.ts" + ], + "@example/contracts/*": [ + "../../../packages/chord/test/next/examples/35-plugins/plugins/contracts/*.ts" + ], + "@example/weather/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/weather/*.ts" + ], + "@example/prices/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/prices/*.ts" + ], + "@example/shapes/*": [ + "../../../packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/*.ts" + ], + "@example/shared/*": [ + "../../../packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/*.ts" + ], + "@earendil-works/chord/next/http": [ + "../../../packages/chord/src/next-node/http/index.ts" + ], + "@earendil-works/chord/next/workers": [ + "../../../packages/chord/src/next-node/workers/index.ts" + ], + "@earendil-works/chord/node": [ + "../../../packages/chord/src/node.ts" + ], + "@earendil-works/chord/output": [ + "../../../packages/chord/src/output.ts" + ], + "@earendil-works/chord/telemetry-sqlite": [ + "../../../packages/chord/src/telemetry-sqlite/index.ts" + ], + "@earendil-works/chord/sqlite": [ + "../../../packages/chord/src/sqlite/index.ts" + ], + "@earendil-works/chord/sqlite/node": [ + "../../../packages/chord/src/sqlite/node.ts" + ], + "@earendil-works/chord/sqlite/cloudflare": [ + "../../../packages/chord/src/sqlite/cloudflare.ts" + ], + "@earendil-works/pi-telemetry": [ + "../../../packages/telemetry/src/index.ts" + ], + "@earendil-works/pi-telemetry/testing": [ + "../../../packages/telemetry/src/testing/index.ts" + ], + "@earendil-works/pi-telemetry/render": [ + "../../../packages/telemetry/src/render.ts" + ], + "@earendil-works/pi-codemode": [ + "../../../packages/codemode/src/index.ts" + ], + "@earendil-works/pi-codemode/cloudflare": [ + "../../../packages/codemode/src/cloudflare.ts" + ], + "@earendil-works/pi-codemode/declarations": [ + "../../../packages/codemode/src/declarations.ts" + ], + "@earendil-works/pi-codemode/portable": [ + "../../../packages/codemode/src/portable.ts" + ], + "@earendil-works/pi-codemode/source": [ + "../../../packages/codemode/src/source.ts" + ], + "@earendil-works/pi-codemode/worker": [ + "../../../packages/codemode/src/runtime/worker.ts" + ], + "@earendil-works/pi-schema": [ + "../../../packages/schema/src/index.ts" + ], + "@earendil-works/pi-schema/formats": [ + "../../../packages/schema/src/formats/index.ts" + ], + "@earendil-works/pi-mcp": [ + "../../../packages/mcp/src/index.ts" + ], + "@earendil-works/pi-mcp/oauth": [ + "../../../packages/mcp/src/oauth/index.ts" + ], + "@earendil-works/pi-mcp/testing": [ + "../../../packages/mcp/src/testing/index.ts" + ], + "@earendil-works/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ], + "@earendil-works/pi-ai/oauth": [ + "../../../packages/ai/src/oauth.ts" + ], + "@earendil-works/pi-ai/*": [ + "../../../packages/ai/src/*.ts", + "../../../packages/ai/src/providers/*.ts" + ], + "@earendil-works/pi-ai/dist/*": [ + "../../../packages/ai/src/*" + ], + "@earendil-works/pi-durable": [ + "../../../packages/durable/src/index.ts" + ], + "@earendil-works/pi-durable/testing": [ + "../../../packages/durable/src/testing/index.ts" + ], + "@earendil-works/pi-durable/*": [ + "../../../packages/durable/src/*.ts", + "../../../packages/durable/src/*/index.ts", + "../../../packages/durable/src/*" + ], + "@earendil-works/pi-env": [ + "../../../packages/env/src/index.ts" + ], + "@earendil-works/pi-agent-core": [ + "../../../packages/agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent": [ + "../../../packages/coding-agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent/hooks": [ + "../../../packages/coding-agent/src/core/hooks/index.ts" + ], + "@earendil-works/pi-coding-agent/*": [ + "../../../packages/coding-agent/src/*" + ], + "typebox": [ + "../../../node_modules/typebox" + ], + "@earendil-works/pi-tui": [ + "../../../packages/tui/src/index.ts" + ], + "@earendil-works/pi-tui/*": [ + "../../../packages/tui/src/*" + ], + "@earendil-works/pi-agent-old": [ + "../../../packages/agent-old/src/index.ts" + ], + "@earendil-works/pi-agent-old/*": [ + "../../../packages/agent-old/src/*" + ], + "@mariozechner/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ] + } + }, + "include": [ + "./extensions/**/*", + "../../*/src/**/*.d.ts", + "../test/suite/regressions/8261-subagent-project-trust.test.ts" + ], + "exclude": [ + "**/node_modules/**", + "./extensions/gondolin/**" + ] +} diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 5d242776c..e37a4fb76 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -1,14 +1,14 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "engines": { "node": ">=22.19.0" @@ -386,32 +386,32 @@ } }, "node_modules/@earendil-works/pi-agent-core": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-ai": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.5.0.tgz", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -481,8 +481,8 @@ } }, "node_modules/@earendil-works/pi-codemode": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.5.0.tgz", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" @@ -492,15 +492,15 @@ } }, "node_modules/@earendil-works/pi-coding-agent": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -511,12 +511,12 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "bin": { @@ -527,8 +527,8 @@ } }, "node_modules/@earendil-works/pi-mcp": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.5.0.tgz", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" @@ -537,17 +537,25 @@ "node": ">=22.19.0" } }, + "node_modules/@earendil-works/pi-schema": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.5.0.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/@earendil-works/pi-telemetry": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.5.0.tgz", "license": "MIT", "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-tui": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.5.0.tgz", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", @@ -582,6 +590,12 @@ }, "hasInstallScript": true }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1174,12 +1188,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" @@ -1429,9 +1444,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", - "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "license": "MIT", "engines": { "node": ">=22.19.0" diff --git a/packages/coding-agent/install-lock/package.json b/packages/coding-agent/install-lock/package.json index 004cf3c90..a8746dfab 100644 --- a/packages/coding-agent/install-lock/package.json +++ b/packages/coding-agent/install-lock/package.json @@ -1,10 +1,10 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "private": true, "description": "Lockfile root used by the Pi installer and updater.", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "overrides": { "protobufjs": "7.6.6" diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 616d6a475..705be3d3d 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "type": "module", "piConfig": { @@ -29,23 +29,23 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "npm run build:unbundled && node ../../scripts/build-coding-agent-bundle.mjs", - "build:unbundled": "tsc -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", + "build:unbundled": "tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs executable dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm run build && bun build --compile --no-compile-autoload-bunfig --no-compile-autoload-dotenv ./dist/bun/cli.js ./src/utils/image-resize-worker.ts ./src/extensions/codemode/worker.ts --outfile dist/pi && npm run copy-binary-assets", - "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/assets && shx cp src/modes/interactive/assets/*.png dist/assets/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/", + "copy-assets": "node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/modes/interactive/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/modes/interactive/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/core/export-html/vendor", + "copy-binary-assets": "node ../../scripts/package-files.mjs copy package.json README.md CHANGELOG.md docs examples ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist && node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html dist/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/export-html/vendor", "test": "vitest --run", "generate:schemas": "node --import ./src/experimental/source-resolver.ts scripts/generate-schemas.ts", "generate:references": "node --import ./src/experimental/source-resolver.ts scripts/generate-references.ts", "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -56,12 +56,12 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "overrides": { @@ -74,7 +74,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "keywords": [ diff --git a/packages/coding-agent/schemas/models.schema.json b/packages/coding-agent/schemas/models.schema.json index 39ec3d63e..c51a82c35 100644 --- a/packages/coding-agent/schemas/models.schema.json +++ b/packages/coding-agent/schemas/models.schema.json @@ -255,13 +255,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -283,7 +276,14 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } @@ -383,12 +383,6 @@ "$defs": { "ModelCost": { "type": "object", - "required": [ - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "input": { "type": "number", @@ -410,13 +404,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -438,11 +425,24 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } - } + }, + "required": [ + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "ModelInputLimits": { "type": "object", @@ -628,9 +628,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -651,7 +648,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -677,9 +677,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -700,7 +697,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -1085,11 +1085,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "provider", - "model", - "cost" - ], "properties": { "provider": { "type": "string", @@ -1103,6 +1098,11 @@ "$ref": "#/$defs/ModelCost" } }, + "required": [ + "provider", + "model", + "cost" + ], "description": "An Anthropic server-side refusal fallback model with local pricing metadata." }, "maxItems": 3, diff --git a/packages/coding-agent/schemas/settings.schema.json b/packages/coding-agent/schemas/settings.schema.json index 4d6c23a6d..c331ae27a 100644 --- a/packages/coding-agent/schemas/settings.schema.json +++ b/packages/coding-agent/schemas/settings.schema.json @@ -11,6 +11,15 @@ "lastChangelogVersion": { "type": "string" }, + "extensionChangelogVersions": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "string" + } + }, + "description": "Last seen changelog version per extension package." + }, "defaultProvider": { "type": "string" }, @@ -258,6 +267,14 @@ }, "description": "Command used for npm package lookup and installation, in argv form such as [\"mise\", \"exec\", \"node@20\", \"--\", \"npm\"]." }, + "selfUpdateCommand": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Command `pi update` runs to update Pi itself, in argv form such as [\"mise\", \"use\", \"-g\", \"npm:{packageName}@{version}\"]. Placeholders: {packageName}, {version}, {installSpec}. PI_SELF_UPDATE_COMMAND takes precedence. Global setting only." + }, "collapseChangelog": { "type": "boolean", "description": "Show the condensed changelog after update; use /changelog for the full changelog.", diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index 1cbfc228b..4bcd28dc2 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -305,11 +305,11 @@ ${chalk.bold("Options:")} --append-system-prompt Append text or file contents to the system prompt (can be used multiple times) --mode Output mode: text (default), json, or rpc --print, -p Non-interactive mode: process prompt and exit - --continue, -c Continue previous session - --resume, -r Select a session to resume - --session Use specific session file or partial UUID - --session-id Use exact project session ID, creating it if missing - --fork Fork specific session file or partial UUID into a new session + --continue, -c Continue the most recent session for the current directory + --resume, -r Pick a session to resume + --session Open a session by file path or full/partial ID + --session-id Use the current directory's existing session with the ID, creating it otherwise + --fork Fork a session by file path or full/partial ID into the current directory --session-dir Directory for session storage and lookup --no-session Don't save session (ephemeral) --name, -n Set session display name @@ -463,6 +463,7 @@ ${chalk.bold("Environment Variables:")} ${ENV_SESSION_DIR.padEnd(32)} - Session storage directory (overridden by --session-dir) PI_PACKAGE_DIR - Override package directory (for Nix/Guix store paths) PI_OFFLINE - Disable startup network operations when set to 1/true/yes + PI_SELF_UPDATE_COMMAND - Command pi update runs to update pi itself ({packageName}, {version}, {installSpec}) PI_TELEMETRY - Override install telemetry when set to 1/true/yes or 0/false/no PI_SHARE_VIEWER_URL - Base URL for /share command (default: https://pi.dev/session/) diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts index 61dd63f38..5a9552d5d 100644 --- a/packages/coding-agent/src/config.ts +++ b/packages/coding-agent/src/config.ts @@ -334,6 +334,32 @@ export function getSelfUpdateCommand( return command; } +/** + * The self-update command configured by the user: `PI_SELF_UPDATE_COMMAND` (whitespace-separated + * argv, no quoting), else the global `selfUpdateCommand` setting. Overrides install-method detection. + */ +export function getConfiguredSelfUpdateArgv(setting: string[] | undefined): string[] | undefined { + const fromEnv = process.env.PI_SELF_UPDATE_COMMAND?.trim().split(/\s+/).filter(Boolean); + if (fromEnv?.length) return fromEnv; + return setting?.length ? setting : undefined; +} + +/** Fill the `{packageName}`, `{version}`, and `{installSpec}` placeholders of a configured self-update command. */ +export function makeConfiguredSelfUpdateCommand( + argv: string[], + release: { packageName: string; version: string }, +): SelfUpdateCommand { + const values: Record = { + packageName: release.packageName, + version: release.version, + installSpec: `${release.packageName}@${release.version}`, + }; + const [command = "", ...args] = argv.map((arg) => + arg.replace(/\{(packageName|version|installSpec)\}/g, (_, name: string) => values[name]), + ); + return makeSelfUpdateCommandStep(command, args); +} + export function getSelfUpdateUnavailableInstruction( packageName: string, npmCommand?: string[], @@ -345,11 +371,10 @@ export function getSelfUpdateUnavailableInstruction( return `Download from: https://github.com/earendil-works/pi/releases/latest`; } const command = getSelfUpdateCommandForMethod(method, packageName, target, npmCommand); - if (command) { - if (isManagedByGlobalPackageManager(method, packageName, npmCommand) && !isSelfUpdatePathWritable()) { - return `This installation is managed by a global ${method} install, but the install path is not writable. Update it yourself with: ${command.display}`; - } - return `This installation is not managed by a global ${method} install. Update it with the package manager, wrapper, or source checkout that provides it.`; + // Path-based detection also matches installs that only live under a package manager's directory, + // such as a wrapper under PNPM_HOME, so name the package manager only when it owns the install. + if (command && isManagedByGlobalPackageManager(method, packageName, npmCommand) && !isSelfUpdatePathWritable()) { + return `This installation is managed by a global ${method} install, but the install path is not writable. Update it yourself with: ${command.display}`; } return `Update ${target.installSpec} using the package manager, wrapper, or source checkout that provides this installation.`; } diff --git a/packages/coding-agent/src/core/agent-session-runtime.ts b/packages/coding-agent/src/core/agent-session-runtime.ts index c1d7655dc..81c4264cf 100644 --- a/packages/coding-agent/src/core/agent-session-runtime.ts +++ b/packages/coding-agent/src/core/agent-session-runtime.ts @@ -1,6 +1,6 @@ import { constants, copyFileSync, existsSync, mkdirSync } from "node:fs"; -import { basename, join, parse, resolve } from "node:path"; -import { resolvePath } from "../utils/paths.ts"; +import { basename, join, parse } from "node:path"; +import { pathsEqual, resolvePath } from "../utils/paths.ts"; import type { AgentSession } from "./agent-session.ts"; import type { AgentSessionRuntimeDiagnostic, AgentSessionServices } from "./agent-session-services.ts"; import type { @@ -368,7 +368,7 @@ export class AgentSessionRuntime { } let destinationPath = join(sessionDir, basename(resolvedPath)); - const sourceAlreadyStored = resolve(destinationPath) === resolvedPath; + const sourceAlreadyStored = pathsEqual(destinationPath, resolvedPath); if (!sourceAlreadyStored) { const { name, ext } = parse(destinationPath); let suffix = 1; diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 3a9174cf6..dbe36c365 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -29,6 +29,7 @@ import { type BeforeToolCallResult, type PrepareNextTurnContext, runToolCall, + type StreamFn, type ThinkingLevel, } from "@earendil-works/pi-agent-core"; import { contentText, getCurrentSystemMessage, getCurrentTools, retryDelayMs } from "@earendil-works/pi-ai"; @@ -158,6 +159,18 @@ import { type VirtualModelStateData, } from "./virtual-models.ts"; +/** + * Stream functions that resolve auth through a given ModelRuntime themselves, such as the one + * createAgentSession installs. Summaries sent through them must not pass pre-resolved credentials: + * the runtime would take them as an explicit API key instead of the stored OAuth login (#11520). + */ +const runtimeStreamFunctions = new WeakMap(); + +/** Mark `streamFn` as resolving auth through `modelRuntime` on its own. */ +export function markRuntimeStreamFunction(streamFn: StreamFn, modelRuntime: ModelRuntime): void { + runtimeStreamFunctions.set(streamFn, modelRuntime); +} + // ============================================================================ // Skill Block Parsing // ============================================================================ @@ -590,6 +603,9 @@ export class AgentSession { if (this.agent.streamFunction === streamSimple) { return { ...(await this._getRequiredRequestAuth(model, signal)), thinkingLevel }; } + if (runtimeStreamFunctions.get(this.agent.streamFunction) === this._modelRuntime) { + return { model, thinkingLevel }; + } try { const result = await this._modelRuntime.getAuth(model, { signal }); @@ -1591,9 +1607,14 @@ export class AgentSession { * the registered, non-hidden ones; they are declared to the model. Active tools with a * `prepareLoadout` hook can change the declared descriptions and hide declarations from * requests (see {@link _installHiddenDeclarationsProjection}). + * + * The active tools are a set: tools that stay active keep their current order and newly active + * ones are appended, so reordering the same names does not change the prompt (#11495). */ private _applyToolLoadout(toolNames: string[]): AgentTool[] { - const tools = [...new Set(toolNames)].flatMap((name) => { + const requested = new Set(toolNames); + const kept = this.getActiveToolNames().filter((name) => requested.has(name)); + const tools = [...new Set([...kept, ...requested])].flatMap((name) => { const tool = this._toolRegistry.get(name); return tool && this._getToolExposure(name) !== "hidden" && this._isActivatable(name) ? [tool] : []; }); diff --git a/packages/coding-agent/src/core/compaction/branch-summarization.ts b/packages/coding-agent/src/core/compaction/branch-summarization.ts index f1db25e14..6f8bc0cb3 100644 --- a/packages/coding-agent/src/core/compaction/branch-summarization.ts +++ b/packages/coding-agent/src/core/compaction/branch-summarization.ts @@ -5,7 +5,7 @@ * a summary of the branch being left so context isn't lost. */ -import type { AgentMessage, StreamFn } from "@earendil-works/pi-agent-core"; +import type { AgentMessage, StreamFn, ThinkingLevel } from "@earendil-works/pi-agent-core"; import type { RetryCallbacks, RetryPolicy } from "@earendil-works/pi-ai"; import { contentText, normalizeContext } from "@earendil-works/pi-ai"; import type { Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat"; @@ -15,7 +15,12 @@ import { createCompactionSummaryMessage, createCustomMessage, } from "../messages.ts"; -import type { ReadonlySessionManager, SessionEntry } from "../session-manager.ts"; +import { + type ContextEditEntry, + projectContextEntry, + type ReadonlySessionManager, + type SessionEntry, +} from "../session-manager.ts"; import { completeSummarization, estimateTokens, getSummarizationFailure } from "./compaction.ts"; import { computeFileLists, @@ -75,6 +80,8 @@ export interface GenerateBranchSummaryOptions { env?: Record; /** Abort signal for cancellation */ signal: AbortSignal; + /** Thinking level for reasoning-capable models */ + thinkingLevel?: ThinkingLevel; /** Optional custom instructions for summarization */ customInstructions?: string; /** If true, customInstructions replaces the default prompt instead of being appended */ @@ -153,14 +160,15 @@ export function collectEntriesForBranchSummary( * Extract AgentMessage from a session entry. * Similar to getMessageFromEntry in compaction.ts but also handles compaction entries. */ -function getMessageFromEntry(entry: SessionEntry): AgentMessage | undefined { +function getMessageFromEntry(entry: SessionEntry, edit?: ContextEditEntry): AgentMessage | undefined { switch (entry.type) { case "message": // Skip tool results - context is in assistant's tool call if (entry.message.role === "toolResult") return undefined; - return entry.message; + return edit ? projectContextEntry(entry, edit)[0] : entry.message; case "custom_message": + if (edit) return projectContextEntry(entry, edit)[0]; return createCustomMessage(entry.customType, entry.content, entry.display, entry.details, entry.timestamp); case "branch_summary": @@ -215,10 +223,16 @@ export function prepareBranchEntries(entries: SessionEntry[], tokenBudget: numbe } } + // Context edits hide or replace earlier entries in model context; honor the latest edit per target. + const edits = new Map(); + for (const entry of entries) { + if (entry.type === "context_edit") edits.set(entry.targetId, entry); + } + // Second pass: walk from newest to oldest, adding messages until token budget for (let i = entries.length - 1; i >= 0; i--) { const entry = entries[i]; - const message = getMessageFromEntry(entry); + const message = getMessageFromEntry(entry, edits.get(entry.id)); if (!message) continue; // Extract file ops from assistant messages (tool calls) @@ -300,6 +314,7 @@ export async function generateBranchSummary( headers, env, signal, + thinkingLevel, customInstructions, replaceInstructions, reserveTokens = 16384, @@ -348,6 +363,9 @@ export async function generateBranchSummary( // so transient stream drops reuse the configured retry policy. const context = normalizeContext({ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages }); const requestOptions: SimpleStreamOptions = { apiKey, headers, env, signal }; + if (model.reasoning && thinkingLevel && thinkingLevel !== "off") { + requestOptions.reasoning = thinkingLevel; + } const response = await completeSummarization(model, context, requestOptions, streamFn, retry, callbacks); // Check if aborted or errored diff --git a/packages/coding-agent/src/core/exec.ts b/packages/coding-agent/src/core/exec.ts index cf4205f7f..b6e845f45 100644 --- a/packages/coding-agent/src/core/exec.ts +++ b/packages/coding-agent/src/core/exec.ts @@ -2,10 +2,14 @@ * Shared command execution utilities for extensions and custom tools. */ +import { constants as bufferConstants } from "node:buffer"; import { spawn } from "node:child_process"; import { StringDecoder } from "node:string_decoder"; import { waitForChildProcess } from "../utils/child-process.ts"; +const DEFAULT_EXEC_MAX_BUFFER = Math.min(256 * 1024 * 1024, bufferConstants.MAX_STRING_LENGTH); +const FORCE_KILL_DELAY_MS = 5000; + /** * Options for executing shell commands. */ @@ -16,6 +20,11 @@ export interface ExecOptions { timeout?: number; /** Working directory */ cwd?: string; + /** + * Combined stdout/stderr byte limit; exceeding it terminates the command with `truncated: true`. + * Nonnegative integer up to the runtime string limit. Default: 256 MiB, capped by that limit. + */ + maxBuffer?: number; } /** @@ -25,12 +34,15 @@ export interface ExecResult { stdout: string; stderr: string; code: number; + /** The command was terminated by pi: timeout, abort, or output over `maxBuffer`. */ killed: boolean; + /** Output exceeded `maxBuffer`; stdout and stderr are incomplete and `code` is nonzero. */ + truncated: boolean; } /** * Execute a shell command and return stdout/stderr/code. - * Supports timeout and abort signal. + * Supports timeout, abort signal, and an output cap (`maxBuffer`). */ export async function execCommand( command: string, @@ -38,79 +50,109 @@ export async function execCommand( cwd: string, options?: ExecOptions, ): Promise { - return new Promise((resolve) => { - const proc = spawn(command, args, { - cwd, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }); + const maxBuffer = options?.maxBuffer ?? DEFAULT_EXEC_MAX_BUFFER; + if (!Number.isInteger(maxBuffer) || maxBuffer < 0 || maxBuffer > bufferConstants.MAX_STRING_LENGTH) { + throw new RangeError(`maxBuffer must be an integer between 0 and ${bufferConstants.MAX_STRING_LENGTH} bytes`); + } - let stdout = ""; - let stderr = ""; - // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, - // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). - const stdoutDecoder = new StringDecoder("utf8"); - const stderrDecoder = new StringDecoder("utf8"); - let killed = false; - let timeoutId: NodeJS.Timeout | undefined; - - const killProcess = () => { - if (!killed) { - killed = true; - proc.kill("SIGTERM"); - // Force kill after 5 seconds if SIGTERM doesn't work - setTimeout(() => { - if (!proc.killed) { - proc.kill("SIGKILL"); - } - }, 5000); - } - }; - - // Handle abort signal - if (options?.signal) { - if (options.signal.aborted) { - killProcess(); - } else { - options.signal.addEventListener("abort", killProcess, { once: true }); - } - } - - // Handle timeout - if (options?.timeout && options.timeout > 0) { - timeoutId = setTimeout(() => { - killProcess(); - }, options.timeout); - } - - proc.stdout?.on("data", (data) => { - stdout += stdoutDecoder.write(data); - }); - - proc.stderr?.on("data", (data) => { - stderr += stderrDecoder.write(data); - }); - - // Wait for process termination without hanging on inherited stdio handles - // held open by detached descendants. - waitForChildProcess(proc) - .then((code) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr, code: code ?? 0, killed }); - }) - .catch((error: Error) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr: stderr || error.message, code: 1, killed }); - }); + const proc = spawn(command, args, { + cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], }); + + let stdout = ""; + let stderr = ""; + // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, + // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). + const stdoutDecoder = new StringDecoder("utf8"); + const stderrDecoder = new StringDecoder("utf8"); + let outputBytes = 0; + let truncated = false; + let killed = false; + let timeoutId: NodeJS.Timeout | undefined; + let forceKillId: NodeJS.Timeout | undefined; + + const killProcess = () => { + if (killed) return; + killed = true; + proc.kill("SIGTERM"); + forceKillId = setTimeout(() => { + // `proc.killed` only says a signal was sent, so check whether the child actually exited. + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + } + }, FORCE_KILL_DELAY_MS); + }; + + if (options?.signal) { + if (options.signal.aborted) { + killProcess(); + } else { + options.signal.addEventListener("abort", killProcess, { once: true }); + } + } + + if (options?.timeout && options.timeout > 0) { + timeoutId = setTimeout(killProcess, options.timeout); + } + + // Keep only bytes within the combined cap (#11360). Close both pipes on overflow so writing + // descendants cannot keep waitForChildProcess busy after the child exits. Pausing is not + // enough: Node resumes the child's stdio on exit. + const takeWithinCap = (data: Buffer): Buffer | undefined => { + if (truncated) return undefined; + const remaining = maxBuffer - outputBytes; + if (data.length <= remaining) { + outputBytes += data.length; + return data; + } + truncated = true; + proc.stdout?.destroy(); + proc.stderr?.destroy(); + killProcess(); + return remaining > 0 ? data.subarray(0, remaining) : undefined; + }; + + proc.stdout?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stdout += stdoutDecoder.write(chunk); + }); + + proc.stderr?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stderr += stderrDecoder.write(chunk); + }); + + let code: number | null; + try { + // Do not hang on inherited stdio handles held open by detached descendants. + code = await waitForChildProcess(proc); + } catch (error) { + if (!(error instanceof Error)) throw error; + // Child-process errors (e.g. ENOENT) are failed executions, not host failures. + code = 1; + if (!stderr) stderr = error.message; + } finally { + clearTimeout(timeoutId); + clearTimeout(forceKillId); + options?.signal?.removeEventListener("abort", killProcess); + } + + // On truncation, discard any incomplete UTF-8 suffix buffered by either decoder. + // Flushing it would append a replacement character that was not in the accepted output. + if (!truncated) { + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); + } + + const exitCode = code ?? 0; + return { + stdout, + stderr, + // A truncated result must never read as success. + code: truncated && exitCode === 0 ? 1 : exitCode, + killed, + truncated, + }; } diff --git a/packages/coding-agent/src/core/export-html/tool-renderer.ts b/packages/coding-agent/src/core/export-html/tool-renderer.ts index 1f8c1fd63..aac0e199b 100644 --- a/packages/coding-agent/src/core/export-html/tool-renderer.ts +++ b/packages/coding-agent/src/core/export-html/tool-renderer.ts @@ -7,6 +7,7 @@ import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import type { Component } from "@earendil-works/pi-tui"; +import { keyHint, keyText } from "../../modes/interactive/components/keybinding-hints.ts"; import type { Theme } from "../../modes/interactive/theme/theme.ts"; import type { ToolRenderContext, ToolRenderers } from "../extensions/types.ts"; import { ansiLinesToHtml } from "./ansi-to-html.ts"; @@ -94,6 +95,8 @@ export function createToolHtmlRenderer(deps: ToolHtmlRendererDeps): ToolHtmlRend isError, durationMs: undefined, outputPad: 1, + keyText, + keyHint, }; }; diff --git a/packages/coding-agent/src/core/extensions/index.ts b/packages/coding-agent/src/core/extensions/index.ts index 1465e8267..99c0044cd 100644 --- a/packages/coding-agent/src/core/extensions/index.ts +++ b/packages/coding-agent/src/core/extensions/index.ts @@ -109,6 +109,7 @@ export type { InputEvent, InputEventResult, InputSource, + KeybindingHints, KeybindingsManager, LoadExtensionsResult, LsToolCallEvent, diff --git a/packages/coding-agent/src/core/extensions/loader.ts b/packages/coding-agent/src/core/extensions/loader.ts index cbc1100fc..ff9f127c4 100644 --- a/packages/coding-agent/src/core/extensions/loader.ts +++ b/packages/coding-agent/src/core/extensions/loader.ts @@ -11,7 +11,7 @@ import type { Provider } from "@earendil-works/pi-ai"; import type { KeyId } from "@earendil-works/pi-tui"; import type { createJiti } from "jiti"; import { CONFIG_DIR_NAME, getAgentDir, isBunBinary, isBundledNode } from "../../config.ts"; -import { resolvePath } from "../../utils/paths.ts"; +import { pathsEqual, resolvePath } from "../../utils/paths.ts"; import { createEventBus, type EventBus } from "../event-bus.ts"; import type { ExecOptions } from "../exec.ts"; import { execCommand } from "../exec.ts"; @@ -88,10 +88,11 @@ function getAliases(): Record { const piCodingAgentEntry = packageIndex; const piAgentCoreEntry = resolveWorkspaceOrImport("agent/dist/index.js", "@earendil-works/pi-agent-core"); const piTuiEntry = resolveWorkspaceOrImport("tui/dist/index.js", "@earendil-works/pi-tui"); - // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - const piAiCompatEntry = resolveWorkspaceOrImport("ai/dist/compat.js", "@earendil-works/pi-ai/compat"); + // Extensions resolve the pi-ai root to the extension view: the compat + // entrypoint (a strict superset of the core entrypoint) plus TypeBox's Type, + // Static and TSchema, so extensions using the old global API and pi-ai's + // former TypeBox re-exports keep working. + const piAiCompatEntry = path.resolve(__dirname, "pi-ai-extension-view.js"); const piAiModelsEntry = resolveWorkspaceOrImport("ai/dist/models.js", "@earendil-works/pi-ai/models"); const piAiOauthEntry = resolveWorkspaceOrImport("ai/dist/oauth.js", "@earendil-works/pi-ai/oauth"); const piAiProvidersEntry = resolveWorkspaceOrImport( @@ -146,7 +147,7 @@ export function clearExtensionCache(): void { function useExtensionCacheCwd(cwd: string): ExtensionCacheToken { const resolvedCwd = resolvePath(cwd); - if (extensionCacheCwd !== undefined && extensionCacheCwd !== resolvedCwd) { + if (extensionCacheCwd !== undefined && !pathsEqual(extensionCacheCwd, resolvedCwd)) { clearExtensionCache(); } extensionCacheCwd = resolvedCwd; diff --git a/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts new file mode 100644 index 000000000..e9c6808f7 --- /dev/null +++ b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts @@ -0,0 +1,28 @@ +/** + * The `@earendil-works/pi-ai` module extensions import: pi-ai's compat entry point plus TypeBox's `Type`, `Static` and + * `TSchema`, which pi-ai's main entry no longer re-exports, and a `StringEnum` built with TypeBox (`TUnsafe`), so + * extensions that use it inside `Type.Object` type-check and infer as before. The TypeBox is the coding agent's own, + * the same one extensions get for `typebox`. + */ + +import { type TUnsafe, Type } from "typebox"; + +export * from "@earendil-works/pi-ai/compat"; +export type { Static, TSchema } from "typebox"; +export { Type }; + +/** + * Creates a string enum schema compatible with Google's API and other providers that don't support anyOf/const + * patterns: `{ type: "string", enum: [...] }`, typed for TypeBox's `Static`. + */ +export function StringEnum( + values: T, + options?: { description?: string; default?: T[number] }, +): TUnsafe { + return Type.Unsafe({ + type: "string", + enum: values as unknown as T[number][], + ...(options?.description && { description: options.description }), + ...(options?.default && { default: options.default }), + }); +} diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index b29e2d4b0..098f7fb0d 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -49,9 +49,11 @@ import type { Component, EditorComponent, EditorTheme, + Keybinding, KeyId, OverlayHandle, OverlayOptions, + ProgramStatus, TUI, } from "@earendil-works/pi-tui"; import type { Static, TSchema } from "typebox"; @@ -142,6 +144,19 @@ export interface WorkingIndicatorOptions { export type AutocompleteProviderFactory = (current: AutocompleteProvider) => AutocompleteProvider; export type EditorFactory = (tui: TUI, theme: EditorTheme, keybindings: KeybindingsManager) => EditorComponent; +/** + * Formats the host's configured keybindings for display. + * Extensions should use these instead of importing `keyText`/`keyHint`: an import may resolve to another copy of + * pi-coding-agent, which does not see the host's keybindings. + */ +export interface KeybindingHints { + /** Configured keys for a keybinding id, for example `"ctrl+o"`. Empty when the id has no keys. */ + keyText(keybinding: Keybinding): string; + + /** Configured keys followed by a description, styled as a hint, for example `"ctrl+o to expand"`. */ + keyHint(keybinding: Keybinding, description: string): string; +} + /** * UI context for extensions to request interactive UI. * Each mode (interactive, RPC, print) provides its own implementation. @@ -224,6 +239,11 @@ export interface ExtensionUIContext { overlayOptions?: OverlayOptions | (() => OverlayOptions); /** Called with the overlay handle after the overlay is shown. Use to control visibility. */ onHandle?: (handle: OverlayHandle) => void; + /** + * Report pi as blocked on user input (OSC 7501 program status) while this component is visible. + * Only applies to inline components, not overlays. + */ + blocked?: { kind: NonNullable; message: string }; }, ): Promise; @@ -464,7 +484,7 @@ export interface ToolRenderResultOptions { } /** Context passed to tool renderers. */ -export interface ToolRenderContext { +export interface ToolRenderContext extends KeybindingHints { /** Current tool call arguments. Shared across call/result renders for the same tool call. */ args: TArgs; /** Unique id for this tool execution. Stable across call/result renders for the same tool call. */ @@ -1532,7 +1552,7 @@ export interface SessionBeforeTreeResult { // Message and Entry Rendering // ============================================================================ -export interface MessageRenderOptions { +export interface MessageRenderOptions extends KeybindingHints { expanded: boolean; /** Horizontal padding configured by the outputPad setting. */ outputPad: number; @@ -1550,7 +1570,7 @@ export interface MarkdownTransformContext { export type MarkdownTransformer = (markdown: string, context: MarkdownTransformContext) => string; -export interface EntryRenderOptions { +export interface EntryRenderOptions extends KeybindingHints { expanded: boolean; } diff --git a/packages/coding-agent/src/core/extensions/virtual-modules.ts b/packages/coding-agent/src/core/extensions/virtual-modules.ts index 4b8449338..3c30317d0 100644 --- a/packages/coding-agent/src/core/extensions/virtual-modules.ts +++ b/packages/coding-agent/src/core/extensions/virtual-modules.ts @@ -1,5 +1,4 @@ import * as bundledPiAgentCore from "@earendil-works/pi-agent-core"; -import * as bundledPiAiCompat from "@earendil-works/pi-ai/compat"; import * as bundledPiAiModels from "@earendil-works/pi-ai/models"; import * as bundledPiAiOauth from "@earendil-works/pi-ai/oauth"; import * as bundledPiAiProviders from "@earendil-works/pi-ai/providers/all"; @@ -10,6 +9,7 @@ import * as bundledTypeboxValue from "typebox/value"; // This import is safe because loader.ts exports are not re-exported from index.ts. // Extensions can therefore import from @earendil-works/pi-coding-agent. import * as bundledPiCodingAgent from "../../index.ts"; +import * as bundledPiAiExtensionView from "./pi-ai-extension-view.ts"; /** Modules available to extensions in source and compiled binary runtimes. */ export const VIRTUAL_MODULES: Record = { @@ -22,18 +22,19 @@ export const VIRTUAL_MODULES: Record = { "@earendil-works/pi-agent-core": bundledPiAgentCore, "@earendil-works/pi-tui": bundledPiTui, // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - "@earendil-works/pi-ai": bundledPiAiCompat, - "@earendil-works/pi-ai/compat": bundledPiAiCompat, + // superset of the core entrypoint) plus TypeBox's Type, Static and TSchema: + // existing extensions using the old global API and pi-ai's former TypeBox + // re-exports keep working. + "@earendil-works/pi-ai": bundledPiAiExtensionView, + "@earendil-works/pi-ai/compat": bundledPiAiExtensionView, "@earendil-works/pi-ai/models": bundledPiAiModels, "@earendil-works/pi-ai/oauth": bundledPiAiOauth, "@earendil-works/pi-ai/providers/all": bundledPiAiProviders, "@earendil-works/pi-coding-agent": bundledPiCodingAgent, "@mariozechner/pi-agent-core": bundledPiAgentCore, "@mariozechner/pi-tui": bundledPiTui, - "@mariozechner/pi-ai": bundledPiAiCompat, - "@mariozechner/pi-ai/compat": bundledPiAiCompat, + "@mariozechner/pi-ai": bundledPiAiExtensionView, + "@mariozechner/pi-ai/compat": bundledPiAiExtensionView, "@mariozechner/pi-ai/models": bundledPiAiModels, "@mariozechner/pi-ai/oauth": bundledPiAiOauth, "@mariozechner/pi-ai/providers/all": bundledPiAiProviders, diff --git a/packages/coding-agent/src/core/mcp-servers.ts b/packages/coding-agent/src/core/mcp-servers.ts index 1f6414928..46b4241d5 100644 --- a/packages/coding-agent/src/core/mcp-servers.ts +++ b/packages/coding-agent/src/core/mcp-servers.ts @@ -112,11 +112,13 @@ export interface McpOAuthConfig { */ clientName?: string; /** - * How pi identifies itself without `clientId`. `dcr` (default): dynamic client registration. `cimd`: - * pi's Client ID Metadata Document on pi.dev, for authorization servers that allow pi by that URL. The - * server must support it for public clients, and the callback must use the default path `/callback`. + * How pi identifies itself without `clientId`. `auto` (default): pi's Client ID Metadata Document on + * pi.dev when the authorization server supports it for public clients, otherwise dynamic client + * registration; with `clientName` or a `callbackUrl` other than `/callback` on localhost or 127.0.0.1, + * always dynamic client registration. `cimd`: always the document, and sign-in fails without support. + * `dcr`: always dynamic client registration. */ - clientRegistration?: "dcr" | "cimd"; + clientRegistration?: "auto" | "dcr" | "cimd"; /** * Authorization server metadata document (RFC 8414 or OpenID Connect discovery) to use instead of * discovery through the server, for servers that advertise a wrong authorization server or none. @@ -134,6 +136,16 @@ export function isLoopbackRedirectUri(value: string): boolean { return url.protocol === "http:" && LOOPBACK_HOSTS.includes(url.hostname) && url.search === "" && url.hash === ""; } +/** + * Whether pi's Client ID Metadata Documents list a redirect URI for the loopback `callbackUrl`: they list + * `/callback` on localhost and 127.0.0.1, on any port. Without `callbackUrl`, pi uses such a URI. + */ +export function isClientMetadataDocumentCallback(callbackUrl: string | undefined): boolean { + if (callbackUrl === undefined) return true; + const url = new URL(callbackUrl); + return url.hostname !== "[::1]" && url.pathname === "/callback"; +} + export interface McpHttpServerConfig extends McpServerConfigBase { type?: "http"; url: string; @@ -188,13 +200,13 @@ function validateOAuth(value: unknown): string | undefined { if (value.clientName !== undefined && (typeof value.clientName !== "string" || !value.clientName.trim())) { return "oauth.clientName must be a non-empty string"; } - if (value.clientRegistration !== undefined && value.clientRegistration !== "dcr") { - if (value.clientRegistration !== "cimd") return 'oauth.clientRegistration must be "dcr" or "cimd"'; + const registration = value.clientRegistration; + if (registration !== undefined && registration !== "auto" && registration !== "dcr") { + if (registration !== "cimd") return 'oauth.clientRegistration must be "auto", "dcr", or "cimd"'; if (value.clientId !== undefined || value.clientName !== undefined) { return 'oauth.clientRegistration "cimd" cannot be combined with oauth.clientId or oauth.clientName'; } - const callback = typeof value.callbackUrl === "string" ? new URL(value.callbackUrl) : undefined; - if (callback && (callback.hostname === "[::1]" || callback.pathname !== "/callback")) { + if (!isClientMetadataDocumentCallback(typeof value.callbackUrl === "string" ? value.callbackUrl : undefined)) { return 'oauth.clientRegistration "cimd" requires oauth.callbackUrl on localhost or 127.0.0.1 with path /callback'; } } diff --git a/packages/coding-agent/src/core/model-config.ts b/packages/coding-agent/src/core/model-config.ts index 553ff35ba..200cdeec0 100644 --- a/packages/coding-agent/src/core/model-config.ts +++ b/packages/coding-agent/src/core/model-config.ts @@ -1,20 +1,21 @@ /** Immutable, credential-blind models.json snapshot. */ import { readFile } from "node:fs/promises"; -import { ProviderCompatSchema } from "@earendil-works/pi-ai/providers/compat-schema"; -import { - ModelCostSchema, - ModelInputLimitsSchema, - ModelInputModalitySchema, - ModelPromptCacheSchema, - ThinkingLevelMapSchema, -} from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, Type } from "typebox"; import { Compile } from "typebox/compile"; import type { TLocalizedValidationError } from "typebox/error"; import { stripJsonComments } from "../utils/json.ts"; import { normalizePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; +import { + ModelCostSchema, + ModelInputLimitsSchema, + ModelInputModalitySchema, + ModelPromptCacheSchema, + PartialModelCostSchema, + ProviderCompatSchema, + ThinkingLevelMapSchema, +} from "./pi-ai-schemas.ts"; const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown()); const SamplingParamsByThinkingLevelSchema = Type.Object( @@ -74,7 +75,7 @@ const ModelOverrideSchema = Type.Object({ thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), input: Type.Optional(Type.Array(ModelInputModalitySchema)), inputLimits: Type.Optional(ModelInputLimitsSchema), - cost: Type.Optional(Type.Partial(ModelCostSchema)), + cost: Type.Optional(PartialModelCostSchema), promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional( Type.Number({ exclusiveMinimum: 0, description: ModelFieldDescriptions.contextWindow }), diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index e6657e9eb..d1abd57cf 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -21,7 +21,7 @@ export const defaultModelPerProvider: Partial> = { "amazon-bedrock": "us.anthropic.claude-opus-4-6-v1", "ant-ling": "Ring-2.6-1T", anthropic: "claude-opus-4-8", - openai: "gpt-5.5", + openai: "gpt-6.1-sol", azure: "gpt-5.4", "openai-codex": "gpt-6.1-sol", radius: "balanced", @@ -128,13 +128,18 @@ function splitProviderPrefix( text: string, models: readonly Model[], ): { provider: string; modelText: string } | undefined { - const slashIndex = text.indexOf("/"); - if (slashIndex === -1) return undefined; - const prefix = text.substring(0, slashIndex).trim().toLowerCase(); - const modelText = text.substring(slashIndex + 1).trim(); - if (!prefix || !modelText) return undefined; - const provider = models.find((model) => model.provider.toLowerCase() === prefix)?.provider; - return provider ? { provider, modelText } : undefined; + // Provider ids may contain "/" themselves (e.g. URL-derived ids), so match known ids, longest first + const lowerText = text.toLowerCase(); + let best: { provider: string; modelText: string } | undefined; + for (const { provider } of models) { + const prefix = provider.trim().toLowerCase(); + if (!prefix || (best && best.provider.length >= provider.length)) continue; + const slashIndex = lowerText.indexOf("/", prefix.length); + if (slashIndex === -1 || lowerText.substring(0, slashIndex).trim() !== prefix) continue; + const modelText = text.substring(slashIndex + 1).trim(); + if (modelText) best = { provider, modelText }; + } + return best; } /** diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index b5015b5ba..5f4229b24 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -7,6 +7,7 @@ import { mkdirSync, readdirSync, readFileSync, + realpathSync, rmSync, statSync, writeFileSync, @@ -32,7 +33,7 @@ function getEnv(): NodeJS.ProcessEnv { } } -import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import type { Readable } from "node:stream"; import ignore from "ignore"; import { minimatch } from "minimatch"; @@ -42,16 +43,18 @@ import { spawnProcess, spawnProcessSync } from "../utils/child-process.ts"; import { type GitSource, parseGitUrl } from "../utils/git.ts"; import { canonicalizePath, + getPathComparisonKey, isLocalPath, markPathIgnoredByCloudSync, normalizePath, + pathsEqual, resolvePath, } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; import { isStdoutTakenOver } from "./output-guard.ts"; import { type PiManifest, readPiManifest } from "./pi-manifest.ts"; import type { PackageSource, SettingsManager } from "./settings-manager.ts"; -import { BUILTIN_PATH_PREFIX } from "./source-info.ts"; +import { BUILTIN_PATH_PREFIX, getResourcePathKey } from "./source-info.ts"; const NETWORK_TIMEOUT_MS = 10000; const UPDATE_CHECK_CONCURRENCY = 4; @@ -76,6 +79,7 @@ export interface PathMetadata { scope: SourceScope; origin: "package" | "top-level"; baseDir?: string; + changelogPath?: string; packageRoot?: string; } @@ -179,6 +183,10 @@ interface ConfiguredUpdateSource { interface NpmUpdateTarget extends ConfiguredUpdateSource { parsed: NpmSource; + /** Version installed before the update, if any. */ + installedVersion?: string; + /** Newer version reported by the registry, if the lookup succeeded. */ + targetVersion?: string; } interface GitUpdateTarget extends ConfiguredUpdateSource { @@ -673,6 +681,37 @@ function collectResourceFiles(dir: string, resourceType: ResourceType): string[] return collectFiles(dir, FILE_PATTERNS[resourceType]); } +// A `pi` object counts as an explicit resource manifest only when it lists resources. Packages whose +// `pi` object holds only metadata (changelogPath, gallery image/video) still use directory auto-discovery. +function hasManifestResourceFields(manifest: PiManifest): boolean { + return RESOURCE_TYPES.some((resourceType) => manifest[resourceType] !== undefined); +} + +function isPathInside(parent: string, child: string): boolean { + const relativePath = relative(parent, child); + return ( + relativePath !== "" && relativePath !== ".." && !relativePath.startsWith(`..${sep}`) && !isAbsolute(relativePath) + ); +} + +function resolvePackageChangelogPath(packageRoot: string, manifest: PiManifest | null): string | undefined { + const changelogPath = manifest?.changelogPath ?? "CHANGELOG.md"; + if (!changelogPath) return undefined; + + const resolvedPath = resolve(packageRoot, changelogPath); + if (!existsSync(resolvedPath)) return undefined; + + const canonicalPackageRoot = canonicalizePath(packageRoot); + const canonicalChangelogPath = canonicalizePath(resolvedPath); + if (!isPathInside(canonicalPackageRoot, canonicalChangelogPath)) return undefined; + + try { + return statSync(resolvedPath).isFile() ? resolvedPath : undefined; + } catch { + return undefined; + } +} + function matchesAnyPattern(filePath: string, patterns: string[], baseDir: string): boolean { const rel = toPosixPath(relative(baseDir, filePath)); const name = basename(filePath); @@ -1155,10 +1194,13 @@ export class DefaultPackageManager implements PackageManager { } } - const npmCheckTasks = npmCandidates.map((entry) => async () => ({ - entry, - shouldUpdate: await this.shouldUpdateNpmSource(entry.parsed, entry.scope), - })); + const npmCheckTasks = npmCandidates.map((entry) => async () => { + const check = await this.shouldUpdateNpmSource(entry.parsed, entry.scope); + return { + entry: { ...entry, installedVersion: check.installedVersion, targetVersion: check.targetVersion }, + shouldUpdate: check.shouldUpdate, + }; + }); const npmCheckResults = await this.runWithConcurrency(npmCheckTasks, UPDATE_CHECK_CONCURRENCY); const userNpmUpdates: NpmUpdateTarget[] = []; const projectNpmUpdates: NpmUpdateTarget[] = []; @@ -1193,19 +1235,22 @@ export class DefaultPackageManager implements PackageManager { await Promise.all(tasks); } - private async shouldUpdateNpmSource(source: NpmSource, scope: InstalledSourceScope): Promise { + private async shouldUpdateNpmSource( + source: NpmSource, + scope: InstalledSourceScope, + ): Promise<{ shouldUpdate: boolean; installedVersion?: string; targetVersion?: string }> { const installedPath = this.getManagedNpmInstallPath(source, scope); const installedVersion = existsSync(installedPath) ? this.getInstalledNpmVersion(installedPath) : undefined; if (!installedVersion) { - return true; + return { shouldUpdate: true }; } try { const targetVersion = await this.getLatestNpmVersion(source.version ? source.spec : source.name, source.range); - return gt(targetVersion, installedVersion); + return { shouldUpdate: gt(targetVersion, installedVersion), installedVersion, targetVersion }; } catch { // Preserve existing update behavior when version lookup fails. - return true; + return { shouldUpdate: true, installedVersion }; } } @@ -1221,6 +1266,31 @@ export class DefaultPackageManager implements PackageManager { await this.withProgress("update", sourceLabel, message, async () => { await this.installNpmBatch(specs, scope); }); + + this.reportUnchangedNpmUpdates(sources, scope); + } + + /** + * npm can resolve stale registry metadata shortly after a publish and install the old + * version without an error. Warn when an announced update did not change the installed version. + */ + private reportUnchangedNpmUpdates(sources: NpmUpdateTarget[], scope: InstalledSourceScope): void { + for (const entry of sources) { + if (!entry.installedVersion || !entry.targetVersion) { + continue; + } + const installedPath = this.getManagedNpmInstallPath(entry.parsed, scope); + const versionAfter = this.getInstalledNpmVersion(installedPath); + if (versionAfter !== entry.installedVersion) { + continue; + } + this.emitProgress({ + type: "progress", + action: "update", + source: entry.source, + message: `${entry.source} is still ${versionAfter} (registry reports ${entry.targetVersion}); npm registry metadata may be stale, try again in a few minutes`, + }); + } } private async installNpmBatch(specs: string[], scope: InstalledSourceScope): Promise { @@ -1341,6 +1411,22 @@ export class DefaultPackageManager implements PackageManager { let installedPath = this.getNpmInstallPath(parsed, resolvedScope); const needsInstall = !existsSync(installedPath) || !(await this.installedNpmMatchesConfiguredVersion(parsed, installedPath)); + if ( + !needsInstall && + resolvedScope === "temporary" && + !parsed.pinned && + (await this.npmHasAvailableUpdate(parsed, installedPath)) + ) { + // Temporary (-e) sources have no `pi update` path, so refresh unpinned ones on load like + // temporary git sources. Like `pi update`, install `name@latest` for unversioned specs because a bare + // `npm install name` keeps the range already saved in the temporary package.json. + const spec = parsed.version ? parsed.spec : `${parsed.name}@latest`; + try { + await this.installNpm({ ...parsed, spec }, resolvedScope, true); + } catch { + // Keep the cached install if the refresh fails. + } + } if (needsInstall) { const installed = await installMissing(); if (!installed) continue; @@ -1357,8 +1443,8 @@ export class DefaultPackageManager implements PackageManager { if (!existsSync(installedPath)) { const installed = await installMissing(); if (!installed) continue; - } else if (resolvedScope === "temporary" && !parsed.pinned && !isOfflineModeEnabled()) { - await this.refreshTemporaryGitSource(parsed, resolvedSource); + } else if (resolvedScope === "temporary" && !isOfflineModeEnabled()) { + await this.refreshTemporaryGitSource(parsed, resolvedSource, installedPath); } metadata.baseDir = installedPath; metadata.packageRoot = installedPath; @@ -1438,7 +1524,7 @@ export class DefaultPackageManager implements PackageManager { if (parsed.type === "git") { return `git:${parsed.host}/${parsed.path}`; } - return `local:${this.resolvePath(parsed.path)}`; + return `local:${getPathComparisonKey(this.resolvePath(parsed.path))}`; } private getSourceMatchKeyForSettings(source: string, scope: SourceScope): string { @@ -1450,7 +1536,7 @@ export class DefaultPackageManager implements PackageManager { return `git:${parsed.host}/${parsed.path}`; } const baseDir = this.getBaseDirForScope(scope); - return `local:${this.resolvePathFromBase(parsed.path, baseDir)}`; + return `local:${getPathComparisonKey(this.resolvePathFromBase(parsed.path, baseDir))}`; } private buildNoMatchingPackageMessage(source: string, configuredPackages: PackageSource[]): string { @@ -1755,9 +1841,9 @@ export class DefaultPackageManager implements PackageManager { } if (scope) { const baseDir = this.getBaseDirForScope(scope); - return `local:${this.resolvePathFromBase(parsed.path, baseDir)}`; + return `local:${getPathComparisonKey(this.resolvePathFromBase(parsed.path, baseDir))}`; } - return `local:${this.resolvePath(parsed.path)}`; + return `local:${getPathComparisonKey(this.resolvePath(parsed.path))}`; } /** @@ -1857,7 +1943,7 @@ export class DefaultPackageManager implements PackageManager { "--config.strict-dep-builds=false", ]; case "npm": - return ["install", "--omit=dev", "--legacy-peer-deps"]; + return ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"]; default: return ["install"]; } @@ -1868,7 +1954,9 @@ export class DefaultPackageManager implements PackageManager { return this.runCommandSync(npmCommand.command, [...npmCommand.args, ...args]); } - private getNpmInstallArgs(specs: string[], installRoot: string): string[] { + private getNpmInstallArgs(specs: string[], root: string): string[] { + // npm writes broken nested package-lock.json paths when --prefix goes through a symlink. + const installRoot = existsSync(root) ? realpathSync(root) : root; const packageManagerName = this.getPackageManagerName(); // Extension packages run inside pi and resolve pi APIs through loader aliases/virtual modules. // Disable peer dependency resolution for managed installs (npm's --legacy-peer-deps, and @@ -1888,7 +1976,8 @@ export class DefaultPackageManager implements PackageManager { "--config.strict-dep-builds=false", ]; } - return ["install", ...specs, "--prefix", installRoot, "--legacy-peer-deps"]; + // npm audit fix would run without --legacy-peer-deps and install pi peers, so skip audit advice. + return ["install", ...specs, "--prefix", installRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"]; } private async installNpm(source: NpmSource, scope: SourceScope, temporary: boolean): Promise { @@ -1898,10 +1987,12 @@ export class DefaultPackageManager implements PackageManager { } private async uninstallNpm(source: NpmSource, scope: SourceScope): Promise { - const installRoot = this.getNpmInstallRoot(scope, false); - if (!existsSync(installRoot)) { + const root = this.getNpmInstallRoot(scope, false); + if (!existsSync(root)) { return; } + // npm writes broken nested package-lock.json paths when --prefix goes through a symlink. + const installRoot = realpathSync(root); const packageManagerName = this.getPackageManagerName(); if (packageManagerName === "bun") { await this.runNpmCommand(["uninstall", source.name, "--cwd", installRoot]); @@ -1909,7 +2000,7 @@ export class DefaultPackageManager implements PackageManager { } const args = ["uninstall", source.name, "--prefix", installRoot]; if (packageManagerName !== "pnpm") { - args.push("--legacy-peer-deps"); + args.push("--legacy-peer-deps", "--no-audit", "--no-fund"); } await this.runNpmCommand(args); } @@ -1999,20 +2090,31 @@ export class DefaultPackageManager implements PackageManager { } private async cleanAndInstallGitDependencies(targetDir: string, markerPath: string): Promise { - // Clean untracked files (extensions should be pristine). If this fails after - // deleting dependencies, repair them so the existing extension still loads. + // Clean untracked files (extensions should be pristine). On Windows, files loaded by + // another process (e.g. a native module of this extension in another pi session) cannot + // be deleted, so git clean fails. Still install dependencies for the new commit, but keep + // the marker so the next update retries the clean. + let cleaned = true; try { await this.runCommand("git", ["clean", "-fdx"], { cwd: targetDir }); } catch (error) { - await this.repairMissingGitDependencies(targetDir).catch(() => {}); - throw error; + cleaned = false; + const reason = error instanceof Error ? error.message : String(error); + this.emitProgress({ + type: "progress", + action: "update", + source: targetDir, + message: `Could not remove all untracked files in ${targetDir} (${reason}). Files may be in use by another pi session; close it and run the update again.`, + }); } const packageJsonPath = join(targetDir, "package.json"); if (existsSync(packageJsonPath)) { await this.runNpmCommand(this.getGitDependencyInstallArgs(), { cwd: targetDir }); } - rmSync(markerPath, { force: true }); + if (cleaned) { + rmSync(markerPath, { force: true }); + } } private async ensureGitRef(targetDir: string, fetchArgs: string[], ref: string): Promise { @@ -2043,11 +2145,22 @@ export class DefaultPackageManager implements PackageManager { await this.cleanAndInstallGitDependencies(targetDir, markerPath); } - private async refreshTemporaryGitSource(source: GitSource, sourceStr: string): Promise { + private async refreshTemporaryGitSource(source: GitSource, sourceStr: string, installedPath: string): Promise { if (isOfflineModeEnabled()) { return; } try { + // A ref can name a moving branch, so refresh refs too. Skip the network call only when the + // ref is a commit hash that is already checked out. + if (source.ref && /^[0-9a-f]{7,64}$/i.test(source.ref)) { + const head = await this.runCommandCapture("git", ["rev-parse", "HEAD"], { + cwd: installedPath, + timeoutMs: NETWORK_TIMEOUT_MS, + }); + if (head.trim().toLowerCase().startsWith(source.ref.toLowerCase())) { + return; + } + } await this.withProgress("pull", sourceStr, `Refreshing ${sourceStr}...`, async () => { await this.updateGit(source, "temporary"); }); @@ -2243,6 +2356,14 @@ export class DefaultPackageManager implements PackageManager { filter: PackageFilter | undefined, metadata: PathMetadata, ): boolean { + const manifest = readPiManifest(join(packageRoot, "package.json")); + const changelogPath = resolvePackageChangelogPath(packageRoot, manifest); + if (changelogPath) { + metadata.changelogPath = changelogPath; + } else { + delete metadata.changelogPath; + } + if (filter) { for (const resourceType of RESOURCE_TYPES) { const patterns = filter[resourceType]; @@ -2258,10 +2379,9 @@ export class DefaultPackageManager implements PackageManager { return true; } - const manifest = readPiManifest(join(packageRoot, "package.json")); - if (manifest) { + if (manifest && hasManifestResourceFields(manifest)) { for (const resourceType of RESOURCE_TYPES) { - const entries = manifest[resourceType as keyof PiManifest]; + const entries = manifest[resourceType]; this.addManifestEntries( entries, packageRoot, @@ -2295,7 +2415,7 @@ export class DefaultPackageManager implements PackageManager { metadata: PathMetadata, ): void { const manifest = readPiManifest(join(packageRoot, "package.json")); - const entries = manifest?.[resourceType as keyof PiManifest]; + const entries = manifest?.[resourceType]; if (entries) { this.addManifestEntries(entries, packageRoot, resourceType, target, metadata); return; @@ -2364,7 +2484,7 @@ export class DefaultPackageManager implements PackageManager { resourceType: ResourceType, ): { allFiles: string[]; enabledByManifest: Set } { const manifest = readPiManifest(join(packageRoot, "package.json")); - const entries = manifest?.[resourceType as keyof PiManifest]; + const entries = manifest?.[resourceType]; if (entries && entries.length > 0) { const allFiles = this.collectFilesFromManifestEntries(entries, packageRoot, resourceType); const manifestPatterns = entries.filter(isOverridePattern); @@ -2484,7 +2604,7 @@ export class DefaultPackageManager implements PackageManager { const userAgentsSkillsDir = join(getHomeDir(), ".agents", "skills"); const projectTrusted = this.settingsManager.isProjectTrusted(); const projectAgentsSkillDirs = projectTrusted - ? collectAncestorAgentsSkillDirs(this.cwd).filter((dir) => resolve(dir) !== resolve(userAgentsSkillsDir)) + ? collectAncestorAgentsSkillDirs(this.cwd).filter((dir) => !pathsEqual(dir, userAgentsSkillsDir)) : []; const addResources = ( @@ -2673,9 +2793,9 @@ export class DefaultPackageManager implements PackageManager { const seen = new Set(); return resolved.filter((entry) => { - const canonicalPath = canonicalizePath(entry.path); - if (seen.has(canonicalPath)) return false; - seen.add(canonicalPath); + const pathKey = getResourcePathKey(entry.path); + if (seen.has(pathKey)) return false; + seen.add(pathKey); return true; }); }; diff --git a/packages/coding-agent/src/core/pi-ai-schemas.ts b/packages/coding-agent/src/core/pi-ai-schemas.ts new file mode 100644 index 000000000..694aa1fe6 --- /dev/null +++ b/packages/coding-agent/src/core/pi-ai-schemas.ts @@ -0,0 +1,40 @@ +/** + * pi-ai's configuration schemas (built with pi-schema) as TypeBox schemas, for the coding agent's own TypeBox-built + * configuration schemas (models.json, settings.json): the same JSON, typed for TypeBox's `Static`. + */ + +import { + type ProviderCompat, + ProviderCompatSchema as ProviderCompatJsonSchema, +} from "@earendil-works/pi-ai/providers/compat-schema"; +import { + type ModelCost, + ModelCostSchema as ModelCostJsonSchema, + type ModelInputLimits, + ModelInputLimitsSchema as ModelInputLimitsJsonSchema, + type ModelInputModality, + ModelInputModalitySchema as ModelInputModalityJsonSchema, + type ModelPromptCache, + ModelPromptCacheSchema as ModelPromptCacheJsonSchema, + type ModelThinkingLevel, + ModelThinkingLevelSchema as ModelThinkingLevelJsonSchema, + type ThinkingLevelMap, + ThinkingLevelMapSchema as ThinkingLevelMapJsonSchema, +} from "@earendil-works/pi-ai/providers/model-schema"; +import { Type } from "typebox"; + +export const ProviderCompatSchema = Type.Unsafe(ProviderCompatJsonSchema); +export const ModelCostSchema = Type.Unsafe(ModelCostJsonSchema); +export const ModelInputLimitsSchema = Type.Unsafe(ModelInputLimitsJsonSchema); +export const ModelInputModalitySchema = Type.Unsafe(ModelInputModalityJsonSchema); +export const ModelPromptCacheSchema = Type.Unsafe(ModelPromptCacheJsonSchema); +export const ModelThinkingLevelSchema = Type.Unsafe(ModelThinkingLevelJsonSchema); +export const ThinkingLevelMapSchema = Type.Unsafe(ThinkingLevelMapJsonSchema); + +/** `ModelCostSchema` with every field optional, for model overrides that change some of the rates. */ +export const PartialModelCostSchema = Type.Unsafe>(withoutRequired(ModelCostJsonSchema)); + +function withoutRequired(schema: object): object { + const { required: _required, ...rest } = schema as { required?: unknown }; + return rest; +} diff --git a/packages/coding-agent/src/core/pi-manifest.ts b/packages/coding-agent/src/core/pi-manifest.ts index fd7dd5edb..0dca39385 100644 --- a/packages/coding-agent/src/core/pi-manifest.ts +++ b/packages/coding-agent/src/core/pi-manifest.ts @@ -6,6 +6,7 @@ export interface PiManifest { skills?: string[]; prompts?: string[]; themes?: string[]; + changelogPath?: string; } const RESOURCE_FIELDS = ["extensions", "skills", "prompts", "themes"] as const; @@ -28,6 +29,12 @@ export function readPiManifest(packageJsonPath: string): PiManifest | null { manifest[field] = entries; } } + + const changelogPath = pkg.pi.changelogPath; + if (typeof changelogPath === "string") { + manifest.changelogPath = changelogPath; + } + return manifest; } catch { return null; diff --git a/packages/coding-agent/src/core/project-trust.ts b/packages/coding-agent/src/core/project-trust.ts index efb14be4f..3920922b8 100644 --- a/packages/coding-agent/src/core/project-trust.ts +++ b/packages/coding-agent/src/core/project-trust.ts @@ -49,7 +49,7 @@ export async function resolveProjectTrusted(options: ResolveProjectTrustedOption return options.trustOverride; } if (!hasTrustRequiringProjectResources(options.cwd)) { - return true; + return options.trustStore.get(options.cwd) ?? true; } if (options.extensionsResult) { diff --git a/packages/coding-agent/src/core/resource-loader.ts b/packages/coding-agent/src/core/resource-loader.ts index 5957abf27..bb0740e49 100644 --- a/packages/coding-agent/src/core/resource-loader.ts +++ b/packages/coding-agent/src/core/resource-loader.ts @@ -8,7 +8,7 @@ import type { ResourceDiagnostic } from "./diagnostics.ts"; export type { ResourceCollision, ResourceDiagnostic } from "./diagnostics.ts"; -import { canonicalizePath, isLocalPath, resolvePath } from "../utils/paths.ts"; +import { canonicalizePath, isLocalPath, isPathWithin, pathsEqual, resolvePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; import { createEventBus, type EventBus } from "./event-bus.ts"; import { @@ -39,6 +39,7 @@ import { loadSkills } from "./skills.ts"; import { BUILTIN_PATH_PREFIX, createSourceInfo, + getResourcePathKey, getSyntheticPathSource, isSyntheticPath, type SourceInfo, @@ -246,12 +247,12 @@ function findShadowedContextFile(cwd: string): string | undefined { const mainRepoRoot = dirname(commonGitDir); // False for an ordinary repo, where the two are the same dir, and for a sibling // worktree (`git worktree add ../feat`), whose main repo is not an ancestor. - if (!worktreeRoot.startsWith(`${mainRepoRoot}${sep}`)) return undefined; + if (!isPathWithin(mainRepoRoot, worktreeRoot) || pathsEqual(worktreeRoot, mainRepoRoot)) return undefined; // dirname of the common git dir is the main worktree root only when that dir is // itself checked out from the same repo. In a bare layout (`proj/.bare` + // `proj/main`) it is just the directory holding `.bare`, which tracks nothing; a // submodule's gitdir has no `commondir`, so it lands under `.git/modules`. - if (canonicalizePath(join(mainRepoRoot, ".git")) !== commonGitDir) return undefined; + if (!pathsEqual(canonicalizePath(join(mainRepoRoot, ".git")), commonGitDir)) return undefined; const worktreeContextFile = loadContextFileFromDir(worktreeRoot); return worktreeContextFile ? join(mainRepoRoot, basename(worktreeContextFile.path)) : undefined; } @@ -284,7 +285,7 @@ export function loadProjectContextFiles(options: { const isShadowed = shadowedContextFile !== undefined && contextFile !== null && - join(canonicalizePath(currentDir), basename(contextFile.path)) === shadowedContextFile; + pathsEqual(join(canonicalizePath(currentDir), basename(contextFile.path)), shadowedContextFile); if (contextFile && !isShadowed && !seenPaths.has(contextFile.path)) { ancestorContextFiles.unshift(contextFile); seenPaths.add(contextFile.path); @@ -750,6 +751,10 @@ export class DefaultResourceLoader implements ResourceLoader { return isSyntheticPath(path) ? path : resolvePath(path, this.cwd, { normalizeUnicodeSpaces: true }); } + private getExtensionPathKey(path: string): string { + return getResourcePathKey(this.resolveExtensionLoadPath(path)); + } + /** Load extension paths: files from disk and `builtin:` paths from the built-in extensions. */ private async loadExtensionPaths(paths: string[], runtime?: ExtensionRuntime): Promise { const isBuiltinPath = (path: string) => path.startsWith(BUILTIN_PATH_PREFIX); @@ -792,26 +797,26 @@ export class DefaultResourceLoader implements ResourceLoader { const preloadedByPath = new Map( preloaded .filter((extension) => !extension.path.startsWith(" [extension.resolvedPath, extension]), + .map((extension) => [getResourcePathKey(extension.resolvedPath), extension]), ); const failedPreloadPaths = new Set( - (preTrustExtensions?.errors ?? []).map((error) => this.resolveExtensionLoadPath(error.path)), + (preTrustExtensions?.errors ?? []).map((error) => this.getExtensionPathKey(error.path)), ); const remainingPaths = extensionPaths.filter((path) => { - const resolvedPath = this.resolveExtensionLoadPath(path); - return !preloadedByPath.has(resolvedPath) && !failedPreloadPaths.has(resolvedPath); + const pathKey = this.getExtensionPathKey(path); + return !preloadedByPath.has(pathKey) && !failedPreloadPaths.has(pathKey); }); const remainingExtensions = await this.loadExtensionPaths(remainingPaths, preTrustExtensions?.runtime); const loadedByPath = new Map(preloadedByPath); for (const extension of remainingExtensions.extensions) { - loadedByPath.set(extension.resolvedPath, extension); + loadedByPath.set(getResourcePathKey(extension.resolvedPath), extension); } const inlineExtensions = preTrustExtensions ? { extensions: preloaded.filter((extension) => extension.path.startsWith(" loadedByPath.get(this.resolveExtensionLoadPath(path))) + .map((path) => loadedByPath.get(this.getExtensionPathKey(path))) .filter((extension): extension is Extension => extension !== undefined); orderedExtensions.push(...inlineExtensions.extensions); @@ -980,11 +985,7 @@ export class DefaultResourceLoader implements ResourceLoader { const normalizedResourcePath = resolve(resourcePath); if (extraSourceInfos) { for (const [sourcePath, sourceInfo] of extraSourceInfos.entries()) { - const normalizedSourcePath = resolve(sourcePath); - if ( - normalizedResourcePath === normalizedSourcePath || - normalizedResourcePath.startsWith(`${normalizedSourcePath}${sep}`) - ) { + if (isPathWithin(sourcePath, normalizedResourcePath)) { return { ...sourceInfo, path: resourcePath }; } } @@ -997,11 +998,7 @@ export class DefaultResourceLoader implements ResourceLoader { } for (const [sourcePath, metadata] of metadataByPath.entries()) { - const normalizedSourcePath = resolve(sourcePath); - if ( - normalizedResourcePath === normalizedSourcePath || - normalizedResourcePath.startsWith(`${normalizedSourcePath}${sep}`) - ) { + if (isPathWithin(sourcePath, normalizedResourcePath)) { return createSourceInfo(resourcePath, metadata); } } @@ -1062,9 +1059,9 @@ export class DefaultResourceLoader implements ResourceLoader { for (const p of [...primary, ...additional]) { const resolved = this.resolveResourcePath(p); - const canonicalPath = canonicalizePath(resolved); - if (seen.has(canonicalPath)) continue; - seen.add(canonicalPath); + const pathKey = getResourcePathKey(resolved); + if (seen.has(pathKey)) continue; + seen.add(pathKey); merged.push(resolved); } diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 0bf874fff..dbab1d1d9 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -4,7 +4,7 @@ import type { ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { clampThinkingLevel, type Message, type Model, streamSimple } from "@earendil-works/pi-ai/compat"; import { getAgentDir } from "../config.ts"; import { resolvePath } from "../utils/paths.ts"; -import { AgentSession } from "./agent-session.ts"; +import { AgentSession, markRuntimeStreamFunction } from "./agent-session.ts"; import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { CacheWarmer } from "./cache-warmer.ts"; import { getClientIdentity } from "./client-identity.ts"; @@ -436,6 +436,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} thinkingBudgets: settingsManager.getThinkingBudgets(), maxRetryDelayMs: settingsManager.getProviderRetrySettings().maxRetryDelayMs, }); + // This stream resolves auth through modelRuntime, so summaries must not pre-resolve it (#11520). + markRuntimeStreamFunction(agent.streamFunction, modelRuntime); // Restore missing settings metadata for older sessions. if (hasExistingSession) { diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index 57f9655ba..20b6edaa9 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -30,7 +30,7 @@ import { basename, join, resolve } from "path"; import { createInterface } from "readline"; import { StringDecoder } from "string_decoder"; import { APP_NAME, getAgentDir as getDefaultAgentDir, getSessionsDir } from "../config.ts"; -import { normalizePath, resolvePath } from "../utils/paths.ts"; +import { normalizePath, pathsEqual, resolvePath } from "../utils/paths.ts"; import { type BashExecutionMessage, type CustomMessage, @@ -516,7 +516,8 @@ export function buildContextEntries( * If leafId is provided, walks from that entry to root. * Handles compaction and branch summaries along the path. */ -function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { +/** Project one entry into model-visible messages, applying its latest context edit if any. */ +export function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { const messages = sessionEntryToContextMessages(entry); if (!edit) return messages; const replacement = edit.replacement; @@ -742,7 +743,7 @@ function getSessionHeaderCwd(header: SessionHeader): string | undefined { } function sessionCwdMatches(cwd: string | undefined, resolvedCwd: string): boolean { - return cwd !== undefined && cwd !== "" && resolvePath(cwd) === resolvedCwd; + return cwd !== undefined && cwd !== "" && pathsEqual(resolvePath(cwd), resolvedCwd); } /** Exported for testing */ @@ -1146,7 +1147,7 @@ export class SessionManager { } usesDefaultSessionDir(): boolean { - return this.sessionDir === getDefaultSessionDirPath(this.cwd); + return pathsEqual(this.sessionDir, getDefaultSessionDirPath(this.cwd)); } getSessionId(): string { @@ -1189,10 +1190,20 @@ export class SessionManager { } private _appendEntry(entry: SessionEntry): void { + const previousLeafId = this.leafId; this.fileEntries.push(entry); this.byId.set(entry.id, entry); this.leafId = entry.id; - this._persist(entry); + try { + this._persist(entry); + } catch (error) { + // Keep memory in line with the file: an entry that was not written must not become the + // parent of later entries, or the file would hold entries pointing to a missing parent. + this.fileEntries.pop(); + this.byId.delete(entry.id); + this.leafId = previousLeafId; + throw error; + } } /** Append a message as child of current leaf, then advance leaf. Returns entry id. @@ -1792,7 +1803,7 @@ export class SessionManager { */ static continueRecent(cwd: string, sessionDir?: string): SessionManager { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const mostRecent = findMostRecentSession(dir, filterCwd ? cwd : undefined); if (mostRecent) { return new SessionManager(cwd, dir, mostRecent, true); @@ -1878,7 +1889,7 @@ export class SessionManager { */ static findById(cwd: string, id: string, sessionDir?: string): string | undefined { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const resolvedCwd = resolvePath(cwd); try { @@ -1909,7 +1920,7 @@ export class SessionManager { signal?: AbortSignal, ): Promise { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const resolvedCwd = resolvePath(cwd); const includeSession = (session: SessionInfo) => !filterCwd || sessionCwdMatches(session.cwd, resolvedCwd); const progress: SessionListProgress | undefined = onProgress diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index ad8b82c9e..99bbeade0 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -687,6 +687,19 @@ export class SettingsManager { this.save(); } + getExtensionChangelogVersion(key: string): string | undefined { + return this.settings.extensionChangelogVersions?.[key]; + } + + setExtensionChangelogVersion(key: string, version: string): void { + this.globalSettings.extensionChangelogVersions = { + ...(this.globalSettings.extensionChangelogVersions ?? {}), + [key]: version, + }; + this.markModified("extensionChangelogVersions", key); + this.save(); + } + getSessionDir(): string | undefined { const sessionDir = this.settings.sessionDir; return sessionDir ? normalizePath(sessionDir) : sessionDir; diff --git a/packages/coding-agent/src/core/settings-schema.ts b/packages/coding-agent/src/core/settings-schema.ts index b17bab563..1ef797338 100644 --- a/packages/coding-agent/src/core/settings-schema.ts +++ b/packages/coding-agent/src/core/settings-schema.ts @@ -1,5 +1,5 @@ -import { ModelThinkingLevelSchema } from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, type TSchemaOptions, Type } from "typebox"; +import { ModelThinkingLevelSchema } from "./pi-ai-schemas.ts"; import { SETTINGS_DEFAULTS } from "./settings-defaults.ts"; function nonNegativeSafeInteger(options: { default?: number } = {}) { @@ -207,6 +207,11 @@ export const SettingsSchema = Type.Object( { $schema: Type.Optional(Type.String({ description: "JSON Schema reference." })), lastChangelogVersion: Type.Optional(Type.String()), + extensionChangelogVersions: Type.Optional( + Type.Record(Type.String(), Type.String(), { + description: "Last seen changelog version per extension package.", + }), + ), defaultProvider: Type.Optional(Type.String()), defaultModel: Type.Optional(Type.String()), defaultThinkingLevel: Type.Optional(ModelThinkingLevelSchema), @@ -278,6 +283,13 @@ export const SettingsSchema = Type.Object( 'Command used for npm package lookup and installation, in argv form such as ["mise", "exec", "node@20", "--", "npm"].', }), ), + selfUpdateCommand: Type.Optional( + Type.Array(Type.String(), { + minItems: 1, + description: + 'Command `pi update` runs to update Pi itself, in argv form such as ["mise", "use", "-g", "npm:{packageName}@{version}"]. Placeholders: {packageName}, {version}, {installSpec}. PI_SELF_UPDATE_COMMAND takes precedence. Global setting only.', + }), + ), collapseChangelog: Type.Optional( Type.Boolean({ description: "Show the condensed changelog after update; use /changelog for the full changelog.", diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts index 3fbf2b926..48df4536b 100644 --- a/packages/coding-agent/src/core/skills.ts +++ b/packages/coding-agent/src/core/skills.ts @@ -3,9 +3,9 @@ import ignore from "ignore"; import { basename, dirname, join, relative, resolve, sep } from "path"; import { CONFIG_DIR_NAME, getAgentDir } from "../config.ts"; import { parseFrontmatter } from "../utils/frontmatter.ts"; -import { canonicalizePath, resolvePath } from "../utils/paths.ts"; +import { isPathWithin, resolvePath } from "../utils/paths.ts"; import type { ResourceDiagnostic } from "./diagnostics.ts"; -import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.ts"; +import { createSyntheticSourceInfo, getResourcePathKey, type SourceInfo } from "./source-info.ts"; /** Max name length per spec */ const MAX_NAME_LENGTH = 64; @@ -369,7 +369,7 @@ export function formatSkillsForPrompt(skills: Skill[], fileReadTool: "read" | "b : fileReadTool === "bash" ? "Use bash to load a skill's file when the task matches its description." : "Load a skill's file when the task matches its description.", - "When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.", + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", "", "", ]; @@ -427,7 +427,7 @@ export function loadSkills(options: LoadSkillsOptions): LoadSkillsResult { allDiagnostics.push(...result.diagnostics); for (const skill of result.skills) { // Resolve symlinks to detect duplicate files - const realPath = canonicalizePath(skill.filePath); + const realPath = getResourcePathKey(skill.filePath); // Skip silently if we've already loaded this exact file (via symlink) if (realPathSet.has(realPath)) { @@ -462,19 +462,10 @@ export function loadSkills(options: LoadSkillsOptions): LoadSkillsResult { const userSkillsDir = join(resolvedAgentDir, "skills"); const projectSkillsDir = resolve(resolvedCwd, CONFIG_DIR_NAME, "skills"); - const isUnderPath = (target: string, root: string): boolean => { - const normalizedRoot = resolve(root); - if (target === normalizedRoot) { - return true; - } - const prefix = normalizedRoot.endsWith(sep) ? normalizedRoot : `${normalizedRoot}${sep}`; - return target.startsWith(prefix); - }; - const getSource = (resolvedPath: string): "user" | "project" | "path" => { if (!includeDefaults) { - if (isUnderPath(resolvedPath, userSkillsDir)) return "user"; - if (isUnderPath(resolvedPath, projectSkillsDir)) return "project"; + if (isPathWithin(userSkillsDir, resolvedPath)) return "user"; + if (isPathWithin(projectSkillsDir, resolvedPath)) return "project"; } return "path"; }; diff --git a/packages/coding-agent/src/core/slash-commands.ts b/packages/coding-agent/src/core/slash-commands.ts index dc98f1d43..97b2caa6f 100644 --- a/packages/coding-agent/src/core/slash-commands.ts +++ b/packages/coding-agent/src/core/slash-commands.ts @@ -23,7 +23,7 @@ export const BUILTIN_SLASH_COMMANDS: ReadonlyArray = [ { name: "thinking", description: "Set thinking level", argumentHint: "" }, { name: "scoped-models", description: "Enable/disable models for Ctrl+P cycling" }, { name: "export", description: "Export session (HTML default, or specify path: .html/.jsonl)" }, - { name: "import", description: "Import and resume a session from a JSONL file" }, + { name: "import", description: "Import a JSONL session, preserving its ID and working directory" }, { name: "share", description: "Share session via a Radius artifact or a GitHub gist" }, { name: "bug", description: "Report a bug to the Pi developers", argumentHint: "" }, { name: "copy", description: "Copy last agent message to clipboard" }, @@ -38,7 +38,7 @@ export const BUILTIN_SLASH_COMMANDS: ReadonlyArray = [ { name: "logout", description: "Remove provider authentication" }, { name: "new", description: "Start a new session" }, { name: "compact", description: "Manually compact the session context" }, - { name: "resume", description: "Resume a different session" }, + { name: "resume", description: "Resume another session" }, { name: "reload", description: "Reload keybindings, extensions, skills, prompts, themes, and context files" }, { name: "quit", description: `Quit ${APP_NAME}` }, ]; diff --git a/packages/coding-agent/src/core/source-info.ts b/packages/coding-agent/src/core/source-info.ts index 765395d64..716e8c069 100644 --- a/packages/coding-agent/src/core/source-info.ts +++ b/packages/coding-agent/src/core/source-info.ts @@ -1,3 +1,4 @@ +import { canonicalizePath, getPathComparisonKey } from "../utils/paths.ts"; import type { PathMetadata } from "./package-manager.ts"; export type SourceScope = "user" | "project" | "temporary"; @@ -9,6 +10,19 @@ export interface SourceInfo { scope: SourceScope; origin: SourceOrigin; baseDir?: string; + changelogPath?: string; +} + +export interface SourcePackageIdentity { + source: string; + scope: SourceScope; +} + +export function getSourcePackageKey(identity: SourcePackageIdentity): string { + return JSON.stringify({ + source: identity.source, + scope: identity.scope, + }); } /** Prefix of built-in tool and extension paths, such as `builtin:read` or `builtin:mcp`. */ @@ -28,6 +42,11 @@ export function isSyntheticPath(path: string): boolean { return path.startsWith(BUILTIN_PATH_PREFIX) || path.startsWith("<"); } +/** Key for deduplicating resource paths: synthetic paths as-is, file paths via realpath and platform casing. */ +export function getResourcePathKey(path: string): string { + return isSyntheticPath(path) ? path : getPathComparisonKey(canonicalizePath(path)); +} + export function createSourceInfo(path: string, metadata: PathMetadata): SourceInfo { return { path, @@ -35,6 +54,7 @@ export function createSourceInfo(path: string, metadata: PathMetadata): SourceIn scope: metadata.scope, origin: metadata.origin, baseDir: metadata.baseDir, + ...(metadata.changelogPath ? { changelogPath: metadata.changelogPath } : {}), }; } @@ -45,6 +65,7 @@ export function createSyntheticSourceInfo( scope?: SourceScope; origin?: SourceOrigin; baseDir?: string; + changelogPath?: string; }, ): SourceInfo { return { @@ -53,5 +74,6 @@ export function createSyntheticSourceInfo( scope: options.scope ?? "temporary", origin: options.origin ?? "top-level", baseDir: options.baseDir, + ...(options.changelogPath ? { changelogPath: options.changelogPath } : {}), }; } diff --git a/packages/coding-agent/src/core/system-prompt.ts b/packages/coding-agent/src/core/system-prompt.ts index 49bcb58ee..e51e14fe4 100644 --- a/packages/coding-agent/src/core/system-prompt.ts +++ b/packages/coding-agent/src/core/system-prompt.ts @@ -153,20 +153,19 @@ export function buildSystemPromptSections(input: BuildSystemPromptOptions): Syst promptSections.preamble = customPrompt; } else { promptSections.preamble = - "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files."; + "You are a coding assistant running inside pi, a coding agent harness. Help users inspect files, run commands, and modify code."; const visibleTools = declaredTools.filter((name) => !!toolSnippets[name]); const tools = visibleTools.length > 0 ? visibleTools.map((name) => `- ${name}: ${toolSnippets[name]}`).join("\n") : "(none)"; - promptSections.tools = `${tools}\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.`; + promptSections.tools = `${tools}\n\nAdditional tools may be available.`; promptSections.rules = buildRules(declaredTools, toolGuidelines, promptGuidelines); - promptSections.docs = `Pi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI): + promptSections.docs = `Pi documentation (read only when the user asks about the harness itself, its SDK, extensions, themes, skills, or TUI): - Main documentation: ${getReadmePath()} - Additional docs: ${getDocsPath()} - Examples: ${getExamplesPath()} (extensions, custom tools, SDK) - When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory -- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) -- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing -- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)`; +- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) +- Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.`; } if (appendSystemPrompt) promptSections.addendum = appendSystemPrompt; diff --git a/packages/coding-agent/src/core/tools/bash.ts b/packages/coding-agent/src/core/tools/bash.ts index 4620e2464..2084d6f7c 100644 --- a/packages/coding-agent/src/core/tools/bash.ts +++ b/packages/coding-agent/src/core/tools/bash.ts @@ -1,7 +1,6 @@ import { constants } from "node:fs"; import { access as fsAccess } from "node:fs/promises"; import { constants as osConstants } from "node:os"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import { type Static, Type } from "typebox"; import { waitForChildProcess } from "../../utils/child-process.ts"; @@ -16,7 +15,7 @@ import { import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { OutputAccumulator } from "./output-accumulator.ts"; import { BASH_UPDATE_THROTTLE_MS, createShellRenderers } from "./renderers/bash.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "./truncate.ts"; const MAX_TIMEOUT_MS = 2_147_483_647; @@ -431,7 +430,7 @@ export function createBashToolDefinition( return createShellToolDefinition(cwd, bashToolConfig, options); } -export function createBashTool(cwd: string, options?: BashToolOptions): AgentTool { +export function createBashTool(cwd: string, options?: BashToolOptions): TypeBoxAgentTool { const definition = createBashToolDefinition(cwd, options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/src/core/tools/edit.ts b/packages/coding-agent/src/core/tools/edit.ts index 62e6fdee0..bb6aeff59 100644 --- a/packages/coding-agent/src/core/tools/edit.ts +++ b/packages/coding-agent/src/core/tools/edit.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile, writeFile as fsWriteFile } from "fs/promises"; import { type Static, Type } from "typebox"; @@ -16,7 +15,7 @@ import { import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { type EditRenderState, editRenderers } from "./renderers/edit.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const replaceEditSchema = Type.Object( { @@ -43,10 +42,8 @@ const editSchema = Type.Object( export const editToolSystemPromptContribution = { snippet: "Make precise file edits with exact text replacement, including multiple disjoint edits in one call", guidelines: [ - "Use edit for precise changes (edits[].oldText must match exactly)", - "When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls", - "Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.", - "Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.", + "Use edit for targeted changes. Batch changes to one file in a single call.", + "Each edits[].oldText must exactly and uniquely match the original file. Keep matches short and non-overlapping; merge nearby changes without including large unchanged regions.", ], } as const; @@ -215,6 +212,6 @@ export function createEditToolDefinition( }; } -export function createEditTool(cwd: string, options?: EditToolOptions): AgentTool { +export function createEditTool(cwd: string, options?: EditToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createEditToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/find.ts b/packages/coding-agent/src/core/tools/find.ts index 8d44c491e..77ba1b356 100644 --- a/packages/coding-agent/src/core/tools/find.ts +++ b/packages/coding-agent/src/core/tools/find.ts @@ -1,5 +1,4 @@ import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -7,7 +6,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { findRenderers } from "./renderers/find.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; /** Relativize a find result against the search root and normalize it to posix separators. */ @@ -313,6 +312,6 @@ export function createFindToolDefinition( }; } -export function createFindTool(cwd: string, options?: FindToolOptions): AgentTool { +export function createFindTool(cwd: string, options?: FindToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createFindToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/grep.ts b/packages/coding-agent/src/core/tools/grep.ts index a9562c69f..e449e0898 100644 --- a/packages/coding-agent/src/core/tools/grep.ts +++ b/packages/coding-agent/src/core/tools/grep.ts @@ -1,6 +1,5 @@ import { readFile as fsReadFile, stat as fsStat } from "node:fs/promises"; import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -8,7 +7,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveToCwd } from "./path-utils.ts"; import { grepRenderers } from "./renderers/grep.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, @@ -150,7 +149,7 @@ export function createGrepToolDefinition( if (!lines) { try { const content = await ops.readFile(filePath); - lines = content.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); + lines = content.replace(/\r\n/g, "\n").split("\n"); } catch { lines = []; } @@ -320,6 +319,6 @@ export function createGrepToolDefinition( }; } -export function createGrepTool(cwd: string, options?: GrepToolOptions): AgentTool { +export function createGrepTool(cwd: string, options?: GrepToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createGrepToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/ls.ts b/packages/coding-agent/src/core/tools/ls.ts index 558ff9f9f..4f5efc8a7 100644 --- a/packages/coding-agent/src/core/tools/ls.ts +++ b/packages/coding-agent/src/core/tools/ls.ts @@ -1,11 +1,10 @@ import { readdir as fsReaddir, stat as fsStat } from "node:fs/promises"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import nodePath from "path"; import { type Static, Type } from "typebox"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { lsRenderers } from "./renderers/ls.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const lsSchema = Type.Object({ @@ -170,6 +169,6 @@ export function createLsToolDefinition( }; } -export function createLsTool(cwd: string, options?: LsToolOptions): AgentTool { +export function createLsTool(cwd: string, options?: LsToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createLsToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index ebe2c2517..8fb93a679 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; @@ -8,7 +7,7 @@ import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveReadPathAsync } from "./path-utils.ts"; import { readRenderers } from "./renderers/read.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const readSchema = Type.Object({ @@ -219,6 +218,6 @@ export function createReadToolDefinition( }; } -export function createReadTool(cwd: string, options?: ReadToolOptions): AgentTool { +export function createReadTool(cwd: string, options?: ReadToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createReadToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/render-utils.ts b/packages/coding-agent/src/core/tools/render-utils.ts index c0fca375d..1bde70bc3 100644 --- a/packages/coding-agent/src/core/tools/render-utils.ts +++ b/packages/coding-agent/src/core/tools/render-utils.ts @@ -1,20 +1,15 @@ -import * as os from "node:os"; import { pathToFileURL } from "node:url"; import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import { getCapabilities, getImageDimensions, hyperlink, imageFallback } from "@earendil-works/pi-tui"; import type { Theme } from "../../modes/interactive/theme/theme.ts"; import { stripAnsi } from "../../utils/ansi.ts"; -import { resolvePath } from "../../utils/paths.ts"; +import { resolvePath, shortenHomePath } from "../../utils/paths.ts"; import { sanitizeBinaryOutput } from "../../utils/shell.ts"; import type { ToolRenderContext } from "../extensions/types.ts"; export function shortenPath(path: unknown): string { if (typeof path !== "string") return ""; - const home = os.homedir(); - if (path.startsWith(home)) { - return `~${path.slice(home.length)}`; - } - return path; + return shortenHomePath(path); } export function linkPath(styledText: string, rawPath: string, cwd: string): string { diff --git a/packages/coding-agent/src/core/tools/renderers/write.ts b/packages/coding-agent/src/core/tools/renderers/write.ts index 7c8f32636..8e2338a7d 100644 --- a/packages/coding-agent/src/core/tools/renderers/write.ts +++ b/packages/coding-agent/src/core/tools/renderers/write.ts @@ -8,7 +8,12 @@ import { Container, Text } from "@earendil-works/pi-tui"; import { keyHint } from "../../../modes/interactive/components/keybinding-hints.ts"; -import { getLanguageFromPath, highlightCode, type Theme } from "../../../modes/interactive/theme/theme.ts"; +import { + CodeHighlighter, + getLanguageFromPath, + highlightCode, + type Theme, +} from "../../../modes/interactive/theme/theme.ts"; import type { ToolDefinition, ToolRenderResultOptions } from "../../extensions/types.ts"; import { normalizeDisplayText, renderToolPath, replaceTabs, str } from "../render-utils.ts"; @@ -20,7 +25,12 @@ type WriteHighlightCache = { highlightedLines: string[]; }; class WriteCallRenderComponent extends Text { + // Line by line highlighting of a call whose content is still streaming. cache?: WriteHighlightCache; + // A streamed call highlights its first lines on every delta, which stop changing once the file is longer, and a + // complete call highlights the whole file on every update and expand toggle. Either way the row reuses the last + // highlighting; it also holds the only copy of a complete call's displayed source. + readonly highlighter = new CodeHighlighter(); constructor() { super("", 0, 0); @@ -31,11 +41,11 @@ function highlightSingleLine(line: string, lang: string): string { const highlighted = highlightCode(line, lang); return highlighted[0] ?? ""; } -function refreshWriteHighlightPrefix(cache: WriteHighlightCache): void { +function refreshWriteHighlightPrefix(cache: WriteHighlightCache, highlighter: CodeHighlighter): void { const prefixCount = Math.min(WRITE_PARTIAL_FULL_HIGHLIGHT_LINES, cache.normalizedLines.length); if (prefixCount === 0) return; const prefixSource = cache.normalizedLines.slice(0, prefixCount).join("\n"); - const prefixHighlighted = highlightCode(prefixSource, cache.lang); + const prefixHighlighted = highlighter.highlight(prefixSource, cache.lang); for (let i = 0; i < prefixCount; i++) { cache.highlightedLines[i] = prefixHighlighted[i] ?? highlightSingleLine(cache.normalizedLines[i] ?? "", cache.lang); @@ -58,12 +68,14 @@ function updateWriteHighlightCacheIncremental( cache: WriteHighlightCache | undefined, rawPath: string | null, fileContent: string, + highlighter: CodeHighlighter, ): WriteHighlightCache | undefined { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; if (!lang) return undefined; if (!cache) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (cache.lang !== lang || cache.rawPath !== rawPath) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (!fileContent.startsWith(cache.rawContent)) return rebuildWriteHighlightCacheFull(rawPath, fileContent); + if (cache.lang !== lang || cache.rawPath !== rawPath || !fileContent.startsWith(cache.rawContent)) { + return rebuildWriteHighlightCacheFull(rawPath, fileContent); + } if (fileContent.length === cache.rawContent.length) return cache; const deltaRaw = fileContent.slice(cache.rawContent.length); @@ -83,10 +95,10 @@ function updateWriteHighlightCacheIncremental( cache.normalizedLines.push(segments[i]); cache.highlightedLines.push(highlightSingleLine(segments[i], cache.lang)); } - refreshWriteHighlightPrefix(cache); + refreshWriteHighlightPrefix(cache, highlighter); return cache; } -function trimTrailingEmptyLines(lines: string[]): string[] { +function trimTrailingEmptyLines(lines: readonly string[]): string[] { let end = lines.length; while (end > 0 && lines[end - 1] === "") { end--; @@ -98,6 +110,7 @@ function formatWriteCall( options: ToolRenderResultOptions, theme: Theme, cache: WriteHighlightCache | undefined, + highlighter: CodeHighlighter, cwd: string, ): string { const rawPath = str(args?.file_path ?? args?.path); @@ -110,7 +123,7 @@ function formatWriteCall( } else if (fileContent) { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; const renderedLines = lang - ? (cache?.highlightedLines ?? highlightCode(replaceTabs(normalizeDisplayText(fileContent)), lang)) + ? (cache?.highlightedLines ?? highlighter.highlight(replaceTabs(normalizeDisplayText(fileContent)), lang)) : normalizeDisplayText(fileContent).split("\n"); const lines = trimTrailingEmptyLines(renderedLines); const totalLines = lines.length; @@ -149,19 +162,17 @@ export const writeRenderers: Pick, "renderCall" | "rend const fileContent = str(renderArgs?.content); const component = (context.lastComponent as WriteCallRenderComponent | undefined) ?? new WriteCallRenderComponent(); - if (fileContent !== null) { - component.cache = context.argsComplete - ? rebuildWriteHighlightCacheFull(rawPath, fileContent) - : updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent); - } else { - component.cache = undefined; - } + component.cache = + fileContent !== null && !context.argsComplete + ? updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent, component.highlighter) + : undefined; component.setText( formatWriteCall( renderArgs, { expanded: context.expanded, isPartial: context.isPartial }, theme, component.cache, + component.highlighter, context.cwd, ), ); diff --git a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts index 9a9465dec..700dfd35c 100644 --- a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts +++ b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts @@ -1,6 +1,25 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { AgentTool, AgentToolResult, AgentToolUpdateCallback } from "@earendil-works/pi-agent-core"; +import type { Static, TSchema } from "typebox"; import type { ExtensionToolContext, ToolDefinition } from "../extensions/types.ts"; +/** + * An `AgentTool` with a TypeBox parameter schema whose `execute` and `prepareArguments` are typed by TypeBox's + * `Static`, as the built-in tools are (pi-agent-core types `params` through pi-schema, which maps only the common + * TypeBox kinds). Spreads into a `ToolDefinition` of the same schema. + */ +export type TypeBoxAgentTool = Omit< + AgentTool, + "prepareArguments" | "execute" +> & { + prepareArguments?: (args: unknown) => Static; + execute: ( + toolCallId: string, + params: Static, + signal?: AbortSignal, + onUpdate?: AgentToolUpdateCallback, + ) => Promise>; +}; + /** Creates the context for one tool call. */ export type ToolContextFactory = (toolCallId: string, signal: AbortSignal | undefined) => ExtensionToolContext; diff --git a/packages/coding-agent/src/core/tools/write.ts b/packages/coding-agent/src/core/tools/write.ts index 374089017..9e6b296dc 100644 --- a/packages/coding-agent/src/core/tools/write.ts +++ b/packages/coding-agent/src/core/tools/write.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { mkdir as fsMkdir, writeFile as fsWriteFile } from "fs/promises"; import { dirname } from "path"; import { type Static, Type } from "typebox"; @@ -6,7 +5,7 @@ import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { writeRenderers } from "./renderers/write.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const writeSchema = Type.Object({ path: Type.String({ description: "Path to the file to write (relative or absolute)" }), @@ -92,6 +91,6 @@ export function createWriteToolDefinition( }; } -export function createWriteTool(cwd: string, options?: WriteToolOptions): AgentTool { +export function createWriteTool(cwd: string, options?: WriteToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createWriteToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/trust-manager.ts b/packages/coding-agent/src/core/trust-manager.ts index 3396c9a39..f0721efef 100644 --- a/packages/coding-agent/src/core/trust-manager.ts +++ b/packages/coding-agent/src/core/trust-manager.ts @@ -3,7 +3,7 @@ import { homedir } from "node:os"; import { dirname, join } from "node:path"; import lockfile from "proper-lockfile"; import { CONFIG_DIR_NAME } from "../config.ts"; -import { canonicalizePath, resolvePath } from "../utils/paths.ts"; +import { canonicalizePath, pathsEqual, resolvePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; export type ProjectTrustDecision = boolean | null; @@ -42,12 +42,19 @@ function normalizeCwd(cwd: string): string { return canonicalizePath(resolvePath(cwd)); } +/** Trust file keys naming `path`. Usually one; several on Windows when saved with different casing. */ +function findTrustFileKeys(data: TrustFile, path: string): string[] { + return Object.keys(data).filter((key) => pathsEqual(key, path)); +} + function findNearestTrustEntry(data: TrustFile, cwd: string): ProjectTrustStoreEntry | null { let currentDir = normalizeCwd(cwd); while (true) { - const value = data[currentDir]; - if (value === true || value === false) { - return { path: currentDir, decision: value }; + for (const key of findTrustFileKeys(data, currentDir)) { + const value = data[key]; + if (value === true || value === false) { + return { path: key, decision: value }; + } } const parentDir = dirname(currentDir); @@ -195,7 +202,7 @@ export function hasTrustRequiringProjectResources(cwd: string): boolean { while (true) { const agentsSkillsDir = join(currentDir, ".agents", "skills"); - if (agentsSkillsDir !== userAgentsSkillsDir && existsSync(agentsSkillsDir)) { + if (!pathsEqual(agentsSkillsDir, userAgentsSkillsDir) && existsSync(agentsSkillsDir)) { return true; } @@ -234,9 +241,10 @@ export class ProjectTrustStore { const data = readTrustFile(this.trustPath); for (const { path, decision } of decisions) { const key = normalizeCwd(path); - if (decision === null) { - delete data[key]; - } else { + for (const existingKey of findTrustFileKeys(data, key)) { + delete data[existingKey]; + } + if (decision !== null) { data[key] = decision; } } diff --git a/packages/coding-agent/src/extensions/codemode/renderer.ts b/packages/coding-agent/src/extensions/codemode/renderer.ts index 8d2937351..660803e68 100644 --- a/packages/coding-agent/src/extensions/codemode/renderer.ts +++ b/packages/coding-agent/src/extensions/codemode/renderer.ts @@ -12,7 +12,7 @@ import type { ToolDefinition } from "../../core/extensions/types.ts"; import { getTextOutput, replaceTabs, startToolTimer, str, toolTimerText } from "../../core/tools/render-utils.ts"; import { keyHint } from "../../modes/interactive/components/keybinding-hints.ts"; import { VisualLinePreview } from "../../modes/interactive/components/visual-truncate.ts"; -import { highlightCode, type Theme } from "../../modes/interactive/theme/theme.ts"; +import { CodeHighlighter, type Theme } from "../../modes/interactive/theme/theme.ts"; import type { CodemodeNestedCall, CodemodeToolDetails } from "./tool.ts"; const CODE_PREVIEW_LINES = 10; @@ -21,6 +21,12 @@ const OUTPUT_PREVIEW_LINES = 5; const COLLAPSED_ARGS_CHARS = 80; const SCRIPT_HEADER = /^Script (completed|failed)\nWall time [\d.]+ seconds\nOutput:\n$/; +/** Renderer state of a codemode row. */ +interface CodemodeRenderState { + // Progress, timer ticks and expanding redraw the call, so the row keeps the script's highlighting. + scriptHighlighter?: CodeHighlighter; +} + function expandHint(theme: Theme, hidden: number, noun: string): string { return `${theme.fg("muted", `... (${hidden} more ${noun},`)} ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`; } @@ -79,7 +85,11 @@ export const codemodeRenderers: Pick< } component.addChild(new Text(title, 0, 0)); if (code) { - const highlighted = highlightCode(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript").join("\n"); + const state = context.state as CodemodeRenderState; + state.scriptHighlighter ??= new CodeHighlighter(); + const highlighted = state.scriptHighlighter + .highlight(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript") + .join("\n"); component.addChild( context.expanded ? new Text(highlighted, 0, 0) diff --git a/packages/coding-agent/src/extensions/codemode/tool.ts b/packages/coding-agent/src/extensions/codemode/tool.ts index b55a117a7..648ba6109 100644 --- a/packages/coding-agent/src/extensions/codemode/tool.ts +++ b/packages/coding-agent/src/extensions/codemode/tool.ts @@ -32,7 +32,7 @@ import type { ToolDefinition, ToolLoadout, ToolLoadoutChanges, ToolNamespace } f import type { ModelRegistry } from "../../core/model-registry.ts"; import { SETTINGS_DEFAULTS } from "../../core/settings-defaults.ts"; import type { CodemodeMode } from "../../core/settings-manager.ts"; -import { wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; import { loadCodemodeExecutor } from "./execute.lazy.ts"; import { codemodeRenderers } from "./renderer.ts"; @@ -271,8 +271,8 @@ export function createCodemodeDescription( /** * The field names of an object schema, `{ output, exit_code, full_output_path? }`, with the fields of - * object-typed fields `depth` levels deep: `{ content, structuredContent?: { results } }`. Undefined - * for other schemas. + * object-typed fields `depth` levels deep: `{ content, structuredContent?: { results } }`, and the + * value of `const` fields: `{ action: "start", id }`. Undefined for other schemas. */ function describeFields(schema: unknown, depth: number): string | undefined { if (typeof schema !== "object" || schema === null) return undefined; @@ -280,7 +280,13 @@ function describeFields(schema: unknown, depth: number): string | undefined { if (type !== "object" || typeof properties !== "object" || properties === null) return undefined; const requiredNames = new Set(Array.isArray(required) ? required : []); const fields = Object.entries(properties).map(([name, field]) => { - const nested = depth > 0 ? describeFields(field, depth - 1) : undefined; + const constValue = (field as Record | null)?.const; + const nested = + constValue !== undefined + ? JSON.stringify(constValue) + : depth > 0 + ? describeFields(field, depth - 1) + : undefined; return `${name}${requiredNames.has(name) ? "" : "?"}${nested ? `: ${nested}` : ""}`; }); return `{ ${fields.join(", ")} }`; @@ -288,13 +294,30 @@ function describeFields(schema: unknown, depth: number): string | undefined { /** * What a script call resolves to, in one line: `a string`, the field names of an object (see - * {@link describeFields}), or the rendered type for anything else. + * {@link describeFields}), the field names of each object in a union, the rendered type when it + * fits on one line, or a pointer to `describeTool()` for anything longer. The full type of a large + * schema would otherwise be sent with every request (#11501). */ -function describeOutput(schema: CodemodeJsonSchema | undefined): string { +function describeOutput(name: string, schema: CodemodeJsonSchema | undefined): string { const type = renderToolOutputType(schema); if (type === "string") return "a string"; - const fields = describeFields(schema, 1); - return `\`${fields ?? type.replace(/\s+/g, " ")}\``; + const fields = describeFields(schema, 1) ?? describeUnionFields(schema); + if (fields) return `\`${fields}\``; + if (!type.includes("\n")) return `\`${type}\``; + return `a value; \`await describeTool(${JSON.stringify(name)})\` shows its type`; +} + +/** + * The field names of each variant of an `anyOf`/`oneOf` whose variants are all objects, joined by + * `|`. + */ +function describeUnionFields(schema: unknown): string | undefined { + if (typeof schema !== "object" || schema === null) return undefined; + const { anyOf, oneOf } = schema as Record; + const variants = anyOf ?? oneOf; + if (!Array.isArray(variants) || variants.length === 0) return undefined; + const described = variants.map((variant) => describeFields(variant, 0)); + return described.every((fields) => fields !== undefined) ? described.join(" | ") : undefined; } /** @@ -302,7 +325,7 @@ function describeOutput(schema: CodemodeJsonSchema | undefined): string { * arguments are the tool's declared parameters, so they are not repeated. */ function describeScriptCall(tool: AgentTool): string { - return `${tool.description.trim()}\n\nCodemode: \`tools.${toCodemodeIdentifier(tool.name)}(args)\` resolves to ${describeOutput(toCodemodeDeclaration(tool).outputSchema)}.`; + return `${tool.description.trim()}\n\nCodemode: \`tools.${toCodemodeIdentifier(tool.name)}(args)\` resolves to ${describeOutput(tool.name, toCodemodeDeclaration(tool).outputSchema)}.`; } /** @@ -399,7 +422,7 @@ export function createCodemodeToolDefinition( export function createCodemodeTool( tools: readonly AgentTool[] = [], options: CodemodeToolOptions = {}, -): AgentTool { +): TypeBoxAgentTool { const definition = createCodemodeToolDefinition(options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/src/extensions/llama/ui.ts b/packages/coding-agent/src/extensions/llama/ui.ts index 694921bde..25f64c07f 100644 --- a/packages/coding-agent/src/extensions/llama/ui.ts +++ b/packages/coding-agent/src/extensions/llama/ui.ts @@ -3,6 +3,7 @@ import { Container, type Focusable, fuzzyFilter, + getSelectPageIndex, Input, type SelectItem, SelectList, @@ -21,9 +22,12 @@ import type { LlamaModelInfo, LlamaProgress } from "./client.ts"; import type { HuggingFaceModel } from "./huggingface.ts"; const DOWNLOAD_VALUE = "\0download"; +const SEARCH_MAX_VISIBLE = 10; export type LlamaManagerAction = { type: "model"; model: LlamaModelInfo } | { type: "download" } | { type: "close" }; +type InputHandler = Pick; + interface ProgressState extends LlamaProgress { title: string; model: string; @@ -94,6 +98,7 @@ function compactCount(value: number): string { } class HuggingFaceSearch extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private readonly tui: TUI; private readonly theme: Theme; private readonly keybindings: KeybindingsManager; @@ -145,12 +150,14 @@ class HuggingFaceSearch extends Container implements Focusable { private updateResults(): void { this.resultsContainer.clear(); - const maxVisible = 10; const start = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredResults.length - maxVisible), + Math.min( + this.selectedIndex - Math.floor(SEARCH_MAX_VISIBLE / 2), + this.filteredResults.length - SEARCH_MAX_VISIBLE, + ), ); - const end = Math.min(start + maxVisible, this.filteredResults.length); + const end = Math.min(start + SEARCH_MAX_VISIBLE, this.filteredResults.length); for (let index = start; index < end; index++) { const model = this.filteredResults[index]; if (!model) continue; @@ -255,6 +262,20 @@ class HuggingFaceSearch extends Container implements Focusable { } return; } + const pageIndex = getSelectPageIndex( + this.keybindings, + data, + this.selectedIndex, + this.filteredResults.length, + SEARCH_MAX_VISIBLE, + ); + if (pageIndex !== undefined) { + if (this.filteredResults.length > 0) { + this.selectedIndex = pageIndex; + this.updateResults(); + } + return; + } if (this.keybindings.matches(data, "tui.select.confirm")) { const exact = /^[^/\s]+\/[^:\s]+(?::[^\s:]+)?$/u.test(this.query) ? this.query : undefined; const selected = exact ?? this.filteredResults[this.selectedIndex]?.id; @@ -279,7 +300,7 @@ class LlamaView implements LlamaUi, Focusable { private readonly keybindings: KeybindingsManager; private readonly searchCache = new Map(); private content: Container; - private inputHandler: { handleInput?(data: string): void } | undefined; + private inputHandler: InputHandler | undefined; private inputTarget: Focusable | undefined; private progressPromise: Promise | undefined; private progressResolver: (() => void) | undefined; @@ -302,11 +323,11 @@ class LlamaView implements LlamaUi, Focusable { if (this.inputTarget) this.inputTarget.focused = value; } - private setContent( - content: Container, - inputHandler?: { handleInput?(data: string): void }, - inputTarget?: Focusable, - ): void { + get capturesViewportInput(): boolean { + return this.inputHandler?.capturesViewportInput === true; + } + + private setContent(content: Container, inputHandler?: InputHandler, inputTarget?: Focusable): void { if (this.inputTarget) this.inputTarget.focused = false; this.progressPromise = undefined; this.progressResolver = undefined; diff --git a/packages/coding-agent/src/extensions/mcp/cli.ts b/packages/coding-agent/src/extensions/mcp/cli.ts index 75cdf1f08..81d28ad66 100644 --- a/packages/coding-agent/src/extensions/mcp/cli.ts +++ b/packages/coding-agent/src/extensions/mcp/cli.ts @@ -35,8 +35,8 @@ const HELP = `${chalk.bold("Usage:")} ${APP_NAME} mcp add [options] --url ${APP_NAME} mcp remove [-l] ${APP_NAME} mcp list [--json] - ${APP_NAME} mcp login [--timeout ] - ${APP_NAME} mcp logout + ${APP_NAME} mcp login [-l] [--timeout ] + ${APP_NAME} mcp logout [-l] Configure and check MCP servers and sign in to OAuth servers without starting a session. Reads ~/${CONFIG_DIR_NAME}/agent/mcp.json and, in trusted projects, ${CONFIG_DIR_NAME}/mcp.json. @@ -48,7 +48,7 @@ Commands: login Sign in through the browser logout Delete the stored OAuth credentials -Options for add and remove: +Options for add, remove, login, and logout: -l, --local Use ${CONFIG_DIR_NAME}/mcp.json in the current project instead of the global file Options for add: @@ -65,6 +65,8 @@ Options for add: Fixed OAuth callback port --oauth-client-name Client name sent when registering with the OAuth server + --oauth-client-registration + auto (default), cimd, or dcr --exposure codemode (default), deferred, direct, or hidden --description What the server offers, shown in the system prompt @@ -151,7 +153,8 @@ function parseOptions( const values = new Map(); const lists = new Map(); for (let index = 0; index < args.length; index++) { - const arg = OPTION_ALIASES.get(args[index]) ?? args[index]; + const typed = args[index]; + const arg = OPTION_ALIASES.get(typed) ?? typed; if (arg === "--" || positional.length >= maxPositionals) { positional.push(...args.slice(arg === "--" ? index + 1 : index)); break; @@ -163,7 +166,7 @@ function parseOptions( const name = arg.slice(2); const kind = known[name]; if (!kind) { - error(`Unknown option ${arg}.\n${HELP_HINT}`); + error(`Unknown option ${typed}.\n${HELP_HINT}`); return undefined; } if (kind === "flag") { @@ -217,20 +220,30 @@ export async function runMcpCommand(args: string[], options: McpCommandOptions): } case "login": case "logout": { - const parsed = parseOptions(rest, command === "login" ? { timeout: "value" } : {}, error); + const parsed = parseOptions( + rest, + command === "login" ? { local: "flag", timeout: "value" } : { local: "flag" }, + error, + ); if (!parsed) return 1; const [name, ...extra] = parsed.positional; if (!name || extra.length > 0) { - error(`Usage: ${APP_NAME} mcp ${command} \n${HELP_HINT}`); + error(`Usage: ${APP_NAME} mcp ${command} [-l]\n${HELP_HINT}`); return 1; } const entry = loaded.servers.find((server) => server.name === name); if (!entry) { + const names = loaded.servers.map((server) => server.name); + const suggestion = names.find((candidate) => editDistance(candidate, name) <= 2); error( - `No MCP server named "${name}".${untrustedNote ? ` ${untrustedNote}` : ""} Configured: ${loaded.servers.map((server) => server.name).join(", ") || "none"}.`, + `No MCP server named "${name}".${suggestion ? ` Did you mean "${suggestion}"?` : ""}${untrustedNote ? ` ${untrustedNote}` : ""} Configured: ${names.join(", ") || "none"}.`, ); return 1; } + if (parsed.values.has("local") && entry.scope !== "project") { + error(`MCP server "${name}" is defined in ${entry.source}, not in the project; omit --local.`); + return 1; + } const connection = createConnection(entry, options, credentials); const url = connection.oauthUrl; if (!url) { @@ -259,6 +272,19 @@ export async function runMcpCommand(args: string[], options: McpCommandOptions): } } +/** Levenshtein distance, to suggest a configured server name for a mistyped one. */ +function editDistance(a: string, b: string): number { + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) { + current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + } + previous = current; + } + return previous[b.length]; +} + /** Parse `KEY=VALUE` pairs of a repeatable option into a record. */ function parsePairs(option: string, pairs: string[] | undefined, error: (line: string) => void) { if (!pairs) return {}; @@ -295,6 +321,7 @@ function add( "oauth-client-secret": "value", "oauth-callback-port": "value", "oauth-client-name": "value", + "oauth-client-registration": "value", exposure: "value", description: "value", }, @@ -321,6 +348,7 @@ function add( "oauth-client-secret", "oauth-callback-port", "oauth-client-name", + "oauth-client-registration", ]; const stdioOnly = ["env", "cwd"]; const misplaced = (url === undefined ? httpOnly : stdioOnly).find( @@ -343,6 +371,9 @@ function add( ...(value("oauth-client-secret") === undefined ? {} : { clientSecret: value("oauth-client-secret") }), ...(port === undefined ? {} : { callbackPort: Number(port) }), ...(value("oauth-client-name") === undefined ? {} : { clientName: value("oauth-client-name") }), + ...(value("oauth-client-registration") === undefined + ? {} + : { clientRegistration: value("oauth-client-registration") }), }; config = { url, diff --git a/packages/coding-agent/src/extensions/mcp/oauth.ts b/packages/coding-agent/src/extensions/mcp/oauth.ts index b1ca5601a..97fb56793 100644 --- a/packages/coding-agent/src/extensions/mcp/oauth.ts +++ b/packages/coding-agent/src/extensions/mcp/oauth.ts @@ -34,7 +34,7 @@ import { import lockfile from "proper-lockfile"; import { APP_NAME, getAgentDir } from "../../config.ts"; import { type AuthStorageBackend, FileAuthStorageBackend } from "../../core/auth-storage.ts"; -import { mcpNamespace } from "../../core/mcp-servers.ts"; +import { isClientMetadataDocumentCallback, mcpNamespace } from "../../core/mcp-servers.ts"; const CALLBACK_HOST = "127.0.0.1"; const CALLBACK_PATH = "/callback"; @@ -67,7 +67,7 @@ export interface McpOAuthSettings { /** `client_name` for dynamic client registration. Default: `APP_NAME`. */ clientName?: string; /** See `McpOAuthConfig.clientRegistration`. */ - clientRegistration?: "dcr" | "cimd"; + clientRegistration?: "auto" | "dcr" | "cimd"; /** See `McpOAuthConfig.authServerMetadataUrl`. */ authServerMetadataUrl?: URL; } @@ -243,23 +243,33 @@ function callbackId(serverUrl: string): string { } /** - * pi's Client ID Metadata Document, for `clientRegistration: "cimd"`, chosen like Codex chooses its own. - * The configuration ensures the default callback path. Without the `iss` parameter in authorization - * responses (RFC 9207), the redirect URI and the document are specific to the MCP server, so a response - * cannot be mixed up with one from another authorization server (RFC 9700 section 4.4.2.2). + * Whether sign-in may identify with pi's Client ID Metadata Document instead of registering. `auto` + * registers when the user asked for a client name, the redirect URI is not one the documents list, or + * a rebranded build would show up as pi. + */ +function offersClientMetadataDocument(settings: McpOAuthSettings): boolean { + const registration = settings.clientRegistration ?? "auto"; + if (registration !== "auto") return registration === "cimd"; + if (settings.clientId || settings.clientName || APP_NAME !== "pi") return false; + return isClientMetadataDocumentCallback(settings.callbackUrl); +} + +/** + * pi's Client ID Metadata Document, chosen like Codex chooses its own, or `undefined` when the + * authorization server does not support documents for public clients. Without the `iss` parameter in + * authorization responses (RFC 9207), the redirect URI and the document are specific to the MCP server, + * so a response cannot be mixed up with one from another authorization server (RFC 9700 section 4.4.2.2). */ function clientMetadataDocument( serverUrl: string, redirectUrl: string, metadata: AuthorizationServerMetadata | undefined, -): OAuthClientMetadataDocument { +): OAuthClientMetadataDocument | undefined { if ( !metadata?.client_id_metadata_document_supported || !metadata.token_endpoint_auth_methods_supported?.includes("none") ) { - throw new Error( - 'The authorization server does not support Client ID Metadata Documents for public clients; remove oauth.clientRegistration "cimd"', - ); + return undefined; } if (metadata.authorization_response_iss_parameter_supported) { return { url: `${CLIENT_METADATA_BASE_URL}/client.json`, redirectUrl }; @@ -281,10 +291,17 @@ function createProvider( serverUrl, redirectUrl, clientMetadata: { client_name: settings.clientName ?? APP_NAME }, - clientMetadataDocument: - settings.clientRegistration === "cimd" - ? (metadata) => clientMetadataDocument(serverUrl, redirectUrl, metadata) - : undefined, + clientMetadataDocument: offersClientMetadataDocument(settings) + ? (metadata) => { + const document = clientMetadataDocument(serverUrl, redirectUrl, metadata); + if (!document && settings.clientRegistration === "cimd") { + throw new Error( + 'The authorization server does not support Client ID Metadata Documents for public clients; remove oauth.clientRegistration "cimd"', + ); + } + return document; + } + : undefined, clientId: settings.clientId, clientSecret: settings.clientSecret, store, @@ -487,11 +504,11 @@ export async function signInMcpServer(options: { const registered = registeredRedirectUrls(stored?.clientInformation)[0]; const preferredPort = callbackOptions.port ?? (registered ? Number(new URL(registered).port) || undefined : undefined); - const cimd = settings.clientRegistration === "cimd"; + const offersDocument = offersClientMetadataDocument(settings); const callback = await listenForCallback( callbackOptions, // The redirect URI of a server-specific Client ID Metadata Document. - cimd ? [`${CALLBACK_PATH}/${callbackId(serverUrl)}`] : [], + offersDocument ? [`${CALLBACK_PATH}/${callbackId(serverUrl)}`] : [], preferredPort, callbackOptions.port !== undefined, ); @@ -501,11 +518,15 @@ export async function signInMcpServer(options: { const next: McpOAuthState = { ...stored }; // Every sign-in gets a fresh `state` parameter. delete next.oauthState; - // A registered client cannot use another redirect URI, and its tokens belong to it. A Client ID - // Metadata Document is not stored, so with one, a stored client was registered before and is replaced. + // A registered client cannot use another redirect URI, and its tokens belong to it. `cimd` replaces a + // registered client; `auto` keeps it. A Client ID Metadata Document is not stored, so tokens without + // a stored client belong to the document. const keepClient = settings.clientId || - (cimd ? !stored.clientInformation : registeredRedirectUrls(stored.clientInformation).includes(redirectUrl)); + (stored.clientInformation + ? settings.clientRegistration !== "cimd" && + registeredRedirectUrls(stored.clientInformation).includes(redirectUrl) + : offersDocument); if (!keepClient) { delete next.clientInformation; delete next.tokens; diff --git a/packages/coding-agent/src/extensions/mcp/ui.ts b/packages/coding-agent/src/extensions/mcp/ui.ts index 771e76b61..38bdfcdae 100644 --- a/packages/coding-agent/src/extensions/mcp/ui.ts +++ b/packages/coding-agent/src/extensions/mcp/ui.ts @@ -70,12 +70,14 @@ function frame(theme: Theme, title: string, body: Component[], footer?: string): const MAX_VISIBLE_ITEMS = 12; +type InputHandler = Pick; + export class McpManagerView implements McpUi, Component, Focusable { private readonly tui: TUI; private readonly theme: Theme; private readonly keybindings: KeybindingsManager; private content: Container; - private inputHandler: ((data: string) => void) | undefined; + private inputHandler: InputHandler | undefined; private inputTarget: Focusable | undefined; private _focused = false; @@ -95,7 +97,11 @@ export class McpManagerView implements McpUi, Component, Focusable { if (this.inputTarget) this.inputTarget.focused = value; } - private setContent(content: Container, inputHandler?: (data: string) => void, inputTarget?: Focusable): void { + get capturesViewportInput(): boolean { + return this.inputHandler?.capturesViewportInput === true; + } + + private setContent(content: Container, inputHandler?: InputHandler, inputTarget?: Focusable): void { if (this.inputTarget) this.inputTarget.focused = false; this.content = content; this.inputHandler = inputHandler; @@ -125,12 +131,11 @@ export class McpManagerView implements McpUi, Component, Focusable { const footer = `${keyHint("tui.select.confirm", menu.confirmLabel)} • ${keyHint("tui.select.cancel", menu.cancelLabel)}`; if (menu.items.length === 0) { body.push(new Text(this.theme.fg("muted", menu.empty ?? "Nothing to show."), 1, 0)); - this.setContent( - frame(this.theme, menu.title, body, keyHint("tui.select.cancel", menu.cancelLabel)), - (data) => { + this.setContent(frame(this.theme, menu.title, body, keyHint("tui.select.cancel", menu.cancelLabel)), { + handleInput: (data) => { if (this.keybindings.matches(data, "tui.select.cancel")) finish(undefined); }, - ); + }); return; } const list = new SelectList( @@ -147,7 +152,7 @@ export class McpManagerView implements McpUi, Component, Focusable { list.onSelect = (item) => finish(item.value); list.onCancel = () => finish(undefined); body.push(list); - this.setContent(frame(this.theme, menu.title, body, footer), (data) => list.handleInput(data)); + this.setContent(frame(this.theme, menu.title, body, footer), list); }; render(); unsubscribe = subscribe?.(() => { @@ -162,8 +167,10 @@ export class McpManagerView implements McpUi, Component, Focusable { this.setContent(frame(this.theme, title, body)); return; } - this.setContent(frame(this.theme, title, body, keyHint("tui.select.cancel", "cancel")), (data) => { - if (this.keybindings.matches(data, "tui.select.cancel")) onCancel(); + this.setContent(frame(this.theme, title, body, keyHint("tui.select.cancel", "cancel")), { + handleInput: (data) => { + if (this.keybindings.matches(data, "tui.select.cancel")) onCancel(); + }, }); } @@ -203,21 +210,23 @@ export class McpManagerView implements McpUi, Component, Focusable { body, `${keyHint("tui.select.confirm", "submit")} • ${keyHint("tui.select.cancel", "cancel")}`, ), - (data) => { - if (this.keybindings.matches(data, "tui.select.confirm")) { - const value = input.getValue().trim(); - if (value) finish(value); - return; - } - if (this.keybindings.matches(data, "tui.select.cancel")) { - finish(undefined); - return; - } - if (this.keybindings.matches(data, "app.message.copy")) { - void link.copy(); - return; - } - input.handleInput(data); + { + handleInput: (data) => { + if (this.keybindings.matches(data, "tui.select.confirm")) { + const value = input.getValue().trim(); + if (value) finish(value); + return; + } + if (this.keybindings.matches(data, "tui.select.cancel")) { + finish(undefined); + return; + } + if (this.keybindings.matches(data, "app.message.copy")) { + void link.copy(); + return; + } + input.handleInput(data); + }, }, input, ); @@ -225,7 +234,7 @@ export class McpManagerView implements McpUi, Component, Focusable { } handleInput(data: string): void { - this.inputHandler?.(data); + this.inputHandler?.handleInput?.(data); this.tui.requestRender(); } diff --git a/packages/coding-agent/src/extensions/tool-search/tool.ts b/packages/coding-agent/src/extensions/tool-search/tool.ts index 1841ebd97..2cf07c769 100644 --- a/packages/coding-agent/src/extensions/tool-search/tool.ts +++ b/packages/coding-agent/src/extensions/tool-search/tool.ts @@ -20,10 +20,13 @@ import type { export const TOOL_SEARCH_TOOL_NAME = "tool_search"; export const DEFAULT_TOOL_SEARCH_LIMIT = 8; -/** A tool as the ranker sees it: its name and the text built by {@link createToolSearchDocument}. */ +/** A tool as the ranker sees it: its name and the texts built by {@link createToolSearchDocument}. */ export interface ToolSearchDocument { name: string; + /** Description, schema descriptions, and property names. */ text: string; + /** Name, description, and instructions of the tool's namespace. */ + namespaceText?: string; } export interface ToolSearchMatch { @@ -36,6 +39,8 @@ export interface ToolRanker { rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[]; } +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + const STOP_WORDS: ReadonlySet = new Set([ "a", "an", @@ -68,15 +73,22 @@ function stem(term: string): string { return term; } -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ export function tokenize(text: string): string[] { - return text + const normalized = text + .normalize("NFC") .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); } function isObject(value: unknown): value is Record { @@ -101,21 +113,91 @@ function schemaText(schema: unknown, parts: string[]): void { } /** - * Search text of a tool: the name, the name with `_` - * as spaces, the description, schema descriptions and property names, and the namespace with its - * description and instructions. + * Search text of a tool: the description and the schema descriptions and property names, and + * separately the namespace name, description, and instructions. The ranker reads the name from `name`. */ export function createToolSearchDocument( tool: Pick, namespace?: ToolNamespace, ): ToolSearchDocument { - const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + const parts = [tool.description]; schemaText(tool.parameters, parts); - if (namespace) parts.push(namespace.name, namespace.description ?? "", namespace.instructions ?? ""); - return { name: tool.name, text: parts.filter((part) => part.trim()).join(" ") }; + const namespaceText = namespace + ? [namespace.name, namespace.description ?? "", namespace.instructions ?? ""].join(" ") + : ""; + return { name: tool.name, text: parts.filter((part) => part.trim()).join(" "), namespaceText }; } -/** Okapi BM25 with the usual parameters. Ties keep document order. */ +/** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */ +function normalizeName(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9_$]/g, "_"); +} + +/** + * Whether the whole query is the tool's name: its full name or a suffix after a `__` separator, so + * `thread_read` and `app__thread_read` both name `mcp__app__thread_read`. + */ +function isExactNameQuery(query: string, name: string): boolean { + const normalized = normalizeName(name); + return normalized === query || normalized.endsWith(`__${query}`); +} + +/** A document split into the fields the ranker weighs differently. */ +interface IndexedDocument { + name: string; + /** Terms of the name, for example `mcp`, `app`, `thread`, `read` for `mcp__app__thread_read`. */ + nameTerms: ReadonlySet; + /** Terms of the last `__` segment of the name, the part models say: `thread`, `read`. */ + localNameTerms: readonly string[]; + textCounts: ReadonlyMap; + textLength: number; + namespaceTerms: ReadonlySet; +} + +function indexDocument(document: ToolSearchDocument): IndexedDocument { + const textCounts = new Map(); + for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1); + const segments = document.name.split("__"); + return { + name: document.name, + nameTerms: new Set(tokenize(document.name)), + localNameTerms: [...new Set(tokenize(segments[segments.length - 1]))], + textCounts, + textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0), + namespaceTerms: new Set(tokenize(document.namespaceText ?? "")), + }; +} + +/** + * A query term in the name counts this many times its IDF. The text part of a term is at most + * `k1 + 1` (2.2) times its IDF and about 1 for a single mention, so a name match beats any number of + * mentions in the description. + */ +const NAME_WEIGHT = 3; + +/** + * A query term in the namespace name, description, or instructions counts this many times its IDF. They + * are shared by every tool of the server, so they help find the server but not the tool in it. + */ +const NAMESPACE_WEIGHT = 0.5; + +/** + * Okapi BM25 over the description, with the name and the namespace as separate fields: + * - a tool that matches more of the query's terms, in any field, ranks above one that matches + * fewer, whatever their scores: for `screenshot window`, a tool whose description has both words + * beats `window`, whose name has one; + * - a query term in the name scores more than any number of mentions in the description; + * - a tool whose whole local name is in the query (`app` in `peekaboo app` for + * `mcp__peekaboo__app`) gets a bonus worth one more match of every query term, so among tools + * matching as many terms it mostly wins over tools that only mention the words; + * - a query that is exactly a tool's name ranks that tool first, before coverage. + * + * The description alone is length-normalized, so a long name or long namespace instructions do + * not lower a tool's score. Ties keep document order. + */ export class Bm25Ranker implements ToolRanker { private readonly k1: number; private readonly b: number; @@ -126,33 +208,44 @@ export class Bm25Ranker implements ToolRanker { } rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[] { - const queryTerms = [...new Set(tokenize(query))]; - if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; - const termCounts = documents.map((document) => { - const counts = new Map(); - for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); - return counts; - }); - const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); - const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; + const queryTerms = new Set(tokenize(query)); + if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return []; + const indexed = documents.map(indexDocument); + const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1; const idf = new Map( - queryTerms.map((term) => { - const frequency = termCounts.filter((counts) => counts.has(term)).length; - return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; + [...queryTerms].map((term) => { + const frequency = indexed.filter( + (document) => + document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term), + ).length; + return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const; }), ); - const matches: ToolSearchMatch[] = []; - documents.forEach((document, index) => { + const fullNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); + const exactQuery = normalizeName(query); + const matches: (ToolSearchMatch & { covered: number; exact: boolean })[] = []; + for (const document of indexed) { + const norm = this.k1 * (1 - this.b + (this.b * document.textLength) / averageLength); let score = 0; + let covered = 0; for (const term of queryTerms) { - const count = termCounts[index].get(term); - if (!count) continue; - const norm = this.k1 * (1 - this.b + (this.b * lengths[index]) / averageLength); - score += (idf.get(term) ?? 0) * ((count * (this.k1 + 1)) / (count + norm)); + const count = document.textCounts.get(term) ?? 0; + const weight = + (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + + (count * (this.k1 + 1)) / (count + norm); + if (weight > 0) covered++; + score += (idf.get(term) ?? 0) * weight; } - if (score > 0) matches.push({ name: document.name, score }); - }); - return matches.sort((a, b) => b.score - a.score).slice(0, limit); + if (score === 0) continue; + const local = document.localNameTerms; + if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += fullNameBonus; + matches.push({ name: document.name, score, covered, exact: isExactNameQuery(exactQuery, document.name) }); + } + return matches + .sort((a, b) => Number(b.exact) - Number(a.exact) || b.covered - a.covered || b.score - a.score) + .slice(0, limit) + .map(({ name, score }) => ({ name, score })); } } diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index f86470ad2..7e25c2cb8 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -118,6 +118,7 @@ export type { InputEvent, InputEventResult, InputSource, + KeybindingHints, KeybindingsManager, LoadExtensionsResult, LsToolCallEvent, diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 723fc910b..88aa3c5d3 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -591,7 +591,18 @@ export async function main(args: string[], options?: MainOptions) { } cleanupManagedInstall(); - const cwd = process.cwd(); + let cwd: string; + try { + cwd = process.cwd(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + console.error( + chalk.red( + "Error: The current working directory no longer exists. Change to an existing directory and try again.", + ), + ); + process.exit(1); + } const agentDir = getAgentDir(); const bootstrapSettingsManager = SettingsManager.create(cwd, agentDir, { projectTrusted: false }); applyHttpProxySettings(bootstrapSettingsManager.getGlobalSettings().httpProxy); @@ -745,9 +756,7 @@ export async function main(args: string[], options?: MainOptions) { parsed.projectTrustOverride === undefined && cachedProjectTrust === undefined && hasTrustRequiringResources; const projectTrusted = shouldResolveProjectTrust ? false - : (cachedProjectTrust ?? - parsed.projectTrustOverride ?? - (!hasTrustRequiringResources || trustStore.get(cwd) === true)); + : (cachedProjectTrust ?? parsed.projectTrustOverride ?? trustStore.get(cwd) ?? !hasTrustRequiringResources); const runtimeSettingsManager = SettingsManager.create(cwd, agentDir, { projectTrusted }); const services = await createAgentSessionServices({ cwd, diff --git a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts index c710c7df7..7cb296031 100644 --- a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts @@ -1,5 +1,13 @@ import type { AssistantMessage } from "@earendil-works/pi-ai"; -import { Container, Markdown, type MarkdownTheme, MouseRegion, Spacer, Text } from "@earendil-works/pi-tui"; +import { + type Component, + Container, + Markdown, + type MarkdownTheme, + MouseRegion, + Spacer, + Text, +} from "@earendil-works/pi-tui"; import type { MarkdownTransformer } from "../../../core/extensions/types.ts"; import { getMarkdownTheme, theme } from "../theme/theme.ts"; import { createMarkdownTransform } from "./markdown-transform.ts"; @@ -8,6 +16,15 @@ const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; const OSC133_ZONE_FINAL = "\x1b]133;C\x07"; +/** + * A Markdown block of the last update. The next update reuses it at the same position for the same kind of content, so + * a streamed answer keeps the blocks it already rendered, also once the stream ends. + */ +interface MarkdownSlot { + kind: "text" | "thinking"; + markdown: Markdown; +} + /** * Component that renders a complete assistant message */ @@ -24,6 +41,7 @@ export class AssistantMessageComponent extends Container { private messageId?: string; private timestamp?: string; private thinkingVisibilityOverrides = new Map(); + private markdownSlots: MarkdownSlot[] = []; constructor( message?: AssistantMessage, @@ -78,6 +96,8 @@ export class AssistantMessageComponent extends Container { setOutputPad(padding: number): void { this.outputPad = padding; + // Markdown takes its padding at construction. + this.markdownSlots = []; if (this.lastMessage) { this.updateContent(this.lastMessage); } @@ -100,14 +120,27 @@ export class AssistantMessageComponent extends Container { // Clear content container this.contentContainer.clear(); + const previousSlots = this.markdownSlots; + this.markdownSlots = []; + const markdownBlock = (kind: MarkdownSlot["kind"], text: string, create: () => Markdown): Markdown => { + const slot = previousSlots[this.markdownSlots.length]; + const markdown = slot?.kind === kind ? slot.markdown : create(); + markdown.setText(text); + this.markdownSlots.push({ kind, markdown }); + return markdown; + }; - const hasVisibleContent = message.content.some( - (c) => (c.type === "text" && c.text.trim()) || (c.type === "thinking" && c.thinking.trim()), - ); - - if (hasVisibleContent) { - this.contentContainer.addChild(new Spacer(1)); - } + // Spacers are added lazily, right before the next visible content block. This puts a blank + // line before the first visible block and after each visible thinking run that is followed + // by more visible content, while hidden thinking runs without a label contribute nothing. + let needsSpacer = true; + const addVisibleChild = (child: Component) => { + if (needsSpacer) { + this.contentContainer.addChild(new Spacer(1)); + needsSpacer = false; + } + this.contentContainer.addChild(child); + }; // Render content in order let thinkingRunIndex = 0; @@ -116,16 +149,22 @@ export class AssistantMessageComponent extends Container { if (content.type === "text" && content.text.trim()) { // Assistant text messages with no background - trim the text // Set paddingY=0 to avoid extra spacing before tool executions - this.contentContainer.addChild( - new Markdown(content.text.trim(), this.outputPad, 0, this.markdownTheme, undefined, { - transform: createMarkdownTransform( - "assistant", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, - ), - }), + const text = content.text.trim(); + addVisibleChild( + markdownBlock( + "text", + text, + () => + new Markdown(text, this.outputPad, 0, this.markdownTheme, undefined, { + transform: createMarkdownTransform( + "assistant", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }), + ), ); } else if (content.type === "thinking") { const thinkingBlocks: string[] = []; @@ -145,36 +184,40 @@ export class AssistantMessageComponent extends Container { continue; } - // Add spacing only when another visible assistant content block follows. - // This avoids a superfluous blank line before separately-rendered tool execution blocks. - const hasVisibleContentAfter = message.content - .slice(i + 1) - .some((c) => (c.type === "text" && c.text.trim()) || (c.type === "thinking" && c.thinking.trim())); - const runIndex = thinkingRunIndex++; const hidden = this.thinkingVisibilityOverrides.get(runIndex) ?? this.hideThinkingBlock; + // A hidden run with an empty label renders nothing, including no spacing. + if (hidden && !this.hiddenThinkingLabel.trim()) { + continue; + } + const thinking = thinkingBlocks.join("\n\n"); const thinkingComponent = hidden ? new Text(theme.italic(theme.fg("thinkingText", this.hiddenThinkingLabel)), this.outputPad, 0) - : new Markdown( - thinkingBlocks.join("\n\n"), - this.outputPad, - 0, - this.markdownTheme, - { - color: (text: string) => theme.fg("thinkingText", text), - italic: true, - }, - { - transform: createMarkdownTransform( - "assistant-thinking", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, + : markdownBlock( + "thinking", + thinking, + () => + new Markdown( + thinking, + this.outputPad, + 0, + this.markdownTheme, + { + color: (text: string) => theme.fg("thinkingText", text), + italic: true, + }, + { + transform: createMarkdownTransform( + "assistant-thinking", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }, ), - }, ); - this.contentContainer.addChild( + addVisibleChild( new MouseRegion(thinkingComponent, (event) => { if (event.type !== "click" || event.button !== "left") return undefined; this.thinkingVisibilityOverrides.set(runIndex, !hidden); @@ -182,9 +225,9 @@ export class AssistantMessageComponent extends Container { return { handled: true }; }), ); - if (hasVisibleContentAfter) { - this.contentContainer.addChild(new Spacer(1)); - } + // Add spacing only when another visible assistant content block follows. + // This avoids a superfluous blank line before separately-rendered tool execution blocks. + needsSpacer = true; } } diff --git a/packages/coding-agent/src/modes/interactive/components/config-selector.ts b/packages/coding-agent/src/modes/interactive/components/config-selector.ts index a8054886e..913b51179 100644 --- a/packages/coding-agent/src/modes/interactive/components/config-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/config-selector.ts @@ -2,7 +2,6 @@ * TUI component for managing package resources (enable/disable) */ -import { homedir } from "node:os"; import { basename, dirname, join, relative } from "node:path"; import { type Component, @@ -18,8 +17,8 @@ import { import { CONFIG_DIR_NAME } from "../../../config.ts"; import type { PathMetadata, ResolvedPaths, ResolvedResource } from "../../../core/package-manager.ts"; import type { PackageSource, SettingsManager } from "../../../core/settings-manager.ts"; -import { BUILTIN_PATH_PREFIX } from "../../../core/source-info.ts"; -import { canonicalizePath, isLocalPath, resolvePath } from "../../../utils/paths.ts"; +import { BUILTIN_PATH_PREFIX, getResourcePathKey } from "../../../core/source-info.ts"; +import { isLocalPath, resolvePath, shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint, rawKeyHint } from "./keybinding-hints.ts"; @@ -65,19 +64,7 @@ interface ResourceGroup { } function formatBaseDir(baseDir: string): string { - const homeDir = homedir(); - let displayPath: string; - - if (baseDir === homeDir) { - displayPath = "~"; - } else if (baseDir.startsWith(homeDir)) { - // Replace home prefix with ~, normalize separators for display - const rest = baseDir.slice(homeDir.length); - displayPath = `~${rest.replace(/\\/g, "/")}`; - } else { - displayPath = baseDir.replace(/\\/g, "/"); - } - + const displayPath = shortenHomePath(baseDir).replace(/\\/g, "/"); return displayPath.endsWith("/") ? displayPath : `${displayPath}/`; } @@ -848,7 +835,7 @@ class ResourceList implements Component, Focusable { } private getResourceItemKey(item: ResourceItem): string { - return `${item.resourceType}:${canonicalizePath(item.path)}`; + return `${item.resourceType}:${getResourcePathKey(item.path)}`; } private getItemScope(item: ResourceItem): SettingsScope { diff --git a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts index 9d345b4e9..1f0c53958 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts @@ -91,6 +91,16 @@ export class CustomEditor extends Editor { return; } + // Selection page keys move an open autocomplete menu before app actions run. + // This lets users bind Ctrl+V even though it pastes images by default. + if ( + this.isShowingAutocomplete() && + (this.keybindings.matches(data, "tui.select.pageUp") || this.keybindings.matches(data, "tui.select.pageDown")) + ) { + super.handleInput(data); + return; + } + // Check for clipboard paste keybinding if (this.keybindings.matches(data, "app.clipboard.pasteImage")) { this.onPasteImage?.(); diff --git a/packages/coding-agent/src/modes/interactive/components/custom-entry.ts b/packages/coding-agent/src/modes/interactive/components/custom-entry.ts index 078ad9fca..81d381ee9 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-entry.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-entry.ts @@ -3,6 +3,7 @@ import { Box, Container, Spacer, Text } from "@earendil-works/pi-tui"; import type { EntryRenderer } from "../../../core/extensions/types.ts"; import type { CustomEntry } from "../../../core/session-manager.ts"; import { theme } from "../theme/theme.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; /** * Component that renders a custom session entry from extensions. @@ -50,7 +51,7 @@ export class CustomEntryComponent extends Container { let component: Component | undefined; try { - component = this.renderer(this.entry, { expanded: this._expanded }, theme); + component = this.renderer(this.entry, { expanded: this._expanded, keyText, keyHint }, theme); } catch (error) { const message = error instanceof Error ? error.message : String(error); const box = new Box(this.outputPad, 1, (text) => theme.bg("customMessageBg", text)); diff --git a/packages/coding-agent/src/modes/interactive/components/custom-message.ts b/packages/coding-agent/src/modes/interactive/components/custom-message.ts index 6c757e0dd..6dcddf402 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-message.ts @@ -4,6 +4,7 @@ import { Box, Container, Markdown, type MarkdownTheme, Spacer, Text } from "@ear import type { MessageRenderer } from "../../../core/extensions/types.ts"; import type { CustomMessage } from "../../../core/messages.ts"; import { getMarkdownTheme, theme } from "../theme/theme.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; /** * Component that renders a custom message entry from extensions. @@ -70,7 +71,7 @@ export class CustomMessageComponent extends Container { try { const component = this.customRenderer( this.message, - { expanded: this._expanded, outputPad: this.outputPad }, + { expanded: this._expanded, outputPad: this.outputPad, keyText, keyHint }, theme, ); if (component) { diff --git a/packages/coding-agent/src/modes/interactive/components/extension-editor.ts b/packages/coding-agent/src/modes/interactive/components/extension-editor.ts index 110447938..44f4bb220 100644 --- a/packages/coding-agent/src/modes/interactive/components/extension-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/extension-editor.ts @@ -30,6 +30,8 @@ export class ExtensionEditorComponent extends Container implements Focusable { private tui: TUI; private keybindings: KeybindingsManager; private externalEditorCommand: string; + /** Shows external editor failures; renders nothing while empty. */ + private externalEditorStatus = new Text("", 1, 0); private _focused = false; get focused(): boolean { @@ -85,6 +87,7 @@ export class ExtensionEditorComponent extends Container implements Focusable { this.onSubmitCallback(text); }; this.addChild(this.editor); + this.addChild(this.externalEditorStatus); this.addChild(new Spacer(1)); @@ -130,6 +133,7 @@ export class ExtensionEditorComponent extends Container implements Focusable { command: this.externalEditorCommand, content, }); + this.externalEditorStatus.setText(result.status === "failed" ? theme.fg("warning", result.message) : ""); if (result.status === "complete") { this.editor.setText(result.content); } diff --git a/packages/coding-agent/src/modes/interactive/components/login-dialog.ts b/packages/coding-agent/src/modes/interactive/components/login-dialog.ts index 0803ff865..95e676803 100644 --- a/packages/coding-agent/src/modes/interactive/components/login-dialog.ts +++ b/packages/coding-agent/src/modes/interactive/components/login-dialog.ts @@ -3,7 +3,9 @@ import { Container, type Focusable, getKeybindings, + hyperlink, Input, + replaceControlCharacters, Spacer, Text, TruncatedText, @@ -16,6 +18,31 @@ import { AuthUrlComponent } from "./auth-url.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint } from "./keybinding-hints.ts"; +/** + * The `http:` or `https:` URL `value` names, normalized, or undefined. The shared launcher also opens files and custom + * protocols, so login only opens web URLs. Normalizing percent-encodes control characters, which would otherwise end a + * terminal hyperlink early. + */ +function webUrl(value: string): string | undefined { + if (!URL.canParse(value)) return undefined; + const url = new URL(value); + return url.protocol === "https:" || url.protocol === "http:" ? url.href : undefined; +} + +/** + * `value` normalized as a web URL, else with control characters replaced: an untrusted string printed as is could end a + * hyperlink early or send its own escape sequences to the terminal. + */ +function displayUrl(value: string): string { + return webUrl(value) ?? replaceControlCharacters(value); +} + +/** `text` as a terminal hyperlink to `url` when that is a web URL, else plain `text`. */ +function webLink(text: string, url: string): string { + const target = webUrl(url); + return target === undefined ? text : hyperlink(text, target); +} + /** * Login dialog component - replaces editor during OAuth login flow */ @@ -121,7 +148,8 @@ export class LoginDialogComponent extends Container implements Focusable { this.clearDeviceCodeAction(); this.contentContainer.clear(); this.contentContainer.addChild(new Spacer(1)); - this.authUrl = new AuthUrlComponent(this.tui, url); + const target = webUrl(url); + this.authUrl = new AuthUrlComponent(this.tui, target ?? replaceControlCharacters(url)); this.contentContainer.addChild(this.authUrl); if (instructions) { @@ -129,7 +157,7 @@ export class LoginDialogComponent extends Container implements Focusable { this.contentContainer.addChild(new Text(theme.fg("warning", instructions), 1, 0)); } - openBrowser(url); + if (target !== undefined) openBrowser(target); this.tui.requestRender(); } @@ -146,12 +174,11 @@ export class LoginDialogComponent extends Container implements Focusable { this.clearDeviceCodeAction(); this.contentContainer.clear(); this.contentContainer.addChild(new Spacer(1)); - const linkedUrl = `\x1b]8;;${info.verificationUri}\x07${info.verificationUri}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("accent", linkedUrl), 1, 0)); + const shownUrl = displayUrl(info.verificationUri); + this.contentContainer.addChild(new Text(theme.fg("accent", webLink(shownUrl, shownUrl)), 1, 0)); const clickHint = process.platform === "darwin" ? "Cmd+click to open" : "Ctrl+click to open"; - const hyperlink = `\x1b]8;;${info.verificationUri}\x07${clickHint}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("dim", hyperlink), 1, 0)); + this.contentContainer.addChild(new Text(theme.fg("dim", webLink(clickHint, info.verificationUri)), 1, 0)); this.contentContainer.addChild(new Spacer(1)); this.contentContainer.addChild(new Text(theme.fg("warning", `Enter code: ${info.userCode}`), 1, 0)); @@ -187,11 +214,10 @@ export class LoginDialogComponent extends Container implements Focusable { } if (info.openBrowserOnConfirm) { try { - // The shared launcher also opens files; device-code login must only open web URLs. - const url = new URL(info.verificationUri); - if (url.protocol === "https:" || url.protocol === "http:") openBrowser(url.href); + const target = webUrl(info.verificationUri); + if (target !== undefined) openBrowser(target); } catch { - // Invalid URLs or launcher failures must not interrupt login. + // Launcher failures must not interrupt login. } } } @@ -260,9 +286,9 @@ export class LoginDialogComponent extends Container implements Focusable { this.contentContainer.addChild(new Spacer(1)); this.contentContainer.addChild(new Text(theme.fg("text", message), 1, 0)); for (const link of links) { - const text = link.label ? `${link.label}: ${link.url}` : link.url; - const hyperlink = `\x1b]8;;${link.url}\x07${text}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("accent", hyperlink), 1, 0)); + const url = displayUrl(link.url); + const text = link.label ? `${link.label}: ${url}` : url; + this.contentContainer.addChild(new Text(theme.fg("accent", webLink(text, url)), 1, 0)); } if (showCloseHint) { this.contentContainer.addChild(new Spacer(1)); diff --git a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts index 4ff7685c2..58bae9094 100644 --- a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts +++ b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts @@ -1,8 +1,9 @@ import type { MarkdownTransformContext, MarkdownTransformer } from "../../../core/extensions/types.ts"; +/** Applies extension Markdown transformers. `isStreaming` is read per transform, so a reused component sees a stream end. */ export function createMarkdownTransform( messageType: MarkdownTransformContext["messageType"], - isStreaming: boolean, + isStreaming: () => boolean, transformers: readonly MarkdownTransformer[], messageId?: string, timestamp?: string, @@ -10,7 +11,7 @@ export function createMarkdownTransform( return (markdown, availableWidth) => applyMarkdownTransformers( markdown, - { messageType, isStreaming, availableWidth, messageId, timestamp }, + { messageType, isStreaming: isStreaming(), availableWidth, messageId, timestamp }, transformers, ); } diff --git a/packages/coding-agent/src/modes/interactive/components/model-selector.ts b/packages/coding-agent/src/modes/interactive/components/model-selector.ts index d603b8bef..f33eecf27 100644 --- a/packages/coding-agent/src/modes/interactive/components/model-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/model-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Spacer, Text, @@ -34,10 +35,13 @@ interface DefaultModelReference { type ModelScope = "all" | "scoped"; +const MAX_VISIBLE = 10; + /** * Component that renders a model selector with search */ export class ModelSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private searchInput: Input; // Focusable implementation - propagate to searchInput for IME cursor positioning @@ -305,12 +309,11 @@ export class ModelSelectorComponent extends Container implements Focusable { private updateList(): void { this.listContainer.clear(); - const maxVisible = 10; const startIndex = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredModels.length - maxVisible), + Math.min(this.selectedIndex - Math.floor(MAX_VISIBLE / 2), this.filteredModels.length - MAX_VISIBLE), ); - const endIndex = Math.min(startIndex + maxVisible, this.filteredModels.length); + const endIndex = Math.min(startIndex + MAX_VISIBLE, this.filteredModels.length); // Show visible slice of filtered models for (let i = startIndex; i < endIndex; i++) { @@ -363,6 +366,7 @@ export class ModelSelectorComponent extends Container implements Focusable { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredModels.length, MAX_VISIBLE); if (kb.matches(keyData, "tui.input.tab")) { if (this.scopedModelItems.length > 0) { const nextScope: ModelScope = this.scope === "all" ? "scoped" : "all"; @@ -385,6 +389,12 @@ export class ModelSelectorComponent extends Container implements Focusable { this.selectedIndex = this.selectedIndex === this.filteredModels.length - 1 ? 0 : this.selectedIndex + 1; this.updateList(); } + // Page up/down + else if (pageIndex !== undefined) { + if (this.filteredModels.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedModel = this.filteredModels[this.selectedIndex]; diff --git a/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts b/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts index c03f74836..e3f70985b 100644 --- a/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Spacer, TruncatedText, @@ -52,10 +53,13 @@ export function formatAuthSelectorProviderStatus(provider: AuthSelectorProvider) return theme.fg("success", ` ✓ ${source}`); } +const MAX_VISIBLE = 8; + /** * Component that renders an auth provider selector */ export class OAuthSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private searchInput: Input; // Focusable implementation - propagate to search input for IME cursor positioning @@ -143,12 +147,11 @@ export class OAuthSelectorComponent extends Container implements Focusable { private updateList(): void { this.listContainer.clear(); - const maxVisible = 8; const startIndex = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredProviders.length - maxVisible), + Math.min(this.selectedIndex - Math.floor(MAX_VISIBLE / 2), this.filteredProviders.length - MAX_VISIBLE), ); - const endIndex = Math.min(startIndex + maxVisible, this.filteredProviders.length); + const endIndex = Math.min(startIndex + MAX_VISIBLE, this.filteredProviders.length); for (let i = startIndex; i < endIndex; i++) { const provider = this.filteredProviders[i]; @@ -192,6 +195,7 @@ export class OAuthSelectorComponent extends Container implements Focusable { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredProviders.length, MAX_VISIBLE); // Up arrow if (kb.matches(keyData, "tui.select.up")) { if (this.filteredProviders.length === 0) return; @@ -204,6 +208,12 @@ export class OAuthSelectorComponent extends Container implements Focusable { this.selectedIndex = Math.min(this.filteredProviders.length - 1, this.selectedIndex + 1); this.updateList(); } + // Page up/down + else if (pageIndex !== undefined) { + if (this.filteredProviders.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedProvider = this.filteredProviders[this.selectedIndex]; diff --git a/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts b/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts index 54a6f97cc..9a63ec294 100644 --- a/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Key, matchesKey, @@ -95,6 +96,7 @@ export interface ModelsCallbacks { * Changes are session-only until explicitly persisted with Ctrl+S. */ export class ScopedModelsSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private modelsById: Map> = new Map(); private allIds: string[] = []; private enabledIds: EnabledIds = null; @@ -295,6 +297,13 @@ export class ScopedModelsSelectorComponent extends Container implements Focusabl this.updateList(); return; } + const pageIndex = getSelectPageIndex(kb, data, this.selectedIndex, this.filteredItems.length, this.maxVisible); + if (pageIndex !== undefined) { + if (this.filteredItems.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + return; + } // Reorder enabled models const reorderUp = kb.matches(data, "app.models.reorderUp"); diff --git a/packages/coding-agent/src/modes/interactive/components/session-selector.ts b/packages/coding-agent/src/modes/interactive/components/session-selector.ts index d60eaffc8..4d6d6c09f 100644 --- a/packages/coding-agent/src/modes/interactive/components/session-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/session-selector.ts @@ -1,7 +1,6 @@ import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import { unlink } from "node:fs/promises"; -import * as os from "node:os"; import { type Component, Container, @@ -15,7 +14,7 @@ import { } from "@earendil-works/pi-tui"; import { KeybindingsManager } from "../../../core/keybindings.ts"; import type { SessionInfo, SessionListProgress } from "../../../core/session-manager.ts"; -import { canonicalizePath as _canonicalizePath } from "../../../utils/paths.ts"; +import { canonicalizePath, getPathComparisonKey, shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint, keyText } from "./keybinding-hints.ts"; @@ -23,15 +22,6 @@ import { filterAndSortSessions, hasSessionName, type NameFilter, type SortMode } type SessionScope = "current" | "all"; -function shortenPath(path: string): string { - const home = os.homedir(); - if (!path) return path; - if (path.startsWith(home)) { - return `~${path.slice(home.length)}`; - } - return path; -} - function formatSessionDate(date: Date): string { const now = new Date(); const diffMs = now.getTime() - date.getTime(); @@ -48,9 +38,9 @@ function formatSessionDate(date: Date): string { return `${Math.floor(diffDays / 365)}y`; } -function canonicalizePath(path: string | undefined): string | undefined { - if (!path) return path; - return _canonicalizePath(path); +/** Comparison key for a session file path; follows symlinks and ignores case on Windows. */ +function getSessionPathKey(path: string): string { + return getPathComparisonKey(canonicalizePath(path)); } class SessionSelectorHeader implements Component { @@ -210,16 +200,16 @@ function buildSessionTree(sessions: SessionInfo[]): SessionTreeNode[] { const byPath = new Map(); for (const session of sessions) { - const sessionPath = canonicalizePath(session.path) ?? session.path; + const sessionPath = getSessionPathKey(session.path); byPath.set(sessionPath, { session, children: [], latestActivity: session.modified.getTime() }); } const roots: SessionTreeNode[] = []; for (const session of sessions) { - const sessionPath = canonicalizePath(session.path) ?? session.path; + const sessionPath = getSessionPathKey(session.path); const node = byPath.get(sessionPath)!; - const parentPath = canonicalizePath(session.parentSessionPath); + const parentPath = session.parentSessionPath ? getSessionPathKey(session.parentSessionPath) : undefined; if (parentPath && byPath.has(parentPath)) { byPath.get(parentPath)!.children.push(node); @@ -296,7 +286,7 @@ class SessionList implements Component, Focusable { private keybindings: KeybindingsManager; private showPath = false; private confirmingDeletePath: string | null = null; - private currentSessionCanonicalPath?: string; + private currentSessionPathKey?: string; public onSelect?: (sessionPath: string) => void; public onCancel?: () => void; public onExit: () => void = () => {}; @@ -335,7 +325,7 @@ class SessionList implements Component, Focusable { this.sortMode = sortMode; this.nameFilter = nameFilter; this.keybindings = keybindings; - this.currentSessionCanonicalPath = canonicalizePath(currentSessionFilePath); + this.currentSessionPathKey = currentSessionFilePath ? getSessionPathKey(currentSessionFilePath) : undefined; this.filterSessions(""); // Handle Enter in search input - select current item @@ -413,8 +403,8 @@ class SessionList implements Component, Focusable { } private isCurrentSessionPath(path: string): boolean { - if (!this.currentSessionCanonicalPath) return false; - return (canonicalizePath(path) ?? path) === this.currentSessionCanonicalPath; + if (!this.currentSessionPathKey) return false; + return getSessionPathKey(path) === this.currentSessionPathKey; } invalidate(): void {} @@ -474,10 +464,10 @@ class SessionList implements Component, Focusable { const msgCount = String(session.messageCount); let rightPart = `${msgCount} ${age}`; if (this.showCwd && session.cwd) { - rightPart = `${shortenPath(session.cwd)} ${rightPart}`; + rightPart = `${shortenHomePath(session.cwd)} ${rightPart}`; } if (this.showPath) { - rightPart = `${shortenPath(session.path)} ${rightPart}`; + rightPart = `${shortenHomePath(session.path)} ${rightPart}`; } // Cursor @@ -692,6 +682,10 @@ async function deleteSessionFile( * Component that renders a session selector */ export class SessionSelectorComponent extends Container implements Focusable { + get capturesViewportInput(): boolean { + return this.mode !== "rename"; + } + handleInput(data: string): void { if (this.mode === "rename") { const kb = getKeybindings(); diff --git a/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts b/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts index 81703323a..5da1d6fcc 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts @@ -123,6 +123,8 @@ export class SelectSubmenu extends Container { const isNav = kb.matches(data, "tui.select.up") || kb.matches(data, "tui.select.down") || + kb.matches(data, "tui.select.pageUp") || + kb.matches(data, "tui.select.pageDown") || kb.matches(data, "tui.select.confirm") || kb.matches(data, "tui.select.cancel"); if (isNav) { diff --git a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts index e739fc503..6d691b426 100644 --- a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts +++ b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts @@ -18,7 +18,7 @@ export type { ToolRenderers }; import { formatToolCallWithArgs, getTextOutput as getRenderedTextOutput } from "../../../core/tools/render-utils.ts"; import { ensurePngTranscoder } from "../../../utils/image-convert.ts"; import { theme } from "../theme/theme.ts"; -import { keyHint } from "./keybinding-hints.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; const FALLBACK_PREVIEW_LINES = 10; @@ -138,6 +138,8 @@ export class ToolExecutionComponent extends Container { isError: this.result?.isError ?? false, durationMs: this.isPartial ? undefined : this.result?.durationMs, outputPad: this.outputPad, + keyText, + keyHint, }; } diff --git a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts index 767740c87..5bcdd846d 100644 --- a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts @@ -14,6 +14,7 @@ import { } from "@earendil-works/pi-tui"; import type { SessionTreeNode } from "../../../core/session-manager.ts"; import { formatReadLineRange } from "../../../core/tools/render-utils.ts"; +import { shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { formatKeyText, keyHint } from "./keybinding-hints.ts"; @@ -945,11 +946,7 @@ class TreeList implements Component { } private formatToolCall(name: string, args: Record): string { - const shortenPath = (p: string): string => { - const home = process.env.HOME || process.env.USERPROFILE || ""; - if (home && p.startsWith(home)) return `~${p.slice(home.length)}`; - return p; - }; + const shortenPath = (p: string): string => shortenHomePath(p, process.env.HOME || process.env.USERPROFILE || ""); switch (name) { case "read": { @@ -1414,6 +1411,10 @@ export class TreeSelectorComponent extends Container implements Focusable { this.treeContainer.addChild(this.treeList); } + get capturesViewportInput(): boolean { + return this.labelInput === null; + } + handleInput(keyData: string): void { if (this.labelInput) { this.labelInput.handleInput(keyData); diff --git a/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts b/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts index 69fbbc82a..d187ad5b6 100644 --- a/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts @@ -1,4 +1,12 @@ -import { type Component, Container, getKeybindings, Spacer, Text, truncateToWidth } from "@earendil-works/pi-tui"; +import { + type Component, + Container, + getKeybindings, + getSelectPageIndex, + Spacer, + Text, + truncateToWidth, +} from "@earendil-works/pi-tui"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; @@ -12,6 +20,7 @@ interface UserMessageItem { * Custom user message list component with selection */ class UserMessageList implements Component { + readonly capturesViewportInput: boolean = true; private messages: UserMessageItem[] = []; private selectedIndex: number = 0; public onSelect?: (entryId: string) => void; @@ -80,6 +89,7 @@ class UserMessageList implements Component { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.messages.length, this.maxVisible); // Up arrow - go to previous (older) message, wrap to bottom when at top if (kb.matches(keyData, "tui.select.up")) { this.selectedIndex = this.selectedIndex === 0 ? this.messages.length - 1 : this.selectedIndex - 1; @@ -88,6 +98,11 @@ class UserMessageList implements Component { else if (kb.matches(keyData, "tui.select.down")) { this.selectedIndex = this.selectedIndex === this.messages.length - 1 ? 0 : this.selectedIndex + 1; } + // Page up/down + else if (pageIndex !== undefined) { + if (this.messages.length === 0) return; + this.selectedIndex = pageIndex; + } // Enter - select message and branch else if (kb.matches(keyData, "tui.select.confirm")) { const selected = this.messages[this.selectedIndex]; diff --git a/packages/coding-agent/src/modes/interactive/components/user-message.ts b/packages/coding-agent/src/modes/interactive/components/user-message.ts index d8ca373cc..e313e0242 100644 --- a/packages/coding-agent/src/modes/interactive/components/user-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/user-message.ts @@ -60,7 +60,7 @@ export class UserMessageComponent extends Container { preserveBackslashEscapes: true, transform: createMarkdownTransform( "user", - false, + () => false, this.markdownTransformers, this.messageId, this.timestamp, diff --git a/packages/coding-agent/src/modes/interactive/external-editor.ts b/packages/coding-agent/src/modes/interactive/external-editor.ts index d672d4e08..c4a0def9f 100644 --- a/packages/coding-agent/src/modes/interactive/external-editor.ts +++ b/packages/coding-agent/src/modes/interactive/external-editor.ts @@ -9,7 +9,7 @@ export interface ExternalEditorOptions { content: string; } -export type ExternalEditorResult = { status: "complete"; content: string } | { status: "failed" }; +export type ExternalEditorResult = { status: "complete"; content: string } | { status: "failed"; message: string }; export async function editInExternalEditor(options: ExternalEditorOptions): Promise { const directory = mkdtempSync(join(tmpdir(), "pi-editor-")); @@ -22,17 +22,29 @@ export async function editInExternalEditor(options: ExternalEditorOptions): Prom // Do not use spawnSync here. On Windows, synchronous child_process calls can keep // Node/libuv's console input read active after the parent pauses stdin, racing // vim/nvim for the console input buffer until Ctrl+C cancels the pending read. - const exitCode = await new Promise((resolve) => { + // Null means the editor could not be launched. + const exit = await new Promise((resolve) => { + // The editor inherits pi's cwd so project-relative paths, integrated terminals, + // and similar editor features keep working. A long cwd can break Windows editors: + // Windows rejects working directories over MAX_PATH, and WSL launches Windows + // editors (code, notepad.exe) from the cwd's \\wsl.localhost\... path (#11592). + // The cwd is intentionally not changed; failures are reported to the user instead. const child = spawn(editor, [...editorArgs, filePath], { stdio: "inherit", shell: process.platform === "win32", }); child.on("error", () => resolve(null)); - child.on("close", (code) => resolve(code)); + child.on("close", (code, signal) => resolve(code ?? signal)); }); - if (exitCode !== 0) { - return { status: "failed" }; + if (exit === null) { + return { status: "failed", message: `Failed to launch external editor: ${options.command}` }; + } + if (exit !== 0) { + return { + status: "failed", + message: `External editor exited with ${typeof exit === "number" ? `code ${exit}` : exit}; prompt unchanged`, + }; } return { status: "complete", content: stripBom(readFileSync(filePath, "utf-8")).replace(/\n$/, "") }; diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index a596eea88..6c6fbcf37 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -115,7 +115,7 @@ import { } from "../../core/session-manager.ts"; import type { FullscreenExitOutput, TuiMode } from "../../core/settings-manager.ts"; import { BUILTIN_SLASH_COMMANDS } from "../../core/slash-commands.ts"; -import type { SourceInfo } from "../../core/source-info.ts"; +import { getSourcePackageKey, type SourceInfo } from "../../core/source-info.ts"; import { isInstallTelemetryEnabled } from "../../core/telemetry.ts"; import { withBuiltInRenderers } from "../../core/tools/renderers/index.ts"; import type { TruncationResult } from "../../core/tools/truncate.ts"; @@ -127,10 +127,11 @@ import { copyToClipboard, readClipboardFilePaths, readClipboardText } from "../. import { extensionForImageMimeType, readClipboardImage } from "../../utils/clipboard-image.ts"; import { parseGitUrl } from "../../utils/git.ts"; import { ensurePngTranscoder } from "../../utils/image-convert.ts"; -import { getCwdRelativePath } from "../../utils/paths.ts"; +import { getCwdRelativePath, shortenHomePath } from "../../utils/paths.ts"; import { getPiUserAgent } from "../../utils/pi-user-agent.ts"; import { killTrackedDetachedChildren } from "../../utils/shell.ts"; import { loadAllHighlightLanguages } from "../../utils/syntax-highlight.ts"; +import { stripBom } from "../../utils/text.ts"; import { ensureTool, type ToolStatus } from "../../utils/tools-manager.ts"; import { checkForNewPiVersion, type LatestPiRelease } from "../../utils/version-check.ts"; import { reportBug } from "./bug-report.ts"; @@ -450,6 +451,12 @@ export interface InteractiveModeOptions { terminal?: Terminal; } +interface ExtensionChangelogUpdate { + name: string; + version: string; + markdown: string; +} + export class InteractiveMode { private runtimeHost: AgentSessionRuntime; private renderer: TuiMainScreen | TuiAltScreen; @@ -493,6 +500,7 @@ export class InteractiveMode { private lastSigintTime = 0; private lastEscapeTime = 0; private changelogMarkdown: string | undefined = undefined; + private extensionChangelogUpdates: ExtensionChangelogUpdate[] = []; private startupNoticesShown = false; private anthropicSubscriptionWarningShown = false; @@ -848,7 +856,7 @@ export class InteractiveMode { } this.startupNoticesShown = true; - if (!this.changelogMarkdown) { + if (!this.changelogMarkdown && this.extensionChangelogUpdates.length === 0) { return; } @@ -856,18 +864,39 @@ export class InteractiveMode { this.chatContainer.addChild(new Spacer(1)); } this.chatContainer.addChild(new DynamicBorder()); - if (this.settingsManager.getCollapseChangelog()) { - const versionMatch = this.changelogMarkdown.match(/##\s+\[?(\d+\.\d+\.\d+)\]?/); - const latestVersion = versionMatch ? versionMatch[1] : this.version; - const condensedText = `Updated to v${latestVersion}. Use ${theme.bold("/changelog")} to view full changelog.`; - this.chatContainer.addChild(new Text(condensedText, 1, 0)); - } else { - this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0)); - this.chatContainer.addChild(new Spacer(1)); - this.chatContainer.addChild( - new Markdown(this.changelogMarkdown.trim(), 1, 0, this.getMarkdownThemeWithSettings()), - ); - this.chatContainer.addChild(new Spacer(1)); + if (this.changelogMarkdown) { + if (this.settingsManager.getCollapseChangelog()) { + const versionMatch = this.changelogMarkdown.match(/##\s+\[?(\d+\.\d+\.\d+)\]?/); + const latestVersion = versionMatch ? versionMatch[1] : this.version; + const condensedText = `Updated to v${latestVersion}. Use ${theme.bold("/changelog")} to view full changelog.`; + this.chatContainer.addChild(new Text(condensedText, 1, 0)); + } else { + this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0)); + this.chatContainer.addChild(new Spacer(1)); + this.chatContainer.addChild( + new Markdown(this.changelogMarkdown.trim(), 1, 0, this.getMarkdownThemeWithSettings()), + ); + this.chatContainer.addChild(new Spacer(1)); + } + } + if (this.extensionChangelogUpdates.length > 0) { + if (this.changelogMarkdown) { + this.chatContainer.addChild(new Spacer(1)); + } + if (this.settingsManager.getCollapseChangelog()) { + const condensedText = this.extensionChangelogUpdates + .map((update) => `Updated ${update.name} to v${update.version}.`) + .join("\n"); + this.chatContainer.addChild(new Text(condensedText, 1, 0)); + } else { + this.chatContainer.addChild( + new ThemedText(() => theme.bold(theme.fg("accent", "Extension Updates")), 1, 0), + ); + this.chatContainer.addChild(new Spacer(1)); + const markdown = this.extensionChangelogUpdates.map((update) => update.markdown).join("\n\n"); + this.chatContainer.addChild(new Markdown(markdown, 1, 0, this.getMarkdownThemeWithSettings())); + this.chatContainer.addChild(new Spacer(1)); + } } this.chatContainer.addChild(new DynamicBorder()); } @@ -947,22 +976,15 @@ export class InteractiveMode { this.registerSignalHandlers(); - // Load changelog (only show new entries, skip for resumed sessions) + // Load changelogs (only show new entries, skip for resumed sessions) this.changelogMarkdown = this.getChangelogForDisplay(); + this.extensionChangelogUpdates = this.getExtensionChangelogUpdates(); - if (this.session.scopedModels.length > 0 && this.shouldShowStartupDetails()) { - const modelList = this.session.scopedModels - .map((sm) => { - const thinkingStr = sm.thinkingLevel ? `:${sm.thinkingLevel}` : ""; - return `${sm.model.id}${thinkingStr}`; - }) - .join(", "); - const cycleKeys = this.keybindings.getKeys("app.model.cycleForward"); - const cycleHint = - cycleKeys.length > 0 - ? theme.fg("muted", ` (${formatKeyText(cycleKeys.join("/"), { capitalize: true })} to cycle)`) - : ""; - console.log(theme.fg("dim", `Model scope: ${modelList}${cycleHint}`)); + // Fullscreen renders on the alternate screen, so text printed to stdout here would stay in the shell + // after exit. Fullscreen shows the notice in the header instead (#11508). + if (this.renderer.mode !== "fullscreen") { + const modelScopeNotice = this.getModelScopeNotice(); + if (modelScopeNotice) console.log(modelScopeNotice); } // Keep one component tree and remount it when changing renderers. @@ -1082,6 +1104,7 @@ export class InteractiveMode { this.builtInHeader = new Text("", 0, 0); this.headerContainer.addChild(this.builtInHeader); } + if (this.renderer.mode === "fullscreen") this.showFullscreenModelScopeNotice(); this.ui.requestRender(); // Ensure fd and rg are available after mounting the TUI (downloads if missing, adds to PATH via getBinDir) @@ -1358,6 +1381,68 @@ export class InteractiveMode { return undefined; } + private getExtensionChangelogUpdates(): ExtensionChangelogUpdate[] { + // Skip changelog for resumed/continued sessions (already have messages) + if (this.session.state.messages.length > 0) { + return []; + } + + const seenKeys = new Set(); + const updates: ExtensionChangelogUpdate[] = []; + for (const { sourceInfo } of this.session.resourceLoader.getExtensions().extensions) { + if (sourceInfo.origin !== "package" || !sourceInfo.baseDir || !sourceInfo.changelogPath) { + continue; + } + + const key = getSourcePackageKey(sourceInfo); + if (seenKeys.has(key)) { + continue; + } + seenKeys.add(key); + + const pkg = this.readPackageJson(sourceInfo.baseDir); + if (!pkg?.version) { + continue; + } + + const lastVersion = this.settingsManager.getExtensionChangelogVersion(key); + if (!lastVersion) { + // Fresh install - record the version, don't show changelog + this.settingsManager.setExtensionChangelogVersion(key, pkg.version); + continue; + } + + const newEntries = getNewEntries(parseChangelog(sourceInfo.changelogPath), lastVersion); + if (newEntries.length > 0) { + this.settingsManager.setExtensionChangelogVersion(key, pkg.version); + const name = pkg.name || sourceInfo.source; + // Prefix each "## [x.y.z]" heading with the package name so entries render like Pi's own. + const markdown = newEntries + .map((entry) => entry.content.replace(/^##\s+/, `## ${name} `)) + .join("\n\n") + .trim(); + updates.push({ name, version: pkg.version, markdown }); + } + } + + return updates; + } + + private readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined { + try { + const pkg = JSON.parse(stripBom(fs.readFileSync(path.join(packageRoot, "package.json"), "utf-8"))) as { + name?: unknown; + version?: unknown; + }; + return { + name: typeof pkg.name === "string" ? pkg.name : undefined, + version: typeof pkg.version === "string" ? pkg.version : undefined, + }; + } catch { + return undefined; + } + } + private reportInstallTelemetry(version: string): void { if (process.env.PI_OFFLINE) { return; @@ -1389,15 +1474,7 @@ export class InteractiveMode { // ========================================================================= private formatDisplayPath(p: string): string { - const home = os.homedir(); - let result = p; - - // Replace home directory with ~ - if (result.startsWith(home)) { - result = `~${result.slice(home.length)}`; - } - - return result; + return shortenHomePath(p); } private formatExtensionDisplayPath(path: string): string { @@ -1427,6 +1504,28 @@ export class InteractiveMode { } /** Startup details (model scope, loaded resources). Hidden by quietStartup: true or "header". */ + private getModelScopeNotice(): string | undefined { + if (this.session.scopedModels.length === 0 || !this.shouldShowStartupDetails()) return undefined; + const modelList = this.session.scopedModels + .map((sm) => { + const thinkingStr = sm.thinkingLevel ? `:${sm.thinkingLevel}` : ""; + return `${sm.model.id}${thinkingStr}`; + }) + .join(", "); + const cycleKeys = this.keybindings.getKeys("app.model.cycleForward"); + const cycleHint = + cycleKeys.length > 0 + ? theme.fg("muted", ` (${formatKeyText(cycleKeys.join("/"), { capitalize: true })} to cycle)`) + : ""; + return theme.fg("dim", `Model scope: ${modelList}${cycleHint}`); + } + + private showFullscreenModelScopeNotice(): void { + if (!this.getModelScopeNotice()) return; + this.headerContainer.addChild(new ThemedText(() => this.getModelScopeNotice() ?? "", 1, 0)); + this.headerContainer.addChild(new Spacer(1)); + } + private shouldShowStartupDetails(): boolean { return this.options.verbose === true || this.settingsManager.getQuietStartup() === false; } @@ -1918,7 +2017,7 @@ export class InteractiveMode { if (skillDiagnostics.length > 0) { const warningLines = () => this.formatDiagnostics(skillDiagnostics, sourceInfos); this.loadedResourcesContainer.addChild( - new ThemedText(() => `${theme.fg("warning", "[Skill conflicts]")}\n${warningLines()}`, 0, 0), + new ThemedText(() => `${theme.fg("warning", "[Skill warnings]")}\n${warningLines()}`, 0, 0), ); this.loadedResourcesContainer.addChild(new Spacer(1)); } @@ -2839,8 +2938,12 @@ export class InteractiveMode { // Save text from current editor before switching const currentText = this.editor.getText(); - this.disposeActiveSelector(); - this.editorContainer.clear(); + // An open extension dialog keeps the editor slot; the new editor is shown when the last dialog closes. + const dialogOpen = this.extensionDialogs.length > 0; + if (!dialogOpen) { + this.disposeActiveSelector(); + this.editorContainer.clear(); + } if (factory) { // Create the custom editor with tui, theme, and keybindings @@ -2898,7 +3001,7 @@ export class InteractiveMode { this.editor = this.defaultEditor; } - this.editorContainer.addChild(this.editor as Component); + if (!dialogOpen) this.editorContainer.addChild(this.editor as Component); if (this.activeStatusIndicator) { this.statusContainer.clear(); this.activeWorkingIndicatorEmbedded = this.setEditorWorkingStatusIndicator(this.activeStatusIndicator); @@ -2906,7 +3009,7 @@ export class InteractiveMode { this.statusContainer.addChild(this.activeStatusIndicator); } } - this.ui.setFocus(this.editor as Component); + if (!dialogOpen) this.ui.setFocus(this.editor as Component); this.ui.requestRender(); } @@ -2935,6 +3038,7 @@ export class InteractiveMode { overlay?: boolean; overlayOptions?: OverlayOptions | (() => OverlayOptions); onHandle?: (handle: OverlayHandle) => void; + blocked?: BlockedStatus; }, ): Promise { const savedText = this.editor.getText(); @@ -2988,7 +3092,7 @@ export class InteractiveMode { // Expose handle to caller for visibility control options?.onHandle?.(overlayHandle); } else { - this.pushExtensionDialog(c); + this.pushExtensionDialog(c, options?.blocked); } }) .catch((err) => { @@ -4596,6 +4700,8 @@ export class InteractiveMode { }); if (result.status === "complete") { this.editor.setText(result.content); + } else { + this.showStatus(result.message); } } finally { this.ui.start(); diff --git a/packages/coding-agent/src/modes/interactive/theme/theme.ts b/packages/coding-agent/src/modes/interactive/theme/theme.ts index 876ba9e86..922b23d9f 100644 --- a/packages/coding-agent/src/modes/interactive/theme/theme.ts +++ b/packages/coding-agent/src/modes/interactive/theme/theme.ts @@ -26,7 +26,7 @@ import chalk from "chalk"; import { getCustomThemesDir, getThemesDir } from "../../../config.ts"; import type { SourceInfo } from "../../../core/source-info.ts"; import { closeWatcher, watchWithErrorHandler } from "../../../utils/fs-watch.ts"; -import { highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; +import { getHighlightLanguageCount, highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; import { stripBom } from "../../../utils/text.ts"; import { generateSystemThemeColors, SYSTEM_THEME_NAME, terminalAppearance } from "./system-theme.ts"; import { @@ -658,6 +658,11 @@ export const theme: Theme = new Proxy({} as Theme, { }, }); +/** The theme instance behind the `theme` proxy, whose own identity never changes. */ +function getGlobalTheme(): Theme | undefined { + return (globalThis as Record)[THEME_KEY]; +} + function setGlobalTheme(t: Theme): void { (globalThis as Record)[THEME_KEY] = t; (globalThis as Record)[THEME_KEY_OLD] = t; @@ -944,7 +949,41 @@ export function highlightCode(code: string, lang?: string): string[] { try { return highlight(code, opts).split("\n"); } catch { - return code.split("\n"); + return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); + } +} + +/** + * Highlights code for one owner, such as a tool row, and reuses the last result while the code, the language, the + * theme and the registered grammars stay the same. Each owner keeps its own highlighter, so cached results live as long + * as the code they show instead of in a process-wide cache. + */ +export class CodeHighlighter { + private last?: { + code: string; + lang: string | undefined; + theme: Theme | undefined; + languageCount: number; + lines: readonly string[]; + }; + + /** Like {@link highlightCode}. The result is shared with later calls, so callers must not modify it. */ + highlight(code: string, lang: string | undefined): readonly string[] { + // Renderers cannot tell a theme change from any other redraw, so the highlighter checks what its result depends on. + const currentTheme = getGlobalTheme(); + const languageCount = getHighlightLanguageCount(); + const last = this.last; + if ( + last?.code === code && + last.lang === lang && + last.theme === currentTheme && + last.languageCount === languageCount + ) { + return last.lines; + } + const lines = highlightCode(code, lang); + this.last = { code, lang, theme: currentTheme, languageCount, lines }; + return lines; } } @@ -1035,26 +1074,7 @@ export function getMarkdownTheme(): MarkdownTheme { italic: (text: string) => theme.italic(text), underline: (text: string) => theme.underline(text), strikethrough: (text: string) => theme.strikethrough(text), - highlightCode: (code: string, lang?: string): string[] => { - // Validate language before highlighting to avoid stderr spam from cli-highlight - const validLang = lang && supportsLanguage(lang) ? lang : undefined; - // Skip highlighting when no valid language is specified. cli-highlight's - // auto-detection is unreliable and can misidentify prose as AppleScript, - // LiveCodeServer, etc., coloring random English words as keywords. - if (!validLang) { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - const opts = { - language: validLang, - ignoreIllegals: true, - theme: getCliHighlightTheme(theme), - }; - try { - return highlight(code, opts).split("\n"); - } catch { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - }, + highlightCode, }; } diff --git a/packages/coding-agent/src/package-manager-cli.ts b/packages/coding-agent/src/package-manager-cli.ts index 5696b66ad..eecf962ed 100644 --- a/packages/coding-agent/src/package-manager-cli.ts +++ b/packages/coding-agent/src/package-manager-cli.ts @@ -17,11 +17,14 @@ import { createProjectTrustContext } from "./cli/project-trust.ts"; import { APP_NAME, CONFIG_DIR_NAME, + detectInstallChange, detectInstallMethod, getAgentDir, + getConfiguredSelfUpdateArgv, getPackageDir, getSelfUpdateCommand, getSelfUpdateUnavailableInstruction, + makeConfiguredSelfUpdateCommand, PACKAGE_NAME, type SelfUpdateCommand, type SelfUpdatePackageTarget, @@ -638,6 +641,9 @@ function printSelfUpdateUnavailable( ): void { console.error(`error: ${APP_NAME} cannot self-update this installation.`); console.error(getSelfUpdateUnavailableInstruction(PACKAGE_NAME, npmCommand, updatePackageTarget)); + console.error( + `To make \`${APP_NAME} update\` run your own update command, set PI_SELF_UPDATE_COMMAND or the selfUpdateCommand setting.`, + ); const entrypoint = process.argv[1]; if (entrypoint) { @@ -735,6 +741,48 @@ async function runSelfUpdate(command: SelfUpdateCommand): Promise { } } +/** Run the user-configured self-update command, then report the version installed on disk. */ +async function runConfiguredSelfUpdate(argv: string[], plan: SelfUpdatePlan): Promise { + const command = makeConfiguredSelfUpdateCommand(argv, plan); + if (plan.note) { + printSelfUpdateNote(plan.note); + } + try { + await runSelfUpdate(command); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown self-update command error"; + console.error(chalk.red(`Error: ${message}`)); + process.exitCode = 1; + return; + } + // Exit code 0 does not prove an update. Wrappers such as mise or Nix install each version into a + // new directory and leave this installation untouched, so fall back to the pi on PATH. + const change = detectInstallChange(); + const installedVersion = + change?.kind === "updated" && change.version === plan.version ? change.version : readVersionOnPath(); + if (installedVersion === plan.version) { + console.log(chalk.green(`Updated ${APP_NAME} from ${VERSION} to ${plan.version}`)); + return; + } + console.log( + chalk.yellow( + `Ran ${command.display}, but \`${APP_NAME} --version\` reports ${installedVersion ?? "no version"}, not ${plan.version}.`, + ), + ); +} + +/** Version printed by `pi --version` as resolved from PATH, the pi the user runs next. */ +function readVersionOnPath(): string | undefined { + // PI_PACKAGE_DIR points at this installation and would make the new pi report the old version. + const { PI_PACKAGE_DIR: _, ...env } = process.env; + const result = spawnProcessSync(APP_NAME, ["--version"], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + env, + }); + return result.status === 0 ? result.stdout.trim() || undefined : undefined; +} + function prepareWindowsNpmSelfUpdate(): void { if (process.platform !== "win32") { return; @@ -982,6 +1030,8 @@ export async function handlePackageCommand( packageManager.setProgressCallback((event) => { if (event.type === "start") { process.stdout.write(chalk.dim(`${event.message}\n`)); + } else if (event.type === "progress" && event.message) { + process.stdout.write(chalk.yellow(`${event.message}\n`)); } }); @@ -1056,7 +1106,11 @@ export async function handlePackageCommand( } } if (updateTargetIncludesSelf(target)) { - const managedInstallRoot = getActiveManagedInstallRoot(); + const configuredSelfUpdateArgv = getConfiguredSelfUpdateArgv( + settingsManager.getGlobalSettings().selfUpdateCommand, + ); + // A configured command is explicit user intent, so it replaces the managed install path too. + const managedInstallRoot = configuredSelfUpdateArgv ? undefined : getActiveManagedInstallRoot(); if (managedInstallRoot && options.force) { console.error( chalk.red( @@ -1070,6 +1124,10 @@ export async function handlePackageCommand( if (!selfUpdatePlan.shouldRun) { return true; } + if (configuredSelfUpdateArgv) { + await runConfiguredSelfUpdate(configuredSelfUpdateArgv, selfUpdatePlan); + return true; + } if (managedInstallRoot) { if (selfUpdatePlan.note) { printSelfUpdateNote(selfUpdatePlan.note); diff --git a/packages/coding-agent/src/utils/ansi.ts b/packages/coding-agent/src/utils/ansi.ts index cad5eb65b..6d49fc1bc 100644 --- a/packages/coding-agent/src/utils/ansi.ts +++ b/packages/coding-agent/src/utils/ansi.ts @@ -32,8 +32,11 @@ const ST = "(?:\\u0007|\\u001B\\u005C|\\u009C)"; // OSC sequences: ESC ] ... ST const OSC_START = "\\u001B\\]"; -// CSI and related: ESC/C1, optional intermediates, optional params (supports ; and :), then final byte -const CSI_START = "[\\u001B\\u009B][[\\]()#;?]*(?:\\d{1,4}(?:[;:]\\d{0,4})*)?"; +// CSI and related: ESC/C1, optional intermediates, optional params (supports ; and :), then final byte. +// A private prefix (<, =, >) is accepted only right after a real CSI introducer (ESC [ or 0x9B), so +// ESC[<35;59;1M (SGR mouse report) is stripped whole. After a bare ESC these bytes stay final bytes: +// ESC = and ESC > (keypad modes) must not swallow the digits of the text that follows. +const CSI_START = "(?:(?:\\u001B\\[|\\u009B)[<=>]|[\\u001B\\u009B][[\\]()#;?]*)(?:\\d{1,4}(?:[;:]\\d{0,4})*)?"; const CSI_FINAL = "[\\dA-PR-TZcf-nq-uy=><~]"; // Complete sequences. OSC is non-greedy until the first ST. diff --git a/packages/coding-agent/src/utils/clipboard-command.ts b/packages/coding-agent/src/utils/clipboard-command.ts index 0cddbf59c..66601ab8c 100644 --- a/packages/coding-agent/src/utils/clipboard-command.ts +++ b/packages/coding-agent/src/utils/clipboard-command.ts @@ -10,8 +10,12 @@ export function runClipboardCommand( // Clipboard writers can daemonize. Do not give them output pipes to retain. // Detach on POSIX so the daemon leaves pi's process group and Ctrl+Z suspend // (SIGTSTP to the group) does not stop the clipboard owner (#11179). + // Run from / (the current drive root on Windows): clipboard tools ignore the cwd, and WSL cannot start + // Windows executables (powershell.exe) when the cwd's \\wsl.localhost\... path + // exceeds MAX_PATH (#11592). This also keeps daemons from pinning pi's cwd. const child = spawn(command, args, { stdio: ["pipe", options?.input === undefined ? "pipe" : "ignore", "ignore"], + cwd: "/", detached: process.platform !== "win32", windowsHide: true, }); diff --git a/packages/coding-agent/src/utils/git.ts b/packages/coding-agent/src/utils/git.ts index 1314edea7..2b5a1369d 100644 --- a/packages/coding-agent/src/utils/git.ts +++ b/packages/coding-agent/src/utils/git.ts @@ -14,7 +14,7 @@ export type GitSource = { path: string; /** Git ref (branch, tag, commit) if specified */ ref?: string; - /** True if ref was specified (package won't be auto-updated) */ + /** True if ref was specified. `pi update` reconciles the checkout to the ref but does not move the ref. */ pinned: boolean; }; diff --git a/packages/coding-agent/src/utils/highlight-js.d.ts b/packages/coding-agent/src/utils/highlight-js.d.ts index d4b71174e..c4c951fce 100644 --- a/packages/coding-agent/src/utils/highlight-js.d.ts +++ b/packages/coding-agent/src/utils/highlight-js.d.ts @@ -17,6 +17,7 @@ interface HighlightJsApi { highlight(code: string, options: HighlightJsOptions): HighlightJsResult; highlightAuto(code: string, languageSubset?: string[]): HighlightJsResult; getLanguage(name: string): HighlightJsLanguageDefinition | undefined; + listLanguages(): string[]; registerLanguage(name: string, language: HighlightJsLanguageFactory): void; } diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts index 3f80111ff..d4a4d8cab 100644 --- a/packages/coding-agent/src/utils/image-resize-core.ts +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -134,9 +134,11 @@ export async function resizeImageInProcess( let currentHeight = targetHeight; while (true) { - const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); - for (const candidate of candidates) { - if (candidate.encodedSize < opts.maxBytes) { + const [png, jpeg, ...lowerQualityJpegs] = tryEncodings(currentWidth, currentHeight, qualitySteps); + // Prefer the smaller of PNG and JPEG at the configured quality; lower JPEG quality only if neither fits. + const preferred = jpeg && jpeg.encodedSize < png.encodedSize ? [jpeg, png] : [png, jpeg]; + for (const candidate of [...preferred, ...lowerQualityJpegs]) { + if (candidate && candidate.encodedSize < opts.maxBytes) { return { data: candidate.data, mimeType: candidate.mimeType, diff --git a/packages/coding-agent/src/utils/open-browser.ts b/packages/coding-agent/src/utils/open-browser.ts index 435e23f97..727603a7d 100644 --- a/packages/coding-agent/src/utils/open-browser.ts +++ b/packages/coding-agent/src/utils/open-browser.ts @@ -18,7 +18,10 @@ export function openBrowser(target: string): void { // spawn reports launcher failures (for example, missing xdg-open) via an // error event. Browser launch is best-effort: callers still present the target // to the user, so keep the launcher failure from becoming a process crash. - spawn(cmd, args, { stdio: "ignore", detached: true }) + // On WSL, xdg-open launches explorer.exe, which fails when the cwd's + // \\wsl.localhost\... path exceeds MAX_PATH (#11592). The target is a URL, + // so no launcher needs pi's cwd. + spawn(cmd, args, { stdio: "ignore", detached: true, cwd: "/" }) .on("error", () => {}) .unref(); } diff --git a/packages/coding-agent/src/utils/paths.ts b/packages/coding-agent/src/utils/paths.ts index 6a5a0cfb8..cf77b5bff 100644 --- a/packages/coding-agent/src/utils/paths.ts +++ b/packages/coding-agent/src/utils/paths.ts @@ -1,6 +1,6 @@ import { realpathSync, statSync } from "node:fs"; import { homedir } from "node:os"; -import { isAbsolute, join, resolve as nodeResolvePath, relative, sep } from "node:path"; +import { isAbsolute, join, resolve as nodeResolvePath, posix, relative, sep, win32 } from "node:path"; import { fileURLToPath } from "node:url"; import { spawnProcessSync } from "./child-process.ts"; @@ -33,6 +33,50 @@ export function canonicalizePath(path: string): string { } } +/** + * Key for comparing filesystem paths: the resolved absolute path, case-folded + * on Windows because Windows paths are case-insensitive (`c:\Users` and + * `C:\Users` are the same directory). NTFS can enable per-directory case + * sensitivity, but it is rarely used outside WSL interop, so it is ignored + * here (https://learn.microsoft.com/en-us/windows/wsl/case-sensitivity). + * Does not touch the filesystem; call `canonicalizePath` first when symlinks + * must compare equal. Use only for comparison, never for display or + * filesystem access. + */ +export function getPathComparisonKey(path: string): string { + const pathApi = getPlatformPathApi(); + const resolvedPath = pathApi.resolve(path); + return pathApi === win32 ? resolvedPath.toLowerCase() : resolvedPath; +} + +/** `node:path` rules for the current platform, read at call time so tests can simulate Windows. */ +function getPlatformPathApi(): typeof win32 { + return process.platform === "win32" ? win32 : posix; +} + +/** True when both paths name the same location (see `getPathComparisonKey`). */ +export function pathsEqual(left: string, right: string): boolean { + return getPathComparisonKey(left) === getPathComparisonKey(right); +} + +/** True when `child` is `parent` or inside it (see `getPathComparisonKey`). */ +export function isPathWithin(parent: string, child: string): boolean { + const parentKey = getPathComparisonKey(parent); + const childKey = getPathComparisonKey(child); + if (childKey === parentKey) return true; + // Filesystem roots (`/`, `C:\`) already end with a separator. + const pathSep = getPlatformPathApi().sep; + return childKey.startsWith(parentKey.endsWith(pathSep) ? parentKey : `${parentKey}${pathSep}`); +} + +/** Replace a leading home directory with `~` for display. Relative paths are returned unchanged. */ +export function shortenHomePath(path: string, homeDir: string = homedir()): string { + const pathApi = getPlatformPathApi(); + if (!homeDir || !pathApi.isAbsolute(path) || !isPathWithin(homeDir, path)) return path; + const relativePath = pathApi.relative(homeDir, path); + return relativePath ? `~${pathApi.sep}${relativePath}` : "~"; +} + export function getFileRevision(path: string): string | undefined { try { const stats = statSync(path, { bigint: true }); diff --git a/packages/coding-agent/src/utils/syntax-highlight.ts b/packages/coding-agent/src/utils/syntax-highlight.ts index 1ee1986b9..91375bb01 100644 --- a/packages/coding-agent/src/utils/syntax-highlight.ts +++ b/packages/coding-agent/src/utils/syntax-highlight.ts @@ -217,3 +217,11 @@ export function highlight(code: string, options: HighlightOptions = {}): string export function supportsLanguage(name: string): boolean { return hljs.getLanguage(name) !== undefined; } + +/** + * Number of registered grammars. It grows when the deferred grammars load, which can change the highlighting of + * code that embeds another language, such as HTML in a JavaScript template literal. + */ +export function getHighlightLanguageCount(): number { + return hljs.listLanguages().length; +} diff --git a/packages/coding-agent/test/ansi-utils.test.ts b/packages/coding-agent/test/ansi-utils.test.ts index 15bb43140..f108e482c 100644 --- a/packages/coding-agent/test/ansi-utils.test.ts +++ b/packages/coding-agent/test/ansi-utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { stripAnsi } from "../src/utils/ansi.ts"; +import { splitIncompleteAnsiSuffix, stripAnsi } from "../src/utils/ansi.ts"; function referenceAnsiRegex(): RegExp { const ST = "(?:\\u0007|\\u001B\\u005C|\\u009C)"; @@ -107,4 +107,29 @@ describe("stripAnsi", () => { const input = "a\x1b[31mred\x1b[0m\x1b]8;;https://example.com\x07link\x1b]8;;\x07z"; expect(stripAnsi(input)).toBe("aredlinkz"); }); + + // https://github.com/earendil-works/pi/issues/11377 + it("strips private-parameter CSI sequences with their parameters", () => { + expect(stripAnsi("\x1b[<35;59;1Mstep 1")).toBe("step 1"); + expect(stripAnsi("a\x1b[<0;10;20mb")).toBe("ab"); + expect(stripAnsi("a\x1b[>0;276;0cb")).toBe("ab"); + expect(stripAnsi("a\x1b[>cb")).toBe("ab"); + expect(stripAnsi("a\x1b[=5hb")).toBe("ab"); + expect(stripAnsi("a\x9b<35;59;1Mb")).toBe("ab"); + }); + + // https://github.com/earendil-works/pi/issues/11377 + it("keeps digits after bare ESC keypad-mode sequences", () => { + expect(stripAnsi("\x1b[?1h\x1b=123 tests passed")).toBe("123 tests passed"); + expect(stripAnsi("\x1b[?1l\x1b>42 files")).toBe("42 files"); + expect(stripAnsi("\x1b<10 items")).toBe("10 items"); + }); +}); + +describe("splitIncompleteAnsiSuffix", () => { + // https://github.com/earendil-works/pi/issues/11377 + it("holds back unfinished private-parameter CSI sequences", () => { + expect(splitIncompleteAnsiSuffix("x\x1b[<35;59")).toEqual({ complete: "x", pending: "\x1b[<35;59" }); + expect(splitIncompleteAnsiSuffix("x\x1b=")).toEqual({ complete: "x\x1b=", pending: "" }); + }); }); diff --git a/packages/coding-agent/test/assistant-message.test.ts b/packages/coding-agent/test/assistant-message.test.ts index ea2618aac..2cf368915 100644 --- a/packages/coding-agent/test/assistant-message.test.ts +++ b/packages/coding-agent/test/assistant-message.test.ts @@ -1,10 +1,11 @@ import type { AssistantMessage } from "@earendil-works/pi-ai"; import type { TuiMouseEvent } from "@earendil-works/pi-tui"; -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { AssistantMessageComponent } from "../src/modes/interactive/components/assistant-message.ts"; import { UserMessageComponent } from "../src/modes/interactive/components/user-message.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; import { stripAnsi } from "../src/utils/ansi.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; @@ -34,6 +35,37 @@ function createAssistantMessage( } describe("AssistantMessageComponent", () => { + afterEach(() => vi.restoreAllMocks()); + + test("reuses its Markdown across streaming and its end, and renders like a new component after a theme change", () => { + initTheme("dark"); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const streaming: boolean[] = []; + const code = "```ts\nconst a = 1;\n```"; + const content = (text: string): AssistantMessage["content"] => [ + { type: "thinking", thinking: "Thinking with **bold** text" }, + { type: "text", text }, + ]; + const component = new AssistantMessageComponent(undefined, false, undefined, "Thinking...", 1, [ + (markdown, { isStreaming }) => { + streaming.push(isStreaming); + return markdown; + }, + ]); + component.updateContent(createAssistantMessage(content(code)), true); + component.render(80); + const message = createAssistantMessage(content(`${code}\n\nThe end.`)); + component.updateContent(message, false); + component.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + expect(streaming.at(-1)).toBe(false); + + initTheme("light"); + component.invalidate(); + expect(component.render(80)).toEqual(new AssistantMessageComponent(message).render(80)); + initTheme("dark"); + }); + test("adds OSC 133 zone markers to assistant messages without tool calls", () => { initTheme("dark"); @@ -92,6 +124,48 @@ describe("AssistantMessageComponent", () => { expect(rendered).toContain("answer"); }); + test("renders nothing for hidden thinking with an empty label", () => { + initTheme("dark"); + + const thinkingOnly = new AssistantMessageComponent( + createAssistantMessage( + [ + { type: "thinking", thinking: "reasoning" }, + { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } }, + ], + { stopReason: "toolUse" }, + ), + true, + undefined, + "", + ); + expect(thinkingOnly.render(80)).toEqual([]); + + const withText = new AssistantMessageComponent( + createAssistantMessage([ + { type: "thinking", thinking: "reasoning" }, + { type: "text", text: "answer" }, + ]), + true, + undefined, + "", + ); + expect(withText.render(80).map((line) => stripAnsi(line).trim())).toEqual(["", "answer"]); + }); + + test("keeps label row and spacing for hidden thinking with a label", () => { + initTheme("dark"); + + const component = new AssistantMessageComponent( + createAssistantMessage([ + { type: "thinking", thinking: "reasoning" }, + { type: "text", text: "answer" }, + ]), + true, + ); + expect(component.render(80).map((line) => stripAnsi(line).trim())).toEqual(["", "Thinking...", "", "answer"]); + }); + test("collapses individual thinking runs when clicked", () => { initTheme("dark"); const component = new AssistantMessageComponent( diff --git a/packages/coding-agent/test/branch-summarization.test.ts b/packages/coding-agent/test/branch-summarization.test.ts index 7018e969f..3164a8773 100644 --- a/packages/coding-agent/test/branch-summarization.test.ts +++ b/packages/coding-agent/test/branch-summarization.test.ts @@ -7,7 +7,7 @@ import { type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; -import { generateBranchSummary } from "../src/core/compaction/index.ts"; +import { generateBranchSummary, prepareBranchEntries } from "../src/core/compaction/index.ts"; import type { SessionEntry } from "../src/core/session-manager.ts"; const model: Model<"anthropic-messages"> = { @@ -44,6 +44,26 @@ function response(content: AssistantMessage["content"]): AssistantMessage { } describe("branch summarization", () => { + it("passes the thinking level only for reasoning-capable models", async () => { + const seen: Array = []; + const streamFn: StreamFn = (_model, _context, options) => { + seen.push(options?.reasoning); + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => + stream.push({ type: "done", reason: "stop", message: response([{ type: "text", text: "summary" }]) }), + ); + return stream; + }; + const reasoningModel = { ...model, reasoning: true }; + const signal = new AbortController().signal; + + await generateBranchSummary(entries, { model: reasoningModel, signal, streamFn, thinkingLevel: "xhigh" }); + await generateBranchSummary(entries, { model: reasoningModel, signal, streamFn, thinkingLevel: "off" }); + await generateBranchSummary(entries, { model, signal, streamFn, thinkingLevel: "high" }); + + expect(seen).toEqual(["xhigh", undefined, undefined]); + }); + it("does not override tool choice for branch summaries", async () => { let requestOptions: SimpleStreamOptions | undefined; const streamFn: StreamFn = (_model, _context, options) => { @@ -111,4 +131,54 @@ describe("branch summarization", () => { "Branch summarization failed: generation hit the token cap and the summary is incomplete", ); }); + + it("applies context edits to branch entries", () => { + const ts = (n: number) => new Date(n).toISOString(); + const branch: SessionEntry[] = [ + { + type: "message", + id: "u1", + parentId: null, + timestamp: ts(1), + message: { role: "user", content: "Investigate login", timestamp: 1 }, + }, + { + type: "message", + id: "failed", + parentId: "u1", + timestamp: ts(2), + message: fauxAssistantMessage("FAILED_ATTEMPT", { stopReason: "error", errorMessage: "lost" }), + }, + { + type: "context_edit", + id: "e1", + parentId: "failed", + timestamp: ts(3), + targetId: "failed", + replacement: null, + }, + { + type: "message", + id: "ok", + parentId: "e1", + timestamp: ts(4), + message: fauxAssistantMessage("ORIGINAL_TEXT"), + }, + { + type: "context_edit", + id: "e2", + parentId: "ok", + timestamp: ts(5), + targetId: "ok", + replacement: { content: "REPLACED_TEXT" }, + }, + ]; + + const serialized = JSON.stringify(prepareBranchEntries(branch).messages); + + expect(serialized).toContain("Investigate login"); + expect(serialized).not.toContain("FAILED_ATTEMPT"); + expect(serialized).not.toContain("ORIGINAL_TEXT"); + expect(serialized).toContain("REPLACED_TEXT"); + }); }); diff --git a/packages/coding-agent/test/clipboard-command.test.ts b/packages/coding-agent/test/clipboard-command.test.ts index 8e250f817..5be4220b0 100644 --- a/packages/coding-agent/test/clipboard-command.test.ts +++ b/packages/coding-agent/test/clipboard-command.test.ts @@ -1,3 +1,4 @@ +import { resolve } from "node:path"; import { describe, expect, test } from "vitest"; import { runClipboardCommand } from "../src/utils/clipboard-command.ts"; @@ -39,6 +40,11 @@ describe("clipboard commands", () => { expect(ownPgid).toMatch(/^\d+$/); expect(childPgid).not.toBe(ownPgid); }); + // #11592: WSL cannot start Windows executables from a cwd whose UNC path exceeds MAX_PATH. + test("runs commands from the filesystem root", async () => { + const cwd = await runClipboardCommand(process.execPath, ["-e", "process.stdout.write(process.cwd())"]); + expect(cwd?.toString("utf8")).toBe(resolve("/")); + }); test("rejects output above the buffer limit", async () => { expect( await runClipboardCommand(process.execPath, ["-e", "process.stdout.write(Buffer.alloc(1024))"], { diff --git a/packages/coding-agent/test/code-highlighter.test.ts b/packages/coding-agent/test/code-highlighter.test.ts new file mode 100644 index 000000000..b0d0433b4 --- /dev/null +++ b/packages/coding-agent/test/code-highlighter.test.ts @@ -0,0 +1,71 @@ +import type { TUI } from "@earendil-works/pi-tui"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { writeRenderers } from "../src/core/tools/renderers/write.ts"; +import { codemodeRenderers } from "../src/extensions/codemode/renderer.ts"; +import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; +import { CodeHighlighter, highlightCode, initTheme } from "../src/modes/interactive/theme/theme.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; + +const tui = { requestRender: () => {} } as unknown as TUI; + +describe("CodeHighlighter", () => { + beforeEach(() => initTheme("dark")); + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("reuses its last result until the code, the theme or the grammars change", async () => { + const code = 'const page = html`
ok
`;'; + const highlighter = new CodeHighlighter(); + const first = highlighter.highlight(code, "javascript"); + expect(highlighter.highlight(code, "javascript")).toBe(first); + expect(highlighter.highlight("const a = 1;", "javascript")).not.toBe(first); + + initTheme("light"); + expect(highlighter.highlight(code, "javascript")).toEqual(highlightCode(code, "javascript")); + const light = highlighter.highlight(code, "javascript"); + await syntaxHighlight.loadAllHighlightLanguages(); + expect(highlighter.highlight(code, "javascript")).not.toEqual(light); + }); + + it("codemode highlights the script once across progress, timer ticks, expanding and completion", () => { + vi.useFakeTimers(); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const row = new ToolExecutionComponent( + "codemode", + "c", + { code: "await tools.read({});" }, + {}, + codemodeRenderers, + tui, + "/", + ); + row.setArgsComplete(); + row.markExecutionStarted(); + row.updateResult({ content: [], details: { calls: [] }, isError: false }, true); + vi.advanceTimersByTime(2_000); + row.setExpanded(true); + row.updateResult({ content: [{ type: "text", text: "done" }], details: { calls: [] }, isError: false }, false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + }); + + it("write highlights unchanged first lines and a complete file only once", () => { + const content = Array.from({ length: 60 }, (_, i) => `const v${i} = ${i};`).join("\n"); + const row = new ToolExecutionComponent("write", "w", { path: "a.ts", content }, {}, writeRenderers, tui, "/"); + row.updateArgs({ path: "a.ts", content: `${content} //` }); + row.render(80); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + // Only the changed last line is highlighted again. + row.updateArgs({ path: "a.ts", content: `${content} // more` }); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + + row.setArgsComplete(); + row.setExpanded(true); + row.setExpanded(false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/coding-agent/test/codemode-renderer.test.ts b/packages/coding-agent/test/codemode-renderer.test.ts index 7c802da71..571933226 100644 --- a/packages/coding-agent/test/codemode-renderer.test.ts +++ b/packages/coding-agent/test/codemode-renderer.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { ToolRenderContext } from "../src/core/extensions/types.ts"; import { codemodeRenderers } from "../src/extensions/codemode/renderer.ts"; import type { CodemodeToolDetails } from "../src/extensions/codemode/tool.ts"; +import { keyHint, keyText } from "../src/modes/interactive/components/keybinding-hints.ts"; import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; import { initTheme, theme } from "../src/modes/interactive/theme/theme.ts"; @@ -29,6 +30,8 @@ function render( isError, durationMs: undefined, outputPad: 1, + keyText, + keyHint, } satisfies ToolRenderContext; const component = codemodeRenderers.renderResult?.( result, diff --git a/packages/coding-agent/test/config.test.ts b/packages/coding-agent/test/config.test.ts index ff652cae3..ff2c8a733 100644 --- a/packages/coding-agent/test/config.test.ts +++ b/packages/coding-agent/test/config.test.ts @@ -6,9 +6,11 @@ import { detectInstallChange, detectInstallMethod, findNodePackageDir, + getConfiguredSelfUpdateArgv, getSelfUpdateCommand, getSelfUpdateUnavailableInstruction, getUpdateInstruction, + makeConfiguredSelfUpdateCommand, } from "../src/config.ts"; const execPathDescriptor = Object.getOwnPropertyDescriptor(process, "execPath"); @@ -182,6 +184,33 @@ describe("detectInstallMethod", () => { ); }); + // Issue #7756 + test("does not name pnpm for unmanaged installs under PNPM_HOME", () => { + tempDir = mkdtempSync(join(tmpdir(), "pi-pnpm-home-")); + const packageDir = join( + tempDir, + ".local", + "share", + "pnpm", + "global-nub", + "5", + "node_modules", + "@earendil-works", + "pi-coding-agent", + ); + mkdirSync(packageDir, { recursive: true }); + process.env.PI_PACKAGE_DIR = packageDir; + process.env.PATH = ""; + process.argv[1] = join(packageDir, "dist", "cli.js"); + setExecPath(join(packageDir, "dist", "cli.js")); + + expect(detectInstallMethod()).toBe("pnpm"); + expect(getSelfUpdateCommand("@earendil-works/pi-coding-agent")).toBeUndefined(); + expect(getSelfUpdateUnavailableInstruction("@earendil-works/pi-coding-agent")).toBe( + "Update @earendil-works/pi-coding-agent using the package manager, wrapper, or source checkout that provides this installation.", + ); + }); + test("self-updates npm installs from custom prefixes", () => { const { prefix } = createNpmPrefixInstall(); @@ -462,3 +491,54 @@ describe("detectInstallChange", () => { expect(detectInstallChange(join(installDir, "package.json"))).toEqual({ kind: "removed" }); }); }); + +describe("configured self-update command", () => { + const originalSelfUpdateCommand = process.env.PI_SELF_UPDATE_COMMAND; + + afterEach(() => { + if (originalSelfUpdateCommand === undefined) { + delete process.env.PI_SELF_UPDATE_COMMAND; + } else { + process.env.PI_SELF_UPDATE_COMMAND = originalSelfUpdateCommand; + } + }); + + test("prefers PI_SELF_UPDATE_COMMAND over the setting and splits it on whitespace", () => { + process.env.PI_SELF_UPDATE_COMMAND = " nub upgrade {installSpec} "; + + expect(getConfiguredSelfUpdateArgv(["mise", "upgrade"])).toEqual(["nub", "upgrade", "{installSpec}"]); + }); + + test("uses the setting when PI_SELF_UPDATE_COMMAND is unset or blank", () => { + process.env.PI_SELF_UPDATE_COMMAND = " "; + + expect(getConfiguredSelfUpdateArgv(["mise", "upgrade", "npm:{packageName}"])).toEqual([ + "mise", + "upgrade", + "npm:{packageName}", + ]); + expect(getConfiguredSelfUpdateArgv([])).toBeUndefined(); + expect(getConfiguredSelfUpdateArgv(undefined)).toBeUndefined(); + }); + + test("fills placeholders in every argument", () => { + const command = makeConfiguredSelfUpdateCommand( + ["wrapper", "install", "{installSpec}", "--name={packageName}", "--version", "{version}", "{unknown}"], + { packageName: "@earendil-works/pi-coding-agent", version: "1.2.3" }, + ); + + expect(command).toEqual({ + command: "wrapper", + args: [ + "install", + "@earendil-works/pi-coding-agent@1.2.3", + "--name=@earendil-works/pi-coding-agent", + "--version", + "1.2.3", + "{unknown}", + ], + display: + "wrapper install @earendil-works/pi-coding-agent@1.2.3 --name=@earendil-works/pi-coding-agent --version 1.2.3 {unknown}", + }); + }); +}); diff --git a/packages/coding-agent/test/custom-editor-history-keybindings.test.ts b/packages/coding-agent/test/custom-editor-history-keybindings.test.ts deleted file mode 100644 index a4de5200d..000000000 --- a/packages/coding-agent/test/custom-editor-history-keybindings.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { setKeybindings, TuiMainScreen } from "@earendil-works/pi-tui"; -import { afterEach, describe, expect, it } from "vitest"; -import { defaultEditorTheme } from "../../tui/test/test-themes.ts"; -import { VirtualTerminal } from "../../tui/test/virtual-terminal.ts"; -import { KeybindingsManager } from "../src/core/keybindings.ts"; -import { CustomEditor } from "../src/modes/interactive/components/custom-editor.ts"; - -afterEach(() => { - setKeybindings(new KeybindingsManager()); -}); - -describe("CustomEditor prompt history keybindings", () => { - it("gives an explicit history binding precedence over model cycling", () => { - const keybindings = new KeybindingsManager({ - "tui.editor.historyPrevious": "ctrl+p", - "tui.editor.historyNext": "ctrl+n", - }); - setKeybindings(keybindings); - const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); - let modelCycles = 0; - editor.onAction("app.model.cycleForward", () => { - modelCycles++; - }); - editor.addToHistory("previous prompt"); - editor.setText("draft"); - - editor.handleInput("\x10"); // Ctrl+P - expect(editor.getText()).toBe("previous prompt"); - expect(modelCycles).toBe(0); - - editor.handleInput("\x0e"); // Ctrl+N - expect(editor.getText()).toBe("draft"); - }); -}); diff --git a/packages/coding-agent/test/custom-editor-keybindings.test.ts b/packages/coding-agent/test/custom-editor-keybindings.test.ts new file mode 100644 index 000000000..355951a15 --- /dev/null +++ b/packages/coding-agent/test/custom-editor-keybindings.test.ts @@ -0,0 +1,79 @@ +import { setKeybindings, TuiMainScreen } from "@earendil-works/pi-tui"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { defaultEditorTheme } from "../../tui/test/test-themes.ts"; +import { VirtualTerminal } from "../../tui/test/virtual-terminal.ts"; +import { KeybindingsManager } from "../src/core/keybindings.ts"; +import { CustomEditor } from "../src/modes/interactive/components/custom-editor.ts"; + +afterEach(() => { + setKeybindings(new KeybindingsManager()); +}); + +describe("CustomEditor prompt history keybindings", () => { + it("gives an explicit history binding precedence over model cycling", () => { + const keybindings = new KeybindingsManager({ + "tui.editor.historyPrevious": "ctrl+p", + "tui.editor.historyNext": "ctrl+n", + }); + setKeybindings(keybindings); + const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); + let modelCycles = 0; + editor.onAction("app.model.cycleForward", () => { + modelCycles++; + }); + editor.addToHistory("previous prompt"); + editor.setText("draft"); + + editor.handleInput("\x10"); // Ctrl+P + expect(editor.getText()).toBe("previous prompt"); + expect(modelCycles).toBe(0); + + editor.handleInput("\x0e"); // Ctrl+N + expect(editor.getText()).toBe("draft"); + }); +}); + +describe("CustomEditor selection page keybindings", () => { + it("pages an open autocomplete menu before pasting an image with the same key (#7629)", async () => { + // Bind both explicitly: the paste default is Alt+V on Windows and WSL. + const keybindings = new KeybindingsManager({ + "tui.select.pageDown": "ctrl+v", + "app.clipboard.pasteImage": "ctrl+v", + }); + setKeybindings(keybindings); + const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); + let pastes = 0; + editor.onPasteImage = () => { + pastes++; + }; + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion: (lines, cursorLine, _cursorCol, item) => ({ + lines: lines.map((line, index) => (index === cursorLine ? item.value : line)), + cursorLine, + cursorCol: item.value.length, + }), + }); + + editor.handleInput("/"); + await vi.waitFor(() => expect(editor.isShowingAutocomplete()).toBe(true)); + editor.handleInput("\x16"); // Ctrl+V + editor.handleInput("\r"); + + expect(pastes).toBe(0); + expect(submitted).toBe("/command-5"); + + editor.handleInput("\x16"); // Ctrl+V with autocomplete closed pastes again + expect(pastes).toBe(1); + }); +}); diff --git a/packages/coding-agent/test/custom-message.test.ts b/packages/coding-agent/test/custom-message.test.ts index cdc598010..3077a6288 100644 --- a/packages/coding-agent/test/custom-message.test.ts +++ b/packages/coding-agent/test/custom-message.test.ts @@ -23,7 +23,7 @@ describe("CustomMessageComponent", () => { }; const component = new CustomMessageComponent(message, renderer, undefined, 1); - expect(optionsSeen).toEqual([{ expanded: false, outputPad: 1 }]); + expect(optionsSeen).toMatchObject([{ expanded: false, outputPad: 1 }]); expect( component .render(40) @@ -33,7 +33,7 @@ describe("CustomMessageComponent", () => { component.setOutputPad(0); - expect(optionsSeen.at(-1)).toEqual({ expanded: false, outputPad: 0 }); + expect(optionsSeen.at(-1)).toMatchObject({ expanded: false, outputPad: 0 }); expect( component .render(40) diff --git a/packages/coding-agent/test/deleted-cwd.test.ts b/packages/coding-agent/test/deleted-cwd.test.ts new file mode 100644 index 000000000..3bf60a8dd --- /dev/null +++ b/packages/coding-agent/test/deleted-cwd.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { main } from "../src/main.ts"; + +// Regression test for https://github.com/earendil-works/pi/issues/11509 +describe("startup in a deleted working directory", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + test("prints an actionable error and exits 1 instead of throwing", async () => { + vi.spyOn(process, "cwd").mockImplementation(() => { + throw Object.assign(new Error("ENOENT: no such file or directory, uv_cwd"), { + code: "ENOENT", + syscall: "uv_cwd", + }); + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as typeof process.exit); + + await expect(main(["--no-extensions", "--version"])).rejects.toThrow("process.exit(1)"); + expect(exitSpy).toHaveBeenCalledWith(1); + const stderr = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stderr).toContain("The current working directory no longer exists"); + }); + + test("rethrows unrelated cwd errors", async () => { + vi.spyOn(process, "cwd").mockImplementation(() => { + throw Object.assign(new Error("EACCES"), { code: "EACCES" }); + }); + + await expect(main(["--no-extensions", "--version"])).rejects.toThrow("EACCES"); + }); +}); diff --git a/packages/coding-agent/test/exec-runtime-limit.test.ts b/packages/coding-agent/test/exec-runtime-limit.test.ts new file mode 100644 index 000000000..879a45911 --- /dev/null +++ b/packages/coding-agent/test/exec-runtime-limit.test.ts @@ -0,0 +1,25 @@ +import * as buffer from "node:buffer"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:buffer", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // 32-bit Node's maximum is 16 bytes below 256 MiB. + constants: { ...actual.constants, MAX_STRING_LENGTH: 268_435_440 }, + }; +}); + +// Regression tests for https://github.com/earendil-works/pi/issues/11360 +it("executes with the default cap when the runtime string limit is below 256 MiB", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("ok")'], process.cwd()); + + expect(result).toEqual({ stdout: "ok", stderr: "", code: 0, killed: false, truncated: false }); +}); + +it("still rejects an explicit cap above the runtime string limit", async () => { + await expect( + execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer: buffer.constants.MAX_STRING_LENGTH + 1 }), + ).rejects.toThrow(RangeError); +}); diff --git a/packages/coding-agent/test/exec-utf8-chunks.test.ts b/packages/coding-agent/test/exec-utf8-chunks.test.ts new file mode 100644 index 000000000..5dad0aae0 --- /dev/null +++ b/packages/coding-agent/test/exec-utf8-chunks.test.ts @@ -0,0 +1,53 @@ +import type * as ChildProcessModule from "node:child_process"; +import { ChildProcess, spawn } from "node:child_process"; +import { PassThrough } from "node:stream"; +import { setImmediate } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: vi.fn(), +})); + +// #11360: real streams control chunk boundaries without relying on OS pipe timing. +// Only process creation and termination are stubbed; decoding and process waiting are real. +it.each(["stdout", "stderr"] as const)( + "discards pending %s UTF-8 bytes when either stream exceeds the cap", + async (stream) => { + const child = new ChildProcess(); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + child.stdout = stdout; + child.stderr = stderr; + vi.mocked(spawn).mockReturnValueOnce(child); + vi.spyOn(child, "kill").mockImplementation(() => { + queueMicrotask(() => { + child.emit("exit", null, "SIGTERM"); + child.emit("close", null, "SIGTERM"); + }); + return true; + }); + + const resultPromise = execCommand("test-command", [], process.cwd(), { maxBuffer: 3 }); + const output = stream === "stdout" ? stdout : stderr; + const otherOutput = stream === "stdout" ? stderr : stdout; + // The decoder first returns "a" and buffers the first byte of €. + output.write(Buffer.from([0x61, 0xe2])); + await setImmediate(); + // The other decoder also buffers the first byte of € before the shared cap is exceeded. + otherOutput.write(Buffer.from([0xe2])); + await setImmediate(); + output.write(Buffer.from([0x82, 0xac])); + + const result = await resultPromise; + expect(result).toMatchObject({ + stdout: stream === "stdout" ? "a" : "", + stderr: stream === "stderr" ? "a" : "", + truncated: true, + killed: true, + }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(3); + }, +); diff --git a/packages/coding-agent/test/exec.test.ts b/packages/coding-agent/test/exec.test.ts index 96232a8b3..df5a63817 100644 --- a/packages/coding-agent/test/exec.test.ts +++ b/packages/coding-agent/test/exec.test.ts @@ -1,3 +1,4 @@ +import { constants as bufferConstants } from "node:buffer"; import { describe, expect, it } from "vitest"; import { execCommand } from "../src/core/exec.ts"; @@ -9,6 +10,7 @@ describe("execCommand", () => { expect(result).toMatchObject({ code: 1, killed: false, + truncated: false, stdout: "", }); expect(result.stderr).toMatch(/ENOENT/); @@ -17,6 +19,110 @@ describe("execCommand", () => { it("keeps stderr empty when a command exits silently", async () => { const result = await execCommand(process.execPath, ["-e", "process.exit(1)"], process.cwd()); - expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false }); + expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false, truncated: false }); }); + + // Regression tests for https://github.com/earendil-works/pi/issues/11360 + it.each([ + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + bufferConstants.MAX_STRING_LENGTH + 1, + ])("rejects invalid maxBuffer %s instead of attempting execution", async (maxBuffer) => { + // A missing command normally resolves to a failed result; invalid options must reject instead. + await expect( + execCommand("pi-command-that-does-not-exist-11360", [], process.cwd(), { maxBuffer }), + ).rejects.toThrow(RangeError); + }); + + it.each([0, bufferConstants.MAX_STRING_LENGTH])("accepts maxBuffer boundary %s", async (maxBuffer) => { + const result = await execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer }); + + expect(result).toEqual({ stdout: "", stderr: "", code: 0, killed: false, truncated: false }); + }); + + it("truncates all output when maxBuffer is zero", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("x")'], process.cwd(), { + maxBuffer: 0, + }); + + expect(result).toMatchObject({ stdout: "", stderr: "", truncated: true, killed: true }); + expect(result.code).not.toBe(0); + }); + + it("caps combined output at maxBuffer, terminates the command, and reports truncation", async () => { + const script = + 'process.stderr.write("e".repeat(600)); process.stdout.write("o".repeat(600)); setInterval(() => process.stdout.write("x".repeat(1000)), 10);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBe(1000); + }); + + // #11360: a byte cap must not turn a partial UTF-8 character into replacement output. + it.each([ + ["stdout", "€", 1, ""], + ["stderr", "€", 1, ""], + ["stdout", "a€🙂", 5, "a€"], + ["stderr", "a€🙂", 5, "a€"], + ] as const)( + "keeps complete UTF-8 characters from %s output %s capped at %i bytes", + async (stream, output, maxBuffer, expected) => { + const script = `process.${stream}.write(${JSON.stringify(output)})`; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer }); + + expect(result).toMatchObject({ [stream]: expected, truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(maxBuffer); + }, + ); + + it("still flushes incomplete UTF-8 at normal EOF when output was not truncated", async () => { + const script = "process.stdout.write(Buffer.from([0xe2])); process.stderr.write(Buffer.from([0xf0]));"; + const result = await execCommand(process.execPath, ["-e", script], process.cwd()); + + expect(result).toEqual({ stdout: "�", stderr: "�", code: 0, killed: false, truncated: false }); + }); + + it("leaves output up to maxBuffer unchanged", async () => { + const result = await execCommand( + process.execPath, + ["-e", 'process.stdout.write("a".repeat(1000))'], + process.cwd(), + { + maxBuffer: 1000, + }, + ); + + expect(result).toEqual({ stdout: "a".repeat(1000), stderr: "", code: 0, killed: false, truncated: false }); + }); + + it.skipIf(process.platform === "win32")( + "returns when a shell's child keeps writing after the shell is terminated", + async () => { + // SIGTERM kills only `sh`; `yes` keeps the output pipes open and writing. + const result = await execCommand("sh", ["-c", "yes; echo done"], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ killed: true, truncated: true }); + expect(result.stdout).toBe("y\n".repeat(500)); + }, + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "force-kills a command that ignores SIGTERM", + async () => { + // The handler is installed before the first write, so the kill triggered by maxBuffer + // cannot arrive before the child ignores SIGTERM. + const script = 'process.on("SIGTERM", () => {}); process.stdout.write("xx"); setInterval(() => {}, 1000);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1 }); + + expect(result).toMatchObject({ stdout: "x", killed: true, truncated: true }); + expect(result.code).not.toBe(0); + }, + 15_000, + ); }); diff --git a/packages/coding-agent/test/extension-keybinding-hints.test.ts b/packages/coding-agent/test/extension-keybinding-hints.test.ts new file mode 100644 index 000000000..7e0a3ebe9 --- /dev/null +++ b/packages/coding-agent/test/extension-keybinding-hints.test.ts @@ -0,0 +1,82 @@ +// Issue #4748: extensions that import keyText/keyHint can resolve their own package copy, which does not see the +// host's keybindings. The host passes its own keyText/keyHint to renderers instead. +import { setKeybindings, Text, type TUI } from "@earendil-works/pi-tui"; +import { Type } from "typebox"; +import { beforeAll, describe, expect, test } from "vitest"; +import type { KeybindingHints, ToolDefinition } from "../src/core/extensions/types.ts"; +import { KeybindingsManager } from "../src/core/keybindings.ts"; +import type { CustomMessage } from "../src/core/messages.ts"; +import type { CustomEntry } from "../src/core/session-manager.ts"; +import { CustomEntryComponent } from "../src/modes/interactive/components/custom-entry.ts"; +import { CustomMessageComponent } from "../src/modes/interactive/components/custom-message.ts"; +import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; +import { initTheme } from "../src/modes/interactive/theme/theme.ts"; +import { stripAnsi } from "../src/utils/ansi.ts"; + +function hintText(hints: KeybindingHints): string { + return `${hints.keyText("app.tools.expand")} | ${stripAnsi(hints.keyHint("app.tools.expand", "to expand"))}`; +} + +const EXPECTED = "ctrl+e | ctrl+e to expand"; + +describe("host keybinding hints for extensions", () => { + beforeAll(() => { + initTheme("dark"); + // A user override proves the hints read the host's configured keybindings, not the defaults. + setKeybindings(new KeybindingsManager({ "app.tools.expand": "ctrl+e" })); + }); + + test("tool renderers receive keyText and keyHint in the render context", () => { + const seen: string[] = []; + const toolDefinition: ToolDefinition = { + name: "custom_tool", + label: "custom_tool", + description: "custom tool", + parameters: Type.Any(), + execute: async () => ({ content: [{ type: "text", text: "ok" }], details: {} }), + renderCall: (_args, _theme, context) => { + seen.push(hintText(context)); + return new Text("call", 0, 0); + }, + }; + const tui = { requestRender: () => {} } as unknown as TUI; + new ToolExecutionComponent("custom_tool", "tool-1", {}, {}, toolDefinition, tui, process.cwd()).render(80); + + expect(seen.at(-1)).toBe(EXPECTED); + }); + + test("message renderers receive keyText and keyHint in the render options", () => { + const seen: string[] = []; + const message: CustomMessage = { + role: "custom", + customType: "test", + content: "custom", + display: true, + timestamp: Date.now(), + }; + new CustomMessageComponent(message, (_message, options) => { + seen.push(hintText(options)); + return new Text("custom", 0, 0); + }); + + expect(seen).toEqual([EXPECTED]); + }); + + test("entry renderers receive keyText and keyHint in the render options", () => { + const seen: string[] = []; + const entry: CustomEntry = { + type: "custom", + id: "entry-1", + parentId: null, + timestamp: new Date().toISOString(), + customType: "test", + data: undefined, + }; + new CustomEntryComponent(entry, (_entry, options) => { + seen.push(hintText(options)); + return new Text("custom", 0, 0); + }); + + expect(seen).toEqual([EXPECTED]); + }); +}); diff --git a/packages/coding-agent/test/external-editor.test.ts b/packages/coding-agent/test/external-editor.test.ts index 5cbbc93b3..e9a540821 100644 --- a/packages/coding-agent/test/external-editor.test.ts +++ b/packages/coding-agent/test/external-editor.test.ts @@ -52,7 +52,7 @@ describe("editInExternalEditor", () => { it("keeps the original content when the editor exits unsuccessfully", async () => { const { result, capture } = await runExternalEditor("--fail"); - expect(result).toEqual({ status: "failed" }); + expect(result).toEqual({ status: "failed", message: "External editor exited with code 1; prompt unchanged" }); expect(existsSync(dirname(capture.filePath))).toBe(false); }); it("returns empty content when the editor clears the prompt", async () => { @@ -60,4 +60,13 @@ describe("editInExternalEditor", () => { expect(result).toEqual({ status: "complete", content: "" }); }); + + // #11592: launch failures must be reported, not silently ignored. + it.skipIf(process.platform === "win32")("reports an editor that cannot be launched", async () => { + const command = "pi-external-editor-does-not-exist"; + expect(await editInExternalEditor({ command, content: "original" })).toEqual({ + status: "failed", + message: `Failed to launch external editor: ${command}`, + }); + }); }); diff --git a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts index 7c8741c7e..7903a4078 100644 --- a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts +++ b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts @@ -7,8 +7,8 @@ import type { ExecResult, ExtensionAPI, ExtensionContext } from "../src/core/ext type AgentEndHandler = (event: { type: "agent_end" }, ctx: ExtensionContext) => Promise; -const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; -const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false }; +const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; +const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false, truncated: false }; /** Standard exec results for a clean repo tracking origin/main, not in a merge. */ function withUpstream(results: Map): Map { diff --git a/packages/coding-agent/test/git-update.test.ts b/packages/coding-agent/test/git-update.test.ts index 80308277c..8b771f688 100644 --- a/packages/coding-agent/test/git-update.test.ts +++ b/packages/coding-agent/test/git-update.test.ts @@ -178,6 +178,59 @@ describe("DefaultPackageManager git update", () => { }); }); + describe("git clean failures", () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install dependencies and retry the clean later when git clean fails", async () => { + setupRemoteAndInstall(); + const newCommit = createCommit( + remoteDir, + "package.json", + JSON.stringify({ name: "test-extension", version: "1.0.0" }), + "Add package.json", + ); + + const executedCommands: string[] = []; + let failClean = true; + const managerWithInternals = packageManager as unknown as { + runCommand: (command: string, args: string[], options?: { cwd?: string }) => Promise; + }; + managerWithInternals.runCommand = async (command, args, options) => { + const commandLine = `${command} ${args.join(" ")}`; + executedCommands.push(commandLine); + if (command === "npm") return; + if (commandLine === "git clean -fdx" && failClean) { + throw new Error("git clean -fdx failed with code 1"); + } + const result = spawnSync(command, args, { cwd: options?.cwd, encoding: "utf-8" }); + if (result.status !== 0) { + throw new Error(`Command failed: ${commandLine}\n${result.stderr}`); + } + }; + const progressMessages: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "progress" && event.message) progressMessages.push(event.message); + }); + const markerPath = join(agentDir, "git", "github.com", "test", ".extension.pi-update-incomplete"); + + await packageManager.update(); + + expect(getCurrentCommit(installedDir)).toBe(newCommit); + expect(executedCommands).toContain("git clean -fdx"); + expect(executedCommands.some((command) => command.startsWith("npm install"))).toBe(true); + expect(progressMessages.some((message) => message.includes("Could not remove all untracked files"))).toBe( + true, + ); + expect(existsSync(markerPath)).toBe(true); + + failClean = false; + executedCommands.length = 0; + await packageManager.update(); + + expect(executedCommands).toContain("git clean -fdx"); + expect(existsSync(markerPath)).toBe(false); + }); + }); + describe("force-push scenarios", () => { it("should handle complete history rewrite", async () => { setupRemoteAndInstall(); diff --git a/packages/coding-agent/test/image-processing.test.ts b/packages/coding-agent/test/image-processing.test.ts index 1bf3ea53e..0d112a3a7 100644 --- a/packages/coding-agent/test/image-processing.test.ts +++ b/packages/coding-agent/test/image-processing.test.ts @@ -5,6 +5,8 @@ import { describe, expect, it } from "vitest"; import { convertToPng, loadPngTranscoder } from "../src/utils/image-convert.ts"; import { formatDimensionNote, resizeImage } from "../src/utils/image-resize.ts"; +import { resizeImageInProcess } from "../src/utils/image-resize-core.ts"; +import { loadPhoton } from "../src/utils/photon.ts"; // Small 2x2 red PNG image (base64) - generated with ImageMagick const TINY_PNG = @@ -25,6 +27,31 @@ const MEDIUM_PNG_100x100 = const LARGE_PNG_200x200 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRP///////wlY99wAAAAHdElNRQfqAQ4AMzqU03BFAAAEEElEQVR42u3dvWqUURSF4XXMRAyIhSAWQvwnWtjaWusFx9bW1kKD/wELESxEiBjDjJdwysOC57mCXW32+zGcGbtdACpsdi+T8Xz1GABzm91xMl6sHgNgblzcT/Y+rh4DYG6cv0/2H64eA2BOEgI1fHQHaoyzbXIwVo8BMCcJgRqSEKgxft9Lrn5aPQbA3Ph1klw7Wj0GwJwkBGpYWECN8fMiub63egyAORcWUMPCAmqMH3eTG59XjwEwN76/S24+Wj0GwJwkBGpYWECN8e1fcmuzegyAORcWUMPCAmqM09vJ4dfVYwDMjS9vkzuPV48BMCcJgRqeSAZqjA/nyYP91WMAzElCoIYkBGqMk8Pk6HT1GABzm+3L5JIkBApIQqCGj+5AjfHmb/Lk8uoxAOYkIVBDEgI1LCygxmZ7nFyShEABFxZQw8ICaozXf5KnV1aPATDnwgJqWFhADQsLqOFnDUANFxZQw8ICaoxXZ8mzg9VjAMy5sIAaFhZQw3tYQA1/QgHUkIRADUkI1JCEQA1JCNSQhEANFxZQwzcsoIYkBGpIQqCGJARqSEKghiQEalhYQA1/QgHUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEaFhZQw4ujQA0XFlDDwgJqSEKghgsLqGFhATUsLKCGb1hADRcWUMPCAmpIQqCGCwuoYWEBNbyHBdTw4ihQQxICNSQhUEMSAjUkIVBDEgI1XFhADd+wgBqSEKghCYEakhCoIQmBGpIQqGFhATW8OArUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEa49lB8ups9RgAc14cBWpIQqCGhQXUkIRADRcWUMPCAmqMp1eS139WjwEw5xsWUEMSAjUsLKCGJARquLCAGhYWUGM8uZy8+bt6DIA5L44CNSQhUMMTyUCNcXSanByuHgNgThICNSQhUGM82E8+nK8eA2DOzxqAGpIQqDHuPE6+vF09BsDcOPyanN5ePQbAnCQEavgdFlBj3Nok3/6tHgNgzoujQA1JCNQYNx8l39+tHgNgbtz4nPy4u3oMgDlJCNSwsIAa4/pe8vNi9RgAc37WANSQhECNce0o+XWyegyAuXH1U/L73uoxAOYkIVDDwgJqjIORnG1XjwEw508ogBqSEKgx9h8m5+9XjwEwN/Y+Jhf3V48BMCcJgRpjPE+2x6vHAJgbSbLbrR4DYO4/GqiSgXN+ksgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDEtMTRUMDA6NTE6NTcrMDA6MDDpysx4AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTAxLTE0VDAwOjUxOjU3KzAwOjAwmJd0xAAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wMS0xNFQwMDo1MTo1NyswMDowMM+CVRsAAAAASUVORK5CYII="; +/** Photo-like JPEG: smooth gradients with light deterministic noise, so PNG compresses it poorly. */ +async function photoLikeJpeg(width: number, height: number): Promise { + const photon = await loadPhoton(); + if (!photon) throw new Error("Photon not available"); + const pixels = new Uint8Array(width * height * 4); + let seed = 1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + const noise = seed % 6; + const i = (y * width + x) * 4; + pixels[i] = ((x / width) * 200 + noise) | 0; + pixels[i + 1] = ((y / height) * 200 + noise) | 0; + pixels[i + 2] = (128 + Math.sin(x / 30) * 60 + noise) | 0; + pixels[i + 3] = 255; + } + } + const image = new photon.PhotonImage(pixels, width, height); + try { + return image.get_bytes_jpeg(85); + } finally { + image.free(); + } +} + function imageBytes(base64Data: string): Uint8Array { return Buffer.from(base64Data, "base64"); } @@ -166,6 +193,18 @@ describe("resizeImage", () => { expect(result).toBeNull(); }); + // Regression test for #11551: PNG was returned whenever it fit, even when JPEG was much smaller. + it("should pick JPEG over a larger PNG when resizing a photo", async () => { + const input = await photoLikeJpeg(1200, 800); + const result = await resizeImageInProcess(input, "image/jpeg", { maxWidth: 1000, maxHeight: 1000 }); + + expect(result).not.toBeNull(); + expect(result!.wasResized).toBe(true); + expect(result!.mimeType).toBe("image/jpeg"); + expect(result!.width).toBe(1000); + expect(result!.data.length).toBeLessThan(Math.ceil(input.byteLength / 3) * 4 * 2); + }); + it("should handle JPEG input", async () => { const result = await resizeImage(imageBytes(TINY_JPEG), "image/jpeg", { maxWidth: 100, diff --git a/packages/coding-agent/test/input-transform-streaming-example.test.ts b/packages/coding-agent/test/input-transform-streaming-example.test.ts index faf95b6ba..0eb1ccf60 100644 --- a/packages/coding-agent/test/input-transform-streaming-example.test.ts +++ b/packages/coding-agent/test/input-transform-streaming-example.test.ts @@ -35,9 +35,9 @@ function setup(execResult: ExecResult) { describe("input-transform-streaming example", () => { const diffOutput = " src/index.ts | 5 ++---\n 1 file changed, 2 insertions(+), 3 deletions(-)"; - const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false }; - const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; - const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false }; + const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false, truncated: false }; + const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; + const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false, truncated: false }; it("skips exec during steering", async () => { const { emit, exec } = setup(gitSuccess); diff --git a/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts b/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts new file mode 100644 index 000000000..f588ba474 --- /dev/null +++ b/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts @@ -0,0 +1,139 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { InMemorySettingsStorage, SettingsManager } from "../src/core/settings-manager.ts"; +import { getSourcePackageKey, type SourceInfo } from "../src/core/source-info.ts"; +import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts"; + +type ExtensionChangelogUpdate = { name: string; version: string; markdown: string }; + +type ExtensionChangelogContext = { + session: { + state: { messages: unknown[] }; + resourceLoader: { getExtensions(): { extensions: Array<{ sourceInfo: SourceInfo }> } }; + }; + settingsManager: SettingsManager; + readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined; +}; + +const prototype = InteractiveMode.prototype as unknown as { + getExtensionChangelogUpdates(this: ExtensionChangelogContext): ExtensionChangelogUpdate[]; + readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined; +}; + +const CHANGELOG = `# Changelog + +## [Unreleased] + +- Not released yet + +## [1.2.0] + +- Added feature B + +## [1.1.0] + +- Added feature A + +## [1.0.0] + +- Initial release +`; + +describe("InteractiveMode extension changelog", () => { + let tempDir: string; + let packageRoot: string; + let sourceInfo: SourceInfo; + let settingsManager: SettingsManager; + let messages: unknown[]; + + function writePackage(version: string): void { + writeFileSync(join(packageRoot, "package.json"), JSON.stringify({ name: "pi-demo-extension", version })); + } + + function createContext(): ExtensionChangelogContext { + return { + session: { + state: { messages }, + resourceLoader: { + getExtensions: () => ({ + // Two extensions from the same package must only produce one changelog section. + extensions: [{ sourceInfo }, { sourceInfo: { ...sourceInfo, path: join(packageRoot, "b.ts") } }], + }), + }, + }, + settingsManager, + readPackageJson: prototype.readPackageJson, + }; + } + + function display(): ExtensionChangelogUpdate[] { + return prototype.getExtensionChangelogUpdates.call(createContext()); + } + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "pi-extension-changelog-")); + packageRoot = join(tempDir, "pkg"); + mkdirSync(packageRoot); + writeFileSync(join(packageRoot, "CHANGELOG.md"), CHANGELOG); + sourceInfo = { + path: join(packageRoot, "a.ts"), + source: "npm:pi-demo-extension", + scope: "user", + origin: "package", + baseDir: packageRoot, + changelogPath: join(packageRoot, "CHANGELOG.md"), + }; + settingsManager = SettingsManager.fromStorage(new InMemorySettingsStorage()); + messages = []; + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + test("records the version on first launch without showing the changelog", () => { + writePackage("1.0.0"); + + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.0.0"); + }); + + test("shows entries newer than the last seen version, headed by package name", () => { + writePackage("1.0.0"); + display(); + writePackage("1.2.0"); + + expect(display()).toEqual([ + { + name: "pi-demo-extension", + version: "1.2.0", + markdown: + "## pi-demo-extension [1.2.0]\n\n- Added feature B\n\n## pi-demo-extension [1.1.0]\n\n- Added feature A", + }, + ]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.2.0"); + expect(display()).toEqual([]); + }); + + test("skips resumed sessions", () => { + writePackage("1.0.0"); + display(); + writePackage("1.2.0"); + messages.push({ role: "user", content: "hi" }); + + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.0.0"); + }); + + test("ignores non-package extensions and packages without a changelog", () => { + writePackage("1.0.0"); + sourceInfo = { ...sourceInfo, origin: "top-level" }; + expect(display()).toEqual([]); + + sourceInfo = { ...sourceInfo, origin: "package", changelogPath: undefined }; + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts index d296ff2b4..91a9d1966 100644 --- a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts +++ b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts @@ -18,7 +18,11 @@ type DialogMethods = { showExtensionConfirm(title: string, message: string, opts?: { signal?: AbortSignal }): Promise; showExtensionCustom( factory: (tui: unknown, theme: unknown, keybindings: unknown, done: (result: T) => void) => Component, + options?: { blocked?: { kind: "permission" | "question" | "auth"; message: string } }, ): Promise; + setCustomEditorComponent( + factory: ((tui: unknown, theme: unknown, keybindings: unknown) => Component) | undefined, + ): void; }; function createContext(): DialogContext & DialogMethods { @@ -149,4 +153,61 @@ describe("InteractiveMode extension dialogs", () => { await expect(custom).resolves.toBe("kept"); expect(context.focused).toBe(context.editor); }); + + // Regression for #11428: inline custom dialogs could not report blocked program status. + it("reports blocked status for an inline custom component that opts in", async () => { + const context = createContext(); + const dones: Array<(result: string) => void> = []; + const first = context.showExtensionCustom( + (_tui, _theme, _keybindings, done) => { + dones.push(done); + return new Text("approval"); + }, + { blocked: { kind: "permission", message: "Approve?" } }, + ); + await vi.waitFor(() => expect(visible(context)).toContain("approval")); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", { + kind: "permission", + message: "Approve?", + }); + + const loader = context.showExtensionCustom((_tui, _theme, _keybindings, done) => { + dones.push(done); + return new Text("loading"); + }); + await vi.waitFor(() => expect(visible(context)).toContain("loading")); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", undefined); + + dones[1]("loaded"); + await expect(loader).resolves.toBe("loaded"); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", { + kind: "permission", + message: "Approve?", + }); + + dones[0]("yes"); + await expect(first).resolves.toBe("yes"); + expect(context.focused).toBe(context.editor); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", undefined); + }); + + // Regression for #11431: replacing the editor while a dialog was open hid the dialog and left it unanswerable. + it("keeps an open dialog visible when the editor component is replaced", async () => { + const context = createContext(); + (context as unknown as { defaultEditor: unknown }).defaultEditor = context.editor; + const confirmed = context.showExtensionConfirm("Title", "Continue?"); + const customEditor: Component & { getText(): string; setText(text: string): void } = { + render: () => ["custom editor"], + invalidate: () => {}, + getText: () => "", + setText: () => {}, + }; + context.setCustomEditorComponent(() => customEditor); + + expect(visible(context)).toContain("Title"); + context.focused?.handleInput?.("\r"); + await expect(confirmed).resolves.toBe(true); + expect(context.focused).toBe(customEditor); + expect(context.editorContainer.children).toEqual([customEditor]); + }); }); diff --git a/packages/coding-agent/test/interactive-mode-model-scope.test.ts b/packages/coding-agent/test/interactive-mode-model-scope.test.ts new file mode 100644 index 000000000..66a2ef82a --- /dev/null +++ b/packages/coding-agent/test/interactive-mode-model-scope.test.ts @@ -0,0 +1,58 @@ +import { Container } from "@earendil-works/pi-tui"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts"; +import { initTheme } from "../src/modes/interactive/theme/theme.ts"; + +type ModelScopeContext = { + session: { scopedModels: Array<{ model: { id: string }; thinkingLevel?: string }> }; + keybindings: { getKeys: (keybinding: string) => string[] }; + headerContainer: Container; + shouldShowStartupDetails: () => boolean; + getModelScopeNotice: () => string | undefined; +}; + +type InteractiveModePrivate = { + getModelScopeNotice(this: ModelScopeContext): string | undefined; + showFullscreenModelScopeNotice(this: ModelScopeContext): void; +}; + +const interactiveModePrototype = InteractiveMode.prototype as unknown as InteractiveModePrivate; + +function createContext(showDetails = true): ModelScopeContext { + return { + session: { scopedModels: [{ model: { id: "model-a" }, thinkingLevel: "high" }, { model: { id: "model-b" } }] }, + keybindings: { getKeys: () => ["ctrl+p"] }, + headerContainer: new Container(), + shouldShowStartupDetails: () => showDetails, + getModelScopeNotice: interactiveModePrototype.getModelScopeNotice, + }; +} + +function stripAnsi(text: string): string { + return text.replace(/\x1b\[[0-9;]*m/g, ""); +} + +describe("InteractiveMode model scope notice", () => { + beforeEach(() => initTheme("dark")); + + // Regression test for #11508: fullscreen must not print the notice to the shell's main screen. + it("renders the model scope inside the fullscreen header instead of printing it", () => { + const context = createContext(); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + interactiveModePrototype.showFullscreenModelScopeNotice.call(context); + expect(log).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + + const rendered = stripAnsi(context.headerContainer.render(120).join("\n")); + expect(rendered).toContain("Model scope: model-a:high, model-b (Ctrl+P to cycle)"); + }); + + it("adds nothing when startup details are hidden", () => { + const context = createContext(false); + interactiveModePrototype.showFullscreenModelScopeNotice.call(context); + expect(context.headerContainer.children).toHaveLength(0); + }); +}); diff --git a/packages/coding-agent/test/interactive-mode-status.test.ts b/packages/coding-agent/test/interactive-mode-status.test.ts index 72de1fa96..3e26d845c 100644 --- a/packages/coding-agent/test/interactive-mode-status.test.ts +++ b/packages/coding-agent/test/interactive-mode-status.test.ts @@ -1366,7 +1366,7 @@ describe("InteractiveMode.showLoadedResources", () => { }); const output = renderAll(fakeThis.loadedResourcesContainer); - expect(output).toContain("[Skill conflicts]"); + expect(output).toContain("[Skill warnings]"); expect(output).not.toContain("[Skills]"); }); }); diff --git a/packages/coding-agent/test/login-dialog-device-code.test.ts b/packages/coding-agent/test/login-dialog-device-code.test.ts index 7d5222ec5..269326c43 100644 --- a/packages/coding-agent/test/login-dialog-device-code.test.ts +++ b/packages/coding-agent/test/login-dialog-device-code.test.ts @@ -140,3 +140,47 @@ describe("device-code login", () => { expect(onComplete).not.toHaveBeenCalled(); }); }); + +describe("login URLs", () => { + beforeAll(() => initTheme("dark")); + beforeEach(() => { + vi.resetAllMocks(); + setKeybindings(new KeybindingsManager()); + }); + + it("opens only http(s) sign-in URLs", () => { + const { dialog } = createDialog(); + dialog.showAuth("file:///Users/victim/Downloads/payload.app"); + expect(openBrowser).not.toHaveBeenCalled(); + + dialog.showAuth("https://example.com/authorize"); + expect(openBrowser).toHaveBeenCalledExactlyOnceWith("https://example.com/authorize"); + }); + + it("keeps control characters in URLs from ending terminal hyperlinks", () => { + const { dialog } = createDialog(); + dialog.showDeviceCode({ ...device, verificationUri: "https://example.com/device\x07\x1b]52;c;Zm9v\x07" }); + dialog.showInfo("Docs", [{ label: "Help", url: "https://example.com/help\x07\x1b]52;c;Zm9v\x07" }]); + const output = dialog.render(120).join("\n"); + + expect(output).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("https://example.com/device%07%1B]52;c;Zm9v"); + expect(render(dialog)).toContain("Help: https://example.com/help%07%1B]52;c;Zm9v"); + }); + + it("replaces control characters in URLs that are not web URLs", () => { + const injected = "x\x07\x1b]52;c;Zm9v\x07"; + const { dialog } = createDialog(); + dialog.showAuth(`file:///tmp/${injected}`); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + + dialog.showDeviceCode({ ...device, verificationUri: injected }); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("x ]52;c;Zm9v"); + + dialog.showInfo("Docs", [{ label: "Help", url: injected }]); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("Help: x ]52;c;Zm9v"); + expect(openBrowser).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/coding-agent/test/mcp-command.test.ts b/packages/coding-agent/test/mcp-command.test.ts index 7bde1e11b..2e9a0a1fa 100644 --- a/packages/coding-agent/test/mcp-command.test.ts +++ b/packages/coding-agent/test/mcp-command.test.ts @@ -2,6 +2,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { ProjectTrustStore } from "../src/core/trust-manager.ts"; import { runMcpCommand } from "../src/extensions/mcp/cli.ts"; const FIXTURE = resolve(import.meta.dirname, "../../mcp/test/fixtures/stdio-server.mjs"); @@ -74,6 +75,30 @@ describe("pi mcp", () => { expect((await run(["frobnicate"], servers)).exitCode).toBe(1); }); + it("suggests a configured server for a mistyped name", async () => { + expect((await run(["login", "fixtures"], servers)).output).toBe( + 'No MCP server named "fixtures". Did you mean "fixture"? Configured: fixture, broken, parked.', + ); + }); + + it("reports options as typed", async () => { + expect((await run(["list", "-l"], servers)).output).toContain("Unknown option -l."); + }); + + it("restricts login and logout to project servers with --local", async () => { + const { agentDir } = await run(["add", "-l", "local", "--", "node", "server.js"], servers); + new ProjectTrustStore(agentDir).set(agentDir, true); + + expect(await run(["logout", "-l", "local"], undefined, agentDir)).toMatchObject({ + exitCode: 1, + output: 'MCP server "local" does not use OAuth. Only HTTP servers without an Authorization header do.', + }); + expect(await run(["login", "-l", "fixture"], undefined, agentDir)).toMatchObject({ + exitCode: 1, + output: `MCP server "fixture" is defined in ${join(agentDir, "mcp.json")}, not in the project; omit --local.`, + }); + }); + it("adds stdio servers and passes options after the command through", async () => { const added = await run( ["add", "--env", "A=1", "--env", "B=x=y", "files", "--", "npx", "-y", "server", "--root", "."], @@ -146,6 +171,15 @@ describe("pi mcp", () => { expect(readConfig(join(agentDir, "mcp.json")).mcpServers).toMatchObject({ sentry: { url: "https://mcp.sentry.dev/mcp", oauth: { clientId: "pi", clientName: "Claude Code" } }, }); + + await run( + ["add", "linear", "--url", "https://mcp.linear.app/mcp", "--oauth-client-registration", "dcr"], + undefined, + agentDir, + ); + expect(readConfig(join(agentDir, "mcp.json")).mcpServers).toMatchObject({ + linear: { url: "https://mcp.linear.app/mcp", oauth: { clientRegistration: "dcr" } }, + }); }); it("rejects invalid add invocations without writing", async () => { @@ -158,6 +192,7 @@ describe("pi mcp", () => { ["add", "x", "--header", "A=1", "--", "cmd"], ["add", "x", "--env", "NOVALUE", "--", "cmd"], ["add", "x", "--exposure", "loud", "--", "cmd"], + ["add", "x", "--url", "https://example.com", "--oauth-client-registration", "maybe"], ]; for (const args of cases) { const result = await run(args, undefined); diff --git a/packages/coding-agent/test/mcp-conformance/README.md b/packages/coding-agent/test/mcp-conformance/README.md index 8aaa36690..dbdbc5bf1 100644 --- a/packages/coding-agent/test/mcp-conformance/README.md +++ b/packages/coding-agent/test/mcp-conformance/README.md @@ -39,7 +39,7 @@ node packages/coding-agent/test/mcp-conformance/run.ts --update-baseline ## Known failures - `elicitation-sep1034-client-defaults`: pi does not support elicitation. -- `auth/basic-cimd`: pi registers clients dynamically instead of using a Client ID Metadata Document. +- `auth/basic-cimd`: pi uses its Client ID Metadata Document on pi.dev, not the suite's `https://conformance-test.local/client-metadata.json`. - `auth/scope-retry-limit`: `pi-client` fails by design; the server never accepts the granted scope. 2026-07-28 is not covered: it is a stateless protocol pi does not implement. diff --git a/packages/coding-agent/test/mcp-conformance/baseline.json b/packages/coding-agent/test/mcp-conformance/baseline.json index 0401d64b0..8e7f205ae 100644 --- a/packages/coding-agent/test/mcp-conformance/baseline.json +++ b/packages/coding-agent/test/mcp-conformance/baseline.json @@ -214,7 +214,6 @@ "passing": [ "authorization-request", "authorization-server-metadata", - "client-registration", "pi-client", "pkce-code-challenge-sent", "pkce-code-verifier-sent", diff --git a/packages/coding-agent/test/mcp-extension.test.ts b/packages/coding-agent/test/mcp-extension.test.ts index 0948434fd..7bc18ca43 100644 --- a/packages/coding-agent/test/mcp-extension.test.ts +++ b/packages/coding-agent/test/mcp-extension.test.ts @@ -208,7 +208,8 @@ describe("MCP config", () => { url: "https://a.example/mcp", oauth: { clientRegistration: "cimd", callbackUrl: "http://localhost/callback" }, }, - badRegistration: { url: "https://a.example/mcp", oauth: { clientRegistration: "auto" } }, + auto: { url: "https://a.example/mcp", oauth: { clientRegistration: "auto", clientName: "x" } }, + badRegistration: { url: "https://a.example/mcp", oauth: { clientRegistration: "oidc" } }, cimdClient: { url: "https://a.example/mcp", oauth: { clientRegistration: "cimd", clientId: "x" } }, cimdPath: { url: "https://a.example/mcp", @@ -219,14 +220,14 @@ describe("MCP config", () => { {}, ); const { servers, errors } = loadMcpConfig({ ...paths, projectTrusted: false }); - expect(servers.map((server) => server.name)).toEqual(["ok", "ipv6", "same", "named", "metadata", "cimd"]); + expect(servers.map((server) => server.name)).toEqual(["ok", "ipv6", "same", "named", "metadata", "cimd", "auto"]); expect(errors).toEqual([ expect.stringContaining('server "remote": oauth.callbackUrl must be an http URI on localhost'), expect.stringContaining('server "both": oauth.callbackUrl and oauth.callbackPort name different ports'), expect.stringContaining('server "scope": oauth.scope must be a string'), expect.stringContaining('server "unnamed": oauth.clientName must be a non-empty string'), expect.stringContaining('server "plainMetadata": oauth.authServerMetadataUrl must be an https URL'), - expect.stringContaining('server "badRegistration": oauth.clientRegistration must be "dcr" or "cimd"'), + expect.stringContaining('server "badRegistration": oauth.clientRegistration must be "auto", "dcr", or "cimd"'), expect.stringContaining('server "cimdClient": oauth.clientRegistration "cimd" cannot be combined'), expect.stringContaining('server "cimdPath": oauth.clientRegistration "cimd" requires oauth.callbackUrl'), ]); diff --git a/packages/coding-agent/test/mcp-manager-view.test.ts b/packages/coding-agent/test/mcp-manager-view.test.ts index ede09a8c9..ea7619717 100644 --- a/packages/coding-agent/test/mcp-manager-view.test.ts +++ b/packages/coding-agent/test/mcp-manager-view.test.ts @@ -38,3 +38,24 @@ describe("MCP manager status screen", () => { view.handleInput(ESCAPE); }); }); + +describe("MCP manager menu", () => { + beforeAll(() => { + initTheme("dark"); + }); + + // #7629 + test("captures fullscreen viewport keys only while a menu list is shown", async () => { + const view = new McpManagerView(tui, theme, new KeybindingsManager()); + const items = Array.from({ length: 20 }, (_, index) => ({ value: `server-${index}`, label: `server-${index}` })); + const result = view.menu(() => ({ title: "MCP servers", items, confirmLabel: "open", cancelLabel: "close" })); + expect(view.capturesViewportInput).toBe(true); + + view.handleInput("\x1b[6~"); // PageDown + view.handleInput("\r"); + await expect(result).resolves.toBe("server-12"); + + view.status("Sign in to issues", "Connecting…"); + expect(view.capturesViewportInput).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/mcp-oauth-refresh.test.ts b/packages/coding-agent/test/mcp-oauth-refresh.test.ts index a901ec35a..aed142a87 100644 --- a/packages/coding-agent/test/mcp-oauth-refresh.test.ts +++ b/packages/coding-agent/test/mcp-oauth-refresh.test.ts @@ -164,6 +164,7 @@ describe("MCP OAuth sign-in", () => { describe("MCP OAuth client ID metadata documents", () => { const cleanups: (() => Promise)[] = []; const cimd: McpOAuthSettings = { clientRegistration: "cimd" }; + const dcr: McpOAuthSettings = { clientRegistration: "dcr" }; afterEach(async () => { while (cleanups.length > 0) await cleanups.pop()?.(); @@ -183,16 +184,17 @@ describe("MCP OAuth client ID metadata documents", () => { return { server, store, signIn }; } - it("registers dynamically by default, even when the server supports documents", async () => { + it("registers dynamically with dcr, even when the server supports documents", async () => { const { server, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn({}); + await signIn(dcr); expect(server.registrations).toHaveLength(1); expect(server.authorizations[0].get("client_id")).toBe("client-1"); }); - it("uses pi's document when authorization responses name their issuer", async () => { + // #11426 + it("uses pi's document by default when authorization responses name their issuer", async () => { const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn(cimd); + await signIn({}); const [authorization] = server.authorizations; expect(authorization.get("client_id")).toBe("https://pi.dev/oauth/client.json"); const redirect = new URL(authorization.get("redirect_uri") ?? ""); @@ -203,12 +205,39 @@ describe("MCP OAuth client ID metadata documents", () => { expect(server.tokenRequests[0].get("redirect_uri")).toBe(redirect.href); // The document is not stored, so signing in again refreshes the tokens instead of discarding them. expect((await store.load())?.clientInformation).toBeUndefined(); - await signIn(cimd); + await signIn({}); expect(server.authorizations).toHaveLength(1); expect(server.tokenRequests[1].get("grant_type")).toBe("refresh_token"); expect(server.tokenRequests[1].get("client_id")).toBe("https://pi.dev/oauth/client.json"); }); + // #11426 + it("registers dynamically by default when the server does not support documents", async () => { + const { server, signIn } = await startServer({ issParameter: true }); + await signIn({}); + expect(server.registrations).toHaveLength(1); + expect(server.authorizations[0].get("client_id")).toBe("client-1"); + }); + + // #11426 + it("registers dynamically by default with a client name", async () => { + const { server, signIn } = await startServer({ cimd: true, issParameter: true }); + await signIn({ clientName: "Claude Code" }); + expect(server.registrations).toMatchObject([{ client_name: "Claude Code" }]); + }); + + // #11426 + it("keeps a registered client by default", async () => { + const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); + await signIn(dcr); + await signIn({}); + // The registered client's tokens are refreshed instead of signing in with the document. + expect(server.authorizations).toHaveLength(1); + expect(server.tokenRequests[1].get("grant_type")).toBe("refresh_token"); + expect(server.tokenRequests[1].get("client_id")).toBe("client-1"); + expect((await store.load())?.clientInformation?.client_id).toBe("client-1"); + }); + it("uses a document and callback path specific to the MCP server without the iss parameter", async () => { const { server, signIn } = await startServer({ cimd: true }); await signIn(cimd); @@ -224,7 +253,7 @@ describe("MCP OAuth client ID metadata documents", () => { it("replaces a registered client when switching to the document", async () => { const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn({}); + await signIn(dcr); expect((await store.load())?.clientInformation?.client_id).toBe("client-1"); await signIn(cimd); // The registered client's tokens are not refreshed with another client. diff --git a/packages/coding-agent/test/model-catalog-protocol.test.ts b/packages/coding-agent/test/model-catalog-protocol.test.ts index 9222af565..197cab282 100644 --- a/packages/coding-agent/test/model-catalog-protocol.test.ts +++ b/packages/coding-agent/test/model-catalog-protocol.test.ts @@ -61,8 +61,6 @@ function startCatalogServer(requests: string[]): Promise { const catalogRequest = parseModelCatalogRequest(url, request.headers["user-agent"]); if (!provider) { response.writeHead(404).end(); - } else if (catalogRequest.kind === "redirect") { - response.writeHead(307, { location: catalogRequest.location, "cache-control": "no-store" }).end(); } else if (catalogRequest.kind === "invalid") { response.writeHead(400).end(catalogRequest.error); } else { @@ -120,7 +118,7 @@ describe("model catalog protocol with the current client", () => { expect([...refresh.errors]).toEqual([]); const catalogUrl = "/api/models/providers/openrouter?types=chat%2Cimage%2Cclassifier"; - expect(requests).toEqual([catalogUrl, `${catalogUrl}&pi-version=${VERSION}`]); + expect(requests).toEqual([catalogUrl]); const expectedModel = selectModelCatalog(index, VERSION)?.revision === legacyRevision ? legacyModel : mixedApiModel; diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index f8226a2be..71cd27f0c 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -115,6 +115,8 @@ const referenceCatalog = [ referenceModel("gw-b", "claude-opus-4-8"), referenceModel("gw-b", "gw-b/free"), referenceModel("gw-b", "zz:free"), + // URL-derived provider id containing "/" (#11394) + referenceModel("llama=http://host:10800", "unsloth/qwen-gguf:BF16"), ]; type ReferenceExpected = @@ -152,6 +154,16 @@ const referenceCases: ReferenceCase[] = [ text: "openrouter/qwen/qwen3-coder:exacto:high", expected: { model: "openrouter/qwen/qwen3-coder:exacto", thinkingLevel: "high" }, }, + { + name: "provider id containing a slash", + text: "llama=http://host:10800/unsloth/qwen-gguf:BF16", + expected: { model: "llama=http://host:10800/unsloth/qwen-gguf:BF16" }, + }, + { + name: "provider id containing a slash, with thinking suffix", + text: "llama=http://host:10800/unsloth/qwen-gguf:BF16:high", + expected: { model: "llama=http://host:10800/unsloth/qwen-gguf:BF16", thinkingLevel: "high" }, + }, { name: "raw id wins when the provider prefix names no matching model", text: "openai/gpt-4o:extended", @@ -686,7 +698,7 @@ describe("resolveCliModel", () => { describe("default model selection", () => { test("openai defaults track current models", () => { - expect(defaultModelPerProvider.openai).toBe("gpt-5.5"); + expect(defaultModelPerProvider.openai).toBe("gpt-6.1-sol"); expect(defaultModelPerProvider["openai-codex"]).toBe("gpt-6.1-sol"); }); diff --git a/packages/coding-agent/test/model-selector.test.ts b/packages/coding-agent/test/model-selector.test.ts index 7ba204ac7..df1a570e6 100644 --- a/packages/coding-agent/test/model-selector.test.ts +++ b/packages/coding-agent/test/model-selector.test.ts @@ -82,6 +82,30 @@ describe("model selector", () => { selector.dispose(); }); + it("moves by one page with remapped selection page keys (#7629)", async () => { + setKeybindings(new KeybindingsManager({ "tui.select.pageUp": "alt+v", "tui.select.pageDown": "ctrl+v" })); + harness = await createHarness({ + models: Array.from({ length: 12 }, (_, index) => ({ id: `model-${String(index).padStart(2, "0")}` })), + }); + const onSelect = vi.fn(); + const selector = new ModelSelectorComponent( + createFakeTui(), + undefined, + harness.session.modelRuntime, + [], + onSelect, + () => {}, + ); + + selector.handleInput("\x16"); // Ctrl+V + selector.handleInput("\x16"); + selector.handleInput("\x1bv"); // Alt+V + selector.handleInput("\r"); + + expect(onSelect).toHaveBeenCalledWith(expect.objectContaining({ id: "model-01" })); + selector.dispose(); + }); + it("uses the configured save binding", async () => { setKeybindings(new KeybindingsManager({ "app.models.save": "ctrl+r" })); harness = await createHarness(); diff --git a/packages/coding-agent/test/package-command-paths.test.ts b/packages/coding-agent/test/package-command-paths.test.ts index 68949f9a3..a12acd856 100644 --- a/packages/coding-agent/test/package-command-paths.test.ts +++ b/packages/coding-agent/test/package-command-paths.test.ts @@ -774,6 +774,140 @@ if (process.platform !== "win32") fs.chmodSync(piPath, 0o755); expect(process.exitCode).toBe(1); }); + function writeNodeShim(binDir: string, name: string, scriptPath: string): void { + const shimPath = join(binDir, process.platform === "win32" ? `${name}.cmd` : name); + writeFileSync( + shimPath, + process.platform === "win32" + ? `@echo off\r\n"${originalExecPath}" "${scriptPath}" %*\r\n` + : `#!/bin/sh\nexec "${originalExecPath}" "${scriptPath}" "$@"\n`, + ); + chmodSync(shimPath, 0o755); + } + + /** Put a `pi-wrapper` update command and a `pi` that prints `piVersion` first on PATH. */ + function prepareSelfUpdateCommand(piVersion: string, exitCode = 0): { scriptPath: string; recordPath: string } { + const binDir = join(tempDir, "self-update-bin"); + const scriptPath = join(tempDir, "self-update.cjs"); + const piScriptPath = join(tempDir, "fake-pi.cjs"); + const recordPath = join(tempDir, "self-update-record.json"); + mkdirSync(binDir, { recursive: true }); + writeFileSync( + scriptPath, + `require("node:fs").writeFileSync(${JSON.stringify(recordPath)}, JSON.stringify(process.argv.slice(2))); +process.exit(${exitCode}); +`, + ); + writeFileSync(piScriptPath, `console.log(${JSON.stringify(piVersion)});\n`); + writeNodeShim(binDir, "pi-wrapper", scriptPath); + writeNodeShim(binDir, "pi", piScriptPath); + process.env.PATH = `${binDir}${delimiter}${originalPath ?? ""}`; + return { scriptPath, recordPath }; + } + + it("runs PI_SELF_UPDATE_COMMAND instead of the managed update and the selfUpdateCommand setting", async () => { + const targetVersion = getNewerPatchVersion(); + const { npmRecordPath } = prepareManagedInstall(targetVersion); + const { scriptPath, recordPath } = prepareSelfUpdateCommand(targetVersion); + const settingRecordPath = join(tempDir, "setting-record.json"); + writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, settingRecordPath] }), + ); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade {installSpec} {version}"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ packageName: "@new-scope/pi", version: targetVersion, note: "Read me" })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + expect(JSON.parse(readFileSync(recordPath, "utf8")) as string[]).toEqual([ + "upgrade", + `@new-scope/pi@${targetVersion}`, + targetVersion, + ]); + expect(existsSync(npmRecordPath)).toBe(false); + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stdout).toContain("Read me"); + // Like mise, the command leaves this installation untouched; the pi on PATH reports the new version. + expect(stdout).toContain(`Updated pi from ${VERSION} to ${targetVersion}`); + expect(errorSpy).not.toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + }); + + it("runs the global selfUpdateCommand setting only when a newer release exists", async () => { + const targetVersion = getNewerPatchVersion(); + const { scriptPath, recordPath } = prepareSelfUpdateCommand(targetVersion); + const projectRecordPath = join(tempDir, "project-record.json"); + writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, "{packageName}"] }), + ); + mkdirSync(join(projectDir, ".pi"), { recursive: true }); + writeFileSync( + join(projectDir, ".pi", "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, projectRecordPath] }), + ); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", ""); + const fetchMock = vi.fn(async () => Response.json({ version: VERSION })); + vi.stubGlobal("fetch", fetchMock); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + expect(existsSync(recordPath)).toBe(false); + expect(logSpy.mock.calls.map(([message]) => String(message)).join("\n")).toContain( + `pi is already up to date (v${VERSION})`, + ); + + fetchMock.mockImplementation(async () => Response.json({ version: targetVersion })); + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + expect(JSON.parse(readFileSync(recordPath, "utf8")) as string[]).toEqual([PACKAGE_NAME]); + expect(process.exitCode).toBeUndefined(); + }); + + it("warns when the pi on PATH does not report the new version", async () => { + const targetVersion = getNewerPatchVersion(); + prepareSelfUpdateCommand(VERSION); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ version: targetVersion })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stdout).toContain( + `Ran pi-wrapper upgrade, but \`pi --version\` reports ${VERSION}, not ${targetVersion}.`, + ); + expect(stdout).not.toContain("Updated pi from"); + expect(process.exitCode).toBeUndefined(); + }); + + it("fails when the configured self-update command fails", async () => { + prepareSelfUpdateCommand(VERSION, 23); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ version: getNewerPatchVersion() })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + expect(errorSpy.mock.calls.map(([message]) => String(message)).join("\n")).toContain( + "pi-wrapper upgrade exited with code 23", + ); + expect(logSpy.mock.calls.map(([message]) => String(message)).join("\n")).not.toContain("Ran pi-wrapper"); + expect(process.exitCode).toBe(1); + }); + it("keeps the managed release active when its update fails", async () => { const targetVersion = getNewerPatchVersion(); const { managedRoot } = prepareManagedInstall(targetVersion, 23); diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index 4aaa60ab3..b5895b331 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from "node:events"; -import { existsSync, mkdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { PassThrough } from "node:stream"; @@ -671,6 +671,92 @@ Content`, expect(result.skills.some((r) => r.path === slashSkillPath && r.enabled)).toBe(true); }); + it("should expose manifest changelogPath on package resources", async () => { + const pkgDir = join(tempDir, "changelog-manifest-package"); + const extensionPath = join(pkgDir, "src", "index.ts"); + const changelogPath = join(pkgDir, "docs", "CHANGELOG.md"); + mkdirSync(join(pkgDir, "src"), { recursive: true }); + mkdirSync(join(pkgDir, "docs"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "changelog-manifest-package", + pi: { + extensions: ["src/index.ts"], + changelogPath: "docs/CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should expose package CHANGELOG.md by convention", async () => { + const pkgDir = join(tempDir, "changelog-convention-package"); + const extensionPath = join(pkgDir, "extensions", "index.ts"); + const changelogPath = join(pkgDir, "CHANGELOG.md"); + mkdirSync(join(pkgDir, "extensions"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should still auto-discover resources when pi manifest only declares changelogPath", async () => { + const pkgDir = join(tempDir, "changelog-only-manifest-package"); + const extensionPath = join(pkgDir, "extensions", "index.ts"); + const changelogPath = join(pkgDir, "CHANGELOG.md"); + mkdirSync(join(pkgDir, "extensions"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "changelog-only-manifest-package", + pi: { + changelogPath: "CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should ignore manifest changelogPath outside the package root", async () => { + const pkgDir = join(tempDir, "unsafe-changelog-package"); + const extensionPath = join(pkgDir, "src", "index.ts"); + mkdirSync(join(pkgDir, "src"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(join(tempDir, "CHANGELOG.md"), "## [1.0.0]\n\n- Outside"); + writeFileSync(join(pkgDir, "CHANGELOG.md"), "## [1.0.0]\n\n- Fallback should not be used"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "unsafe-changelog-package", + pi: { + extensions: ["src/index.ts"], + changelogPath: "../CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBeUndefined(); + }); + it("should handle directories with auto-discovery layout", async () => { const pkgDir = join(tempDir, "auto-pkg"); mkdirSync(join(pkgDir, "extensions"), { recursive: true }); @@ -754,8 +840,10 @@ Content`, "install", "@scope/pkg", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); @@ -769,7 +857,44 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["uninstall", "@scope/pkg", "--prefix", join(agentDir, "npm"), "--legacy-peer-deps"], + [ + "uninstall", + "@scope/pkg", + "--prefix", + realpathSync(join(agentDir, "npm")), + "--legacy-peer-deps", + "--no-audit", + "--no-fund", + ], + undefined, + ); + }); + + // Regression test for #11358: npm writes nested package-lock.json entries for a symlinked --prefix + it("should pass the real path as --prefix when the agent dir is a symlink", async () => { + const realAgentDir = join(tempDir, "dotfiles", "agent"); + mkdirSync(realAgentDir, { recursive: true }); + const linkedAgentDir = join(tempDir, "linked-agent"); + symlinkSync(realAgentDir, linkedAgentDir, "dir"); + packageManager = new DefaultPackageManager({ + cwd: tempDir, + agentDir: linkedAgentDir, + settingsManager, + }); + const runCommandSpy = vi.spyOn(packageManager as any, "runCommand").mockResolvedValue(undefined); + const realNpmRoot = join(realpathSync(realAgentDir), "npm"); + + await packageManager.install("npm:@scope/pkg"); + await packageManager.remove("npm:@scope/pkg"); + + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"], + undefined, + ); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["uninstall", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"], undefined, ); }); @@ -790,7 +915,17 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "mise", - ["exec", "bun@1", "--", "bun", "install", "@scope/pkg", "--cwd", join(agentDir, "npm"), "--omit=peer"], + [ + "exec", + "bun@1", + "--", + "bun", + "install", + "@scope/pkg", + "--cwd", + realpathSync(join(agentDir, "npm")), + "--omit=peer", + ], undefined, ); }); @@ -810,9 +945,13 @@ Content`, await packageManager.install(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should remove a newly created checkout when git clone fails", async () => { @@ -876,9 +1015,13 @@ Content`, cwd: targetDir, }); expect(runCommandSpy).toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should reconcile an existing git checkout to its update target when installing without a ref", async () => { @@ -1027,9 +1170,13 @@ Content`, await packageManager.update(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should repair missing git package dependencies when the checkout is already current", async () => { @@ -1054,13 +1201,18 @@ Content`, await packageManager.update(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); expect(runCommandSpy).not.toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); }); - it("should repair deleted git package dependencies when cleaning fails", async () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install git package dependencies when cleaning fails", async () => { const source = "git:github.com/user/repo"; const targetDir = join(agentDir, "git", "github.com", "user", "repo"); const fetchArgs = ["fetch", "--prune", "--no-tags", "origin", "+refs/heads/main:refs/remotes/origin/main"]; @@ -1086,11 +1238,15 @@ Content`, if (args[0] === "clean") throw new Error("simulated clean failure"); }); - await expect(packageManager.update(source)).rejects.toThrow("simulated clean failure"); + await packageManager.update(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should disable peer installation through wrapped pnpm when updating git dependencies", async () => { @@ -1206,7 +1362,7 @@ Content`, "install", "pnpm-pkg", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--config.auto-install-peers=false", "--config.strict-peer-dependencies=false", "--config.strict-dep-builds=false", @@ -2467,7 +2623,15 @@ export default function(api) { api.registerTool({ name: "test", description: "te ); expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["install", "example@^1.0.0", "--prefix", join(tempDir, ".pi", "npm"), "--legacy-peer-deps"], + [ + "install", + "example@^1.0.0", + "--prefix", + realpathSync(join(tempDir, ".pi", "npm")), + "--legacy-peer-deps", + "--no-audit", + "--no-fund", + ], undefined, ); }); @@ -2531,8 +2695,10 @@ export default function(api) { api.registerTool({ name: "test", description: "te "install", "legacy-pkg@latest", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ]); mkdirSync(managedPath, { recursive: true }); writeFileSync( @@ -2648,8 +2814,10 @@ export default function(api) { api.registerTool({ name: "test", description: "te "user-old@latest", "user-unknown@latest", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); @@ -2661,8 +2829,10 @@ export default function(api) { api.registerTool({ name: "test", description: "te "project-old@latest", "project-missing@latest", "--prefix", - join(tempDir, ".pi", "npm"), + realpathSync(join(tempDir, ".pi", "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); @@ -2671,6 +2841,33 @@ export default function(api) { api.registerTool({ name: "test", description: "te expect(maxConcurrentGitUpdates).toBeGreaterThan(1); }); + it("should warn when an npm update leaves the installed version unchanged", async () => { + const stalePath = join(agentDir, "npm", "node_modules", "stale-pkg"); + const freshPath = join(agentDir, "npm", "node_modules", "fresh-pkg"); + mkdirSync(stalePath, { recursive: true }); + mkdirSync(freshPath, { recursive: true }); + writeFileSync(join(stalePath, "package.json"), JSON.stringify({ name: "stale-pkg", version: "1.0.0" })); + writeFileSync(join(freshPath, "package.json"), JSON.stringify({ name: "fresh-pkg", version: "1.0.0" })); + settingsManager.setPackages(["npm:stale-pkg", "npm:fresh-pkg"]); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.0.1"'); + vi.spyOn(packageManager as any, "runCommand").mockImplementation(async () => { + // Simulate npm resolving stale metadata for stale-pkg only. + writeFileSync(join(freshPath, "package.json"), JSON.stringify({ name: "fresh-pkg", version: "1.0.1" })); + }); + + const events: ProgressEvent[] = []; + packageManager.setProgressCallback((event) => events.push(event)); + + await packageManager.update(); + + const warnings = events.filter((event) => event.type === "progress"); + expect(warnings).toHaveLength(1); + expect(warnings[0].source).toBe("npm:stale-pkg"); + expect(warnings[0].message).toContain("is still 1.0.0"); + expect(warnings[0].message).toContain("1.0.1"); + }); + it("should suggest npm source prefixes for update lookups", async () => { settingsManager.setProjectPackages(["npm:example"]); @@ -2744,6 +2941,81 @@ export default function(api) { api.registerTool({ name: "test", description: "te expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/old.ts"))).toBe(false); }); + // https://github.com/earendil-works/pi/issues/11375 + it("should refresh temporary git sources with a branch ref", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "git:github.com/example/repo@main"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "git") throw new Error("Expected git source"); + const installedPath = managerWithInternals.getGitInstallPath(parsed, "temporary"); + mkdirSync(join(installedPath, "extensions"), { recursive: true }); + writeFileSync(join(installedPath, "extensions", "index.ts"), "export default function() {};"); + + const updateGitSpy = vi.spyOn(packageManager as any, "updateGit").mockResolvedValue(undefined); + + const result = await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(updateGitSpy).toHaveBeenCalledTimes(1); + expect(updateGitSpy.mock.calls[0][0]).toMatchObject({ ref: "main" }); + expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/index.ts") && r.enabled)).toBe(true); + }); + + it("should not refresh temporary git sources pinned to the checked-out commit", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "git:github.com/example/repo@abc1234"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "git") throw new Error("Expected git source"); + mkdirSync(managerWithInternals.getGitInstallPath(parsed, "temporary"), { recursive: true }); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue( + "ABC1234def5678901234567890123456789012345\n", + ); + const updateGitSpy = vi.spyOn(packageManager as any, "updateGit").mockResolvedValue(undefined); + + await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(updateGitSpy).not.toHaveBeenCalled(); + }); + + it("should refresh unpinned temporary npm sources when a newer version exists", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "npm:example"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "npm") throw new Error("Expected npm source"); + const installedPath = managerWithInternals.getNpmInstallPath(parsed, "temporary"); + mkdirSync(join(installedPath, "extensions"), { recursive: true }); + writeFileSync(join(installedPath, "package.json"), JSON.stringify({ name: "example", version: "1.0.0" })); + writeFileSync(join(installedPath, "extensions", "index.ts"), "export default function() {};"); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.1.0"'); + const runNpmCommandSpy = vi + .spyOn(packageManager as any, "runNpmCommand") + .mockRejectedValue(new Error("network down")); + + const result = await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(runNpmCommandSpy).toHaveBeenCalledTimes(1); + expect(runNpmCommandSpy.mock.calls[0][0]).toContain("example@latest"); + // A failed refresh keeps loading the cached install. + expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/index.ts") && r.enabled)).toBe(true); + }); + + it("should not refresh temporary npm sources when already up to date or pinned", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + for (const source of ["npm:example", "npm:example@1.0.0"]) { + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "npm") throw new Error("Expected npm source"); + const installedPath = managerWithInternals.getNpmInstallPath(parsed, "temporary"); + mkdirSync(installedPath, { recursive: true }); + writeFileSync(join(installedPath, "package.json"), JSON.stringify({ name: "example", version: "1.0.0" })); + } + + const runCommandCaptureSpy = vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.0.0"'); + const runNpmCommandSpy = vi.spyOn(packageManager as any, "runNpmCommand").mockResolvedValue(undefined); + + await packageManager.resolveExtensionSources(["npm:example"], { temporary: true }); + await packageManager.resolveExtensionSources(["npm:example@1.0.0"], { temporary: true }); + expect(runCommandCaptureSpy).toHaveBeenCalledTimes(1); + expect(runNpmCommandSpy).not.toHaveBeenCalled(); + }); + it("should not run npm view during resolve for installed unpinned packages", async () => { process.env.PI_OFFLINE = "1"; const installedPath = join(tempDir, ".pi", "npm", "node_modules", "example"); diff --git a/packages/coding-agent/test/paths.test.ts b/packages/coding-agent/test/paths.test.ts index fbdb4f56f..b57238c13 100644 --- a/packages/coding-agent/test/paths.test.ts +++ b/packages/coding-agent/test/paths.test.ts @@ -6,10 +6,14 @@ import { afterEach, describe, expect, it } from "vitest"; import { canonicalizePath, getCwdRelativePath, + getPathComparisonKey, isLocalPath, + isPathWithin, normalizePath, normalizeWindowsShellPath, + pathsEqual, resolvePath, + shortenHomePath, } from "../src/utils/paths.ts"; let tempDir: string; @@ -26,6 +30,64 @@ function createTempDir(): string { return tempDir; } +describe("path comparison", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + + afterEach(() => { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + }); + + function simulatePlatform(platform: NodeJS.Platform): void { + Object.defineProperty(process, "platform", { value: platform }); + } + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("treats drive letters and path components case-insensitively on Windows", () => { + simulatePlatform("win32"); + expect(getPathComparisonKey("c:\\Users\\me\\.agents\\skills")).toBe( + getPathComparisonKey("C:\\Users\\Me\\.agents\\skills"), + ); + expect(pathsEqual("c:\\Users\\me", "C:/Users/Me/")).toBe(true); + expect(pathsEqual("C:\\Users\\me", "D:\\Users\\me")).toBe(false); + expect(isPathWithin("C:\\Users\\Me", "c:\\users\\me\\project")).toBe(true); + expect(isPathWithin("C:\\Users\\Me", "C:\\Users\\Me2")).toBe(false); + expect(isPathWithin("C:\\", "c:\\Users")).toBe(true); + expect(isPathWithin("\\\\server\\share", "\\\\SERVER\\Share\\folder")).toBe(true); + }); + + it("keeps casing on case-sensitive platforms", () => { + simulatePlatform("linux"); + expect(pathsEqual("/home/me", "/home/Me")).toBe(false); + expect(isPathWithin("/home/me", "/home/Me/project")).toBe(false); + }); + + it("isPathWithin matches whole path segments", () => { + simulatePlatform("linux"); + expect(isPathWithin("/home/me", "/home/me")).toBe(true); + expect(isPathWithin("/home/me", "/home/me/project")).toBe(true); + expect(isPathWithin("/home/me/", "/home/me/project")).toBe(true); + expect(isPathWithin("/home/me", "/home/me2")).toBe(false); + expect(isPathWithin("/", "/home")).toBe(true); + }); + + it("shortenHomePath replaces only a leading home directory", () => { + simulatePlatform("linux"); + expect(shortenHomePath("/home/me", "/home/me")).toBe("~"); + expect(shortenHomePath("/home/me/project", "/home/me")).toBe("~/project"); + expect(shortenHomePath("/home/me2/project", "/home/me")).toBe("/home/me2/project"); + expect(shortenHomePath("/home/Me/project", "/home/me")).toBe("/home/Me/project"); + expect(shortenHomePath("/etc/passwd", "/")).toBe("~/etc/passwd"); + expect(shortenHomePath("home/me/project", "/home/me")).toBe("home/me/project"); + expect(shortenHomePath("/home/me/project", "")).toBe("/home/me/project"); + }); + + it("shortenHomePath ignores casing and keeps separators on Windows", () => { + simulatePlatform("win32"); + expect(shortenHomePath("c:\\users\\me\\project", "C:\\Users\\Me")).toBe("~\\project"); + expect(shortenHomePath("\\\\server\\share\\folder", "\\\\server\\share")).toBe("~\\folder"); + }); +}); + describe("canonicalizePath", () => { it("returns the real path for a regular file", () => { const dir = createTempDir(); diff --git a/packages/coding-agent/test/project-trust-saved-decision.test.ts b/packages/coding-agent/test/project-trust-saved-decision.test.ts new file mode 100644 index 000000000..f356e44a3 --- /dev/null +++ b/packages/coding-agent/test/project-trust-saved-decision.test.ts @@ -0,0 +1,150 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { createProjectTrustContext } from "../src/cli/project-trust.ts"; +import { ENV_AGENT_DIR } from "../src/config.ts"; +import { resolveProjectTrusted } from "../src/core/project-trust.ts"; +import { SettingsManager } from "../src/core/settings-manager.ts"; +import { ProjectTrustStore } from "../src/core/trust-manager.ts"; + +const cliPath = resolve(__dirname, "../src/cli.ts"); +// --import takes a module specifier, not a filesystem path. +const sourceResolverUrl = pathToFileURL(resolve(__dirname, "../src/experimental/source-resolver.ts")).href; +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +interface Dirs { + agentDir: string; + projectDir: string; + probeFile: string; + extensionFile: string; +} + +function setup(): Dirs { + const tempRoot = realpathSync(mkdtempSync(join(tmpdir(), "pi-project-trust-saved-"))); + tempDirs.push(tempRoot); + const dirs = { + agentDir: join(tempRoot, "agent"), + projectDir: join(tempRoot, "project"), + probeFile: join(tempRoot, "probe.json"), + extensionFile: join(tempRoot, "probe-extension.ts"), + }; + mkdirSync(dirs.agentDir, { recursive: true }); + mkdirSync(dirs.projectDir, { recursive: true }); + // AGENTS.md alone is not a trust-requiring project resource. + writeFileSync(join(dirs.projectDir, "AGENTS.md"), "# untrusted\n"); + writeFileSync( + dirs.extensionFile, + `import { writeFileSync } from "node:fs"; +export default function (pi) { + pi.registerProvider("trust-probe", { + baseUrl: "https://trust-probe.invalid/v1", + apiKey: "trust-probe-key", + api: "openai-completions", + models: [{ + id: "trust-probe-model", + name: "trust-probe-model", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 10000, + maxTokens: 1000, + }], + }); + pi.on("session_start", (_event, ctx) => { + writeFileSync(${JSON.stringify(dirs.probeFile)}, JSON.stringify({ trusted: ctx.isProjectTrusted() })); + process.exit(0); + }); +} +`, + ); + return dirs; +} + +async function probeProjectTrust(dirs: Dirs): Promise { + const child = spawn( + process.execPath, + [ + "--import", + sourceResolverUrl, + cliPath, + "--no-session", + "-e", + dirs.extensionFile, + "--provider", + "trust-probe", + "--model", + "trust-probe-model", + "-p", + "hi", + ], + { + cwd: dirs.projectDir, + env: { ...process.env, [ENV_AGENT_DIR]: dirs.agentDir, PI_OFFLINE: "1" }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + const code = await new Promise((resolvePromise, reject) => { + const timeout = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (exitCode) => { + clearTimeout(timeout); + resolvePromise(exitCode); + }); + }); + expect(code, stderr).toBe(0); + return (JSON.parse(readFileSync(dirs.probeFile, "utf8")) as { trusted: boolean }).trusted; +} + +// Regression tests for https://github.com/earendil-works/pi/issues/10947 +describe("saved project trust for projects without trust-requiring resources", () => { + it("reports a saved Do not trust decision to extensions", async () => { + const dirs = setup(); + new ProjectTrustStore(dirs.agentDir).set(dirs.projectDir, false); + + expect(await probeProjectTrust(dirs)).toBe(false); + }, 30_000); + + it("auto-trusts when no decision is saved", async () => { + const dirs = setup(); + + expect(await probeProjectTrust(dirs)).toBe(true); + }, 30_000); + + it("resolveProjectTrusted honors the saved decision before auto-trust", async () => { + const dirs = setup(); + const trustStore = new ProjectTrustStore(dirs.agentDir); + const resolve = () => + resolveProjectTrusted({ + cwd: dirs.projectDir, + trustStore, + projectTrustContext: createProjectTrustContext({ + cwd: dirs.projectDir, + mode: "print", + settingsManager: SettingsManager.inMemory(), + hasUI: false, + }), + }); + + expect(await resolve()).toBe(true); + trustStore.set(dirs.projectDir, false); + expect(await resolve()).toBe(false); + trustStore.set(dirs.projectDir, true); + expect(await resolve()).toBe(true); + }); +}); diff --git a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts index 7c569f6e8..2813adc0f 100644 --- a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts +++ b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts @@ -21,8 +21,8 @@ import { getModel, type Model, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai/compat"; +import { Type } from "typebox"; import { getOpenAICodexWebSocketDebugStats, streamSimple as streamSimpleOpenAICodexResponses, diff --git a/packages/coding-agent/test/session-manager/file-operations.test.ts b/packages/coding-agent/test/session-manager/file-operations.test.ts index 8353ccdb7..ac6441490 100644 --- a/packages/coding-agent/test/session-manager/file-operations.test.ts +++ b/packages/coding-agent/test/session-manager/file-operations.test.ts @@ -1,6 +1,7 @@ import { constants as bufferConstants } from "buffer"; import { appendFileSync, + chmodSync, closeSync, existsSync, mkdirSync, @@ -471,4 +472,42 @@ describe("SessionManager session file creation", () => { expect(readSessionFileRoles(session.getSessionFile()!)).toEqual(["session", "user", "custom", "assistant"]); }); + + // #10166, #10661, #11460: a failed write must not leave later entries pointing to a missing parent + it("does not keep an entry in memory when writing it fails", () => { + const session = SessionManager.create(tempDir, tempDir); + const firstId = session.appendMessage(userMsg("first question")); + const file = session.getSessionFile()!; + + chmodSync(file, 0o444); + expect(() => session.appendMessage(assistantMsg("lost answer"))).toThrow(); + expect(session.getLeafId()).toBe(firstId); + expect(session.getEntries().map((entry) => entry.id)).toEqual([firstId]); + + chmodSync(file, 0o644); + const secondId = session.appendMessage(assistantMsg("second answer")); + expect(session.getEntry(secondId)?.parentId).toBe(firstId); + + const reopened = SessionManager.open(file, tempDir); + expect(reopened.getBranch().map((entry) => entry.id)).toEqual([firstId, secondId]); + }); + + it("does not keep an entry in memory when creating the file fails", () => { + const sessionDir = join(tempDir, "sessions"); + mkdirSync(sessionDir); + const session = SessionManager.create(tempDir, sessionDir); + const modelChangeId = session.appendModelChange("anthropic", "claude-sonnet-4-5"); + + chmodSync(sessionDir, 0o555); + try { + expect(() => session.appendMessage(userMsg("first question"))).toThrow(); + } finally { + chmodSync(sessionDir, 0o755); + } + expect(session.getLeafId()).toBe(modelChangeId); + expect(session.getEntries().map((entry) => entry.id)).toEqual([modelChangeId]); + + session.appendMessage(userMsg("second question")); + expect(readSessionFileRoles(session.getSessionFile()!)).toEqual(["session", "model_change", "user"]); + }); }); diff --git a/packages/coding-agent/test/settings-selector.test.ts b/packages/coding-agent/test/settings-selector.test.ts index 35b9bfe23..6a39219cb 100644 --- a/packages/coding-agent/test/settings-selector.test.ts +++ b/packages/coding-agent/test/settings-selector.test.ts @@ -6,6 +6,7 @@ import { type SettingsConfig, SettingsSelectorComponent, } from "../src/modes/interactive/components/settings-selector.ts"; +import { SelectSubmenu } from "../src/modes/interactive/components/settings-submenu.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; import { stripAnsi } from "../src/utils/ansi.ts"; import { createHarness, type Harness } from "./suite/harness.ts"; @@ -165,4 +166,23 @@ describe("SettingsSelectorComponent", () => { expect(output).toContain(" ✓ medium"); expect(output).toContain("→ high"); }); + + it("pages a searchable submenu with selection page keys (#7629)", () => { + const onSelect = vi.fn(); + const submenu = new SelectSubmenu( + "Theme", + "", + Array.from({ length: 20 }, (_, index) => ({ value: `theme-${index}`, label: `theme-${index}` })), + "theme-0", + onSelect, + () => {}, + undefined, + { searchable: true }, + ); + + submenu.handleInput("\x1b[6~"); // PageDown + submenu.handleInput("\r"); + + expect(onSelect).toHaveBeenCalledWith("theme-10"); + }); }); diff --git a/packages/coding-agent/test/source-info.test.ts b/packages/coding-agent/test/source-info.test.ts new file mode 100644 index 000000000..35e535704 --- /dev/null +++ b/packages/coding-agent/test/source-info.test.ts @@ -0,0 +1,19 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { getResourcePathKey } from "../src/core/source-info.ts"; + +describe("getResourcePathKey", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + + afterEach(() => { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("folds file path casing on Windows but keeps synthetic identifiers unchanged", () => { + Object.defineProperty(process, "platform", { value: "win32" }); + expect(getResourcePathKey("C:\\Users\\Me\\ext.ts")).toBe(getResourcePathKey("c:\\users\\me\\ext.ts")); + expect(getResourcePathKey("builtin:MCP")).toBe("builtin:MCP"); + expect(getResourcePathKey("builtin:MCP")).not.toBe(getResourcePathKey("builtin:mcp")); + expect(getResourcePathKey("")).toBe(""); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index 4aeeb9940..3e0620635 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -18,7 +18,10 @@ import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import type { ExtensionAPI } from "../../src/core/extensions/types.ts"; import type { CustomEntry } from "../../src/core/session-manager.ts"; -import { createToolDefinitionFromAgentTool } from "../../src/core/tools/tool-definition-wrapper.ts"; +import { + createToolDefinitionFromAgentTool, + type TypeBoxAgentTool, +} from "../../src/core/tools/tool-definition-wrapper.ts"; import { readCodemodeStore } from "../../src/extensions/codemode/execute.ts"; import { createCodemodeExtension } from "../../src/extensions/codemode/index.ts"; import { @@ -33,7 +36,7 @@ const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; const echoSchema = Type.Object({ text: Type.String({ description: "Text to echo" }) }); -const echoTool: AgentTool = { +const echoTool: TypeBoxAgentTool = { name: "echo", label: "Echo", description: "Echo text back.\n\nSecond paragraph.", @@ -184,6 +187,49 @@ describe("AgentSession codemode tool", () => { expect(description("echo")).toBe("Echo text back.\n\nSecond paragraph."); }); + // #11501 + it("summarizes union and multi-line output types instead of inlining them", async () => { + const outputTool = (name: string, outputSchema: AgentTool["outputSchema"]): AgentTool => ({ + name, + label: name, + description: `${name} tool`, + parameters: Type.Object({}), + outputSchema, + execute: async () => ({ content: [], details: {} }), + }); + const process = outputTool( + "process", + Type.Union([ + Type.Object({ + action: Type.Literal("start"), + process: Type.Object({ id: Type.String({ description: "Process id" }), pid: Type.Integer() }), + }), + Type.Object({ + action: Type.Literal("list"), + processes: Type.Array(Type.Object({ id: Type.String() })), + truncated: Type.Optional(Type.Boolean()), + }), + ]), + ); + const entries = outputTool("entries", Type.Array(Type.Object({ id: Type.String(), size: Type.Integer() }))); + const count = outputTool("count", Type.Number()); + const harness = await setup([ + (pi: ExtensionAPI) => { + for (const tool of [process, entries, count]) pi.registerTool(createToolDefinitionFromAgentTool(tool)); + }, + ]); + harness.session.setActiveToolsByName(["process", "entries", "count", "codemode"]); + const description = (name: string) => + harness.session.agent.state.tools.find((tool) => tool.name === name)?.description ?? ""; + expect(description("process")).toBe( + 'process tool\n\nCodemode: `tools.process(args)` resolves to `{ action: "start", process } | { action: "list", processes, truncated? }`.', + ); + expect(description("entries")).toBe( + 'entries tool\n\nCodemode: `tools.entries(args)` resolves to a value; `await describeTool("entries")` shows its type.', + ); + expect(description("count")).toBe("count tool\n\nCodemode: `tools.count(args)` resolves to `number`."); + }); + // #10343 it("shows the guidelines of tools that do not fit the inline budget through describeTool()", async () => { const harness = await setup(); diff --git a/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts new file mode 100644 index 000000000..02639a701 --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type Api, + createAssistantMessageEventStream, + type Model, + type SimpleStreamOptions, +} from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { AuthStorage } from "../../../src/core/auth-storage.ts"; +import { ModelRuntime } from "../../../src/core/model-runtime.ts"; +import { createAgentSession } from "../../../src/core/sdk.ts"; +import { SessionManager } from "../../../src/core/session-manager.ts"; +import { SettingsManager } from "../../../src/core/settings-manager.ts"; +import { assistantMsg, createTestResourceLoader, userMsg } from "../../utilities.ts"; + +// Issue #11520: summaries in SDK sessions passed the resolved OAuth token back to the +// ModelRuntime as an explicit API key, so provider auth took the API-key override branch. +describe("issue #11520 SDK summary OAuth auth", () => { + const tempDirs: string[] = []; + + afterEach(() => { + while (tempDirs.length > 0) rmSync(tempDirs.pop()!, { recursive: true, force: true }); + }); + + it("lets the runtime resolve stored OAuth for branch summaries", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-11520-")); + tempDirs.push(cwd); + const credentials = AuthStorage.inMemory({ + "oauth-only-provider": { + type: "oauth", + access: "oauth-access", + refresh: "refresh", + expires: Date.now() + 3_600_000, + }, + }); + const modelRuntime = await ModelRuntime.create({ credentials, modelsPath: null, allowModelNetwork: false }); + const model: Model = { + id: "oauth-model", + name: "OAuth Model", + api: "openai-completions", + provider: "oauth-only-provider", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; + const requests: SimpleStreamOptions[] = []; + modelRuntime.registerProvider(model.provider, { + name: "OAuth Only", + baseUrl: model.baseUrl, + api: model.api, + // With an API key method as well, an explicit key would bypass the stored OAuth login. + apiKey: "UNUSED_ENV_KEY", + oauth: { + name: "Subscription", + login: async () => ({ access: "oauth-access", refresh: "refresh", expires: Date.now() + 60_000 }), + refreshToken: async (credential) => credential, + getApiKey: (credential) => credential.access, + }, + models: [model], + streamSimple: (requestModel, _context, options) => { + requests.push(options ?? {}); + const stream = createAssistantMessageEventStream(); + stream.end({ + role: "assistant", + content: [{ type: "text", text: "branch summary text" }], + api: requestModel.api, + provider: requestModel.provider, + model: requestModel.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }); + return stream; + }, + }); + const authOverrides: Array = []; + const getAuth = modelRuntime.getAuth.bind(modelRuntime); + modelRuntime.getAuth = (async (...args: Parameters) => { + authOverrides.push(args[1]?.apiKey); + return getAuth(...args); + }) as ModelRuntime["getAuth"]; + + const sessionManager = SessionManager.inMemory(cwd); + const { session } = await createAgentSession({ + cwd, + agentDir: cwd, + model, + modelRuntime, + settingsManager: SettingsManager.inMemory(), + sessionManager, + resourceLoader: createTestResourceLoader(), + }); + try { + const targetId = sessionManager.appendMessage(userMsg("first branch")); + sessionManager.appendMessage(assistantMsg("first reply")); + sessionManager.appendMessage(userMsg("abandoned branch work")); + sessionManager.appendMessage(assistantMsg("abandoned reply")); + + const result = await session.navigateTree(targetId, { summarize: true }); + + expect(result.summaryEntry?.summary).toContain("branch summary text"); + expect(requests).toHaveLength(1); + expect(requests[0].apiKey).toBe("oauth-access"); + expect(authOverrides.length).toBeGreaterThan(0); + expect(authOverrides.every((apiKey) => apiKey === undefined)).toBe(true); + } finally { + session.dispose(); + } + }); +}); diff --git a/packages/coding-agent/test/system-prompt-updates.test.ts b/packages/coding-agent/test/system-prompt-updates.test.ts index a55c493d9..bbe5fb9f0 100644 --- a/packages/coding-agent/test/system-prompt-updates.test.ts +++ b/packages/coding-agent/test/system-prompt-updates.test.ts @@ -53,6 +53,23 @@ describe("system prompt updates", () => { } }); + // #11495 + test("does not patch the prompt when the active tools are only reordered", async () => { + const harness = await createHarness(); + try { + harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + await harness.session.prompt("one"); + const before = harness.session.getActiveToolNames(); + harness.session.setActiveToolsByName([...before].reverse()); + expect(harness.session.getActiveToolNames()).toEqual(before); + await harness.session.prompt("two"); + const systemMessages = harness.session.messages.filter((message) => message.role === "system"); + expect(systemMessages).toHaveLength(1); + } finally { + harness.cleanup(); + } + }); + test("opens a transcript without a system message and declares the prompt on the first request", async () => { const tempDir = mkdtempSync(join(tmpdir(), "pi-system-prompt-migration-")); try { diff --git a/packages/coding-agent/test/system-prompt.test.ts b/packages/coding-agent/test/system-prompt.test.ts index 276cc63ff..86d2c6994 100644 --- a/packages/coding-agent/test/system-prompt.test.ts +++ b/packages/coding-agent/test/system-prompt.test.ts @@ -48,7 +48,9 @@ describe("buildSystemPrompt", () => { skills: [], }); - expect(defaultPrompt.startsWith("You are an expert coding assistant operating inside pi")).toBe(true); + expect(defaultPrompt.startsWith("You are a coding assistant running inside pi, a coding agent harness.")).toBe( + true, + ); expect(customPrompt.startsWith("You are Exact.\n\n")).toBe(true); }); @@ -119,6 +121,9 @@ describe("buildSystemPrompt", () => { "- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory", ); expect(prompt).toContain("environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md)"); + expect(prompt).toContain( + "Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.", + ); }); }); @@ -192,6 +197,9 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain(""); expect(prompt).toContain("test-skill"); expect(prompt).toContain("Use bash to load a skill's file"); + expect(prompt).toContain( + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", + ); }); test("omits skills without read or bash", () => { diff --git a/packages/coding-agent/test/tool-search.test.ts b/packages/coding-agent/test/tool-search.test.ts index ba07448a0..68a59cec7 100644 --- a/packages/coding-agent/test/tool-search.test.ts +++ b/packages/coding-agent/test/tool-search.test.ts @@ -19,6 +19,40 @@ describe("tokenize", () => { it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t，。!? 😀 ")).toEqual([]); }); }); @@ -45,6 +79,22 @@ describe("Bm25Ranker", () => { expect(ranker.rank("closed", documents, 8).map((match) => match.name)).toEqual(["mcp__github__list_issues"]); }); + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (description, query) => { + const localized = createToolSearchDocument(tool("localized", description)); + expect(new Bm25Ranker().rank(query, [...documents, localized], 8)).toEqual([ + { name: "localized", score: expect.any(Number) }, + ]); + }); + it("returns nothing for unknown or empty queries", () => { const ranker = new Bm25Ranker(); expect(ranker.rank("kubernetes", documents, 8)).toEqual([]); @@ -53,6 +103,87 @@ describe("Bm25Ranker", () => { expect(ranker.rank("tickets", documents, 8)).toEqual([]); }); + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query is its name", () => { + const named = [ + tool("mcp__app__queue_read", "Read a queued message of this thread."), + tool( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans. Continue with afterPosition until hasMore is false.", + ), + ].map((entry) => createToolSearchDocument(entry)); + const ranker = new Bm25Ranker(); + for (const query of [ + "thread_read", + "dev_app__thread_read", + "mcp__dev-app__thread_read", + "Thread_Read", + "thread read", + ]) { + expect(ranker.rank(query, named, 1).map((match) => match.name)).toEqual(["mcp__dev-app__thread_read"]); + } + // Partial names still rank by BM25. + expect(ranker.rank("read", named, 1).map((match) => match.name)).toEqual(["mcp__app__queue_read"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a name match above mentions in descriptions", () => { + const documents = [ + tool("mcp__ci__status", "Deploy status: last deploy, deploy log, deploy errors."), + tool( + "mcp__ci__deploy_release", + "Start a release of the current branch to an environment, run the migrations, switch traffic over, and wait until the new version is live and healthy.", + ), + ].map((entry) => createToolSearchDocument(entry)); + expect(new Bm25Ranker().rank("deploy", documents, 1).map((match) => match.name)).toEqual([ + "mcp__ci__deploy_release", + ]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query contains its whole name", () => { + const peekaboo = { name: "mcp__peekaboo", instructions: "Peekaboo controls macOS apps by app name." }; + const documents = [ + tool("mcp__peekaboo__see", "Capture the app window of the peekaboo app and describe the app."), + tool("mcp__peekaboo__menu", "Click a menu item of an app."), + tool( + "mcp__peekaboo__app", + "Launch, quit, relaunch, focus, hide, or switch applications. Targets an application by name, bundle identifier, or process ID, and can wait until it is ready.", + ), + tool("mcp__app__queue_read", "Read a queued message of this thread."), + tool( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans.", + ), + ].map((entry) => + createToolSearchDocument(entry, entry.name.startsWith("mcp__peekaboo__") ? peekaboo : undefined), + ); + const ranker = new Bm25Ranker(); + const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); + expect(first("peekaboo app")).toEqual(["mcp__peekaboo__app"]); + expect(first("open the peekaboo app tool")).toEqual(["mcp__peekaboo__app"]); + expect(first("read thread_read state")).toEqual(["mcp__dev-app__thread_read"]); + // Without the whole name, mentions in descriptions decide. + expect(first("describe window")).toEqual(["mcp__peekaboo__see"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool matching more query terms first, whatever the scores", () => { + const documents = [ + tool("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier."), + tool("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app."), + tool("mcp__peekaboo__window", "Move, resize, or close a window of an app."), + tool("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window."), + ].map((entry) => createToolSearchDocument(entry)); + const ranker = new Bm25Ranker(); + const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); + // `window` and `app` are whole names, but each covers one of the two terms. + expect(first("screenshot window")).toEqual(["mcp__peekaboo__image"]); + expect(first("describe app")).toEqual(["mcp__peekaboo__see"]); + // An exact name still ranks first. + expect(first("window")).toEqual(["mcp__peekaboo__window"]); + }); + it("includes the namespace in the search text", () => { const document = createToolSearchDocument(tool("mcp__x__run", "Run it."), { name: "mcp__x", diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index 91432623f..a096c2f91 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -908,6 +908,23 @@ describe("Coding Agent Tools", () => { expect(output).not.toContain("match two"); }); + // https://github.com/earendil-works/pi/issues/11398 + it("should keep context line numbers aligned with ripgrep when a file contains a lone CR", async () => { + const testFile = join(testDir, "lone-cr.txt"); + writeFileSync(testFile, "before\rprogress\nneedle\nafter"); + + const result = await grepTool.execute("test-call-lone-cr", { + pattern: "needle", + path: testFile, + context: 1, + }); + + const output = getTextOutput(result); + expect(output).toContain("lone-cr.txt-1- beforeprogress"); + expect(output).toContain("lone-cr.txt:2: needle"); + expect(output).toContain("lone-cr.txt-3- after"); + }); + // https://github.com/earendil-works/pi/issues/11203 it("should ignore RIPGREP_CONFIG_PATH", async () => { const configFile = join(testDir, "rg-config"); diff --git a/packages/coding-agent/test/trust-manager.test.ts b/packages/coding-agent/test/trust-manager.test.ts index 01500bac2..36d6f89ec 100644 --- a/packages/coding-agent/test/trust-manager.test.ts +++ b/packages/coding-agent/test/trust-manager.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -64,4 +64,45 @@ describe("ProjectTrustStore", () => { } } }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("ignores the user .agents/skills dir when HOME differs only by case on Windows", () => { + const originalHome = process.env.HOME; + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + const mixedCaseHome = join(tempDir, "Home"); + const projectDir = join(mixedCaseHome, "project"); + mkdirSync(join(mixedCaseHome, ".agents", "skills"), { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + process.env.HOME = mixedCaseHome.toLowerCase(); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + expect(hasTrustRequiringProjectResources(projectDir)).toBe(false); + } finally { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + } + }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("matches saved decisions regardless of casing on Windows", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + const projectDir = join(tempDir, "Project-Mixed"); + mkdirSync(projectDir, { recursive: true }); + const store = new ProjectTrustStore(agentDir); + store.set(projectDir, true); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + expect(store.get(projectDir.toLowerCase())).toBe(true); + store.set(projectDir.toLowerCase(), false); + const saved = JSON.parse(readFileSync(join(agentDir, "trust.json"), "utf-8")) as Record; + expect(Object.values(saved)).toEqual([false]); + expect(store.get(projectDir)).toBe(false); + } finally { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + } + }); }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 409f1ddc2..14750ec5a 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -2,31 +2,191 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + ### Breaking Changes -- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption fails the Session without rolling the adopted changes back. +- Tool schemas are JSON Schema typed through `@earendil-works/pi-schema`: `defineTool()` and `ToolRegistration` take any JSON Schema object as `parameters` and `structuredOutputSchema`, and type `args` with pi-schema's `Static` (`ToolArgs`; TypeBox schemas give TypeBox's static type for the common kinds, as pi-schema's `Static` describes; other schemas give `unknown`). The built-in tools (`read`, `write`, `edit`, `bash`, `powershell`, `codemode`, `tool_search`, the model tools) are built with pi-schema, with the same JSON Schema up to key order. `structuredOutput` is validated with pi-schema, so `invalid_structured_output` messages use its wording. `typebox` is no longer a dependency. Migration: replace `Type.Object({ n: Type.Number() })` from `@earendil-works/pi-ai` with `obj({ n: num() })` from `@earendil-works/pi-schema` (closed by default; `{ open: true }` allows other properties). +- `SqliteExecutor` (re-exported from Chord) gains `runCounted(sql, ...params)`, resolving with the rows the statement changed; custom `SqliteDatabase` implementations add it. +- `Storage.commit()` stores a group of commits in one call: `commit(batches, receive, context)` resolves with a `CommitResult` (`{ stored, refused? }`) and replaces `commit(writes, adopt, context)`. Each batch is one atomic commit with its own seq, checked against the state the batches before it left. The first batch Durable's own checks refuse (global IDs, document actions, copy sources) is not stored, nor anything after it; the batches before it are, and the call resolves with their number in `stored` and the refusal in `refused`. Any other failure (a driver or I/O error, a failed `COMMIT`) rejects the whole call as an unknown outcome and stores nothing the backend could roll back. `receive(index, seq)` is called once per stored batch, in order, synchronously, once they are durable and before the next call starts; receipts made before a rejection stand. Every batch has at least one write. The optional `publishing(publish, context)` runs each commit's publication inside the storage's own scope (`StoreStorage`: the attached host's batch). Memory, SQLite, JSONL, IndexedDB and `StoreStorage` store a call in one transaction or append, and the storage conformance suite checks the contract. Migration for a custom Storage: + + ```ts + // Before + async commit(writes, adopt, context) { + const seq = await this.transaction((tx) => { + check(tx, writes); // throws when Durable's rules refuse the commit + return apply(tx, writes); + }); + adopt(seq); + return seq; + } + + // After + async commit(batches, receive, context) { + let refused: { error: unknown } | undefined; + const seqs = await this.transaction((tx) => { + const stored: Seq[] = []; + for (const writes of batches) { + try { + check(tx, writes); // only Durable's own checks; reads the earlier batches' writes + } catch (error) { + refused = { error }; + break; // keep the batches before it + } + stored.push(apply(tx, writes)); // a throw rejects the call and rolls all of it back + } + return stored; + }); + seqs.forEach((seq, index) => receive(index, seq)); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; + } + ``` + + A database error during the checks is not a refusal: it must reject the call. A wrapper forwards `commit()` and `publishing()`. Code that called `storage.commit(writes, adopt, context)` directly, such as a test seeding a storage, calls `commitWrites(storage, writes, context)` from `@earendil-works/pi-durable/testing` (one batch; resolves with its seq, throws the refusal). - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. -- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both). End reasons are `stopped`, `cancelled`, `retired`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`). Retirement is an explicit event of the watched document incarnation (last frame `[["r", null]]`, then `retired`), never inferred from a `null` value. An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. -- Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. `watchEvents()` still delivers every event batch in order, with one `snapshot` past 100 pending batches. +- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. +- Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`. +- Document watches follow their address (kind, scope, and key) instead of one incarnation: `DocumentWatch` is `WatchHandle>` (no `null`). A retirement no longer ends the watch with a last frame `[["r", null]]` and `retired`: the next call has `retired: true`, the last value the watch was given, and no ops (folded with what is pending; a change made in the retiring commit itself is not shown), and a document created at the address later arrives on the same watch as `[["r", value]]`, possibly in the same call. A task document's watch ends `ended` once the retirement call of the commit that makes its task terminal has returned; Session close or failure still ends it first if that call was not delivered yet (a watch never started). Acquiring an absent document still gives `undefined`. A watch pins the incarnation at its address, nothing while it is absent. Migration: test `change.retired` instead of a `null` value or a `retired` end, and stop a watch whose document may be retired for good, as it otherwise stays until the Session ends. +- Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. - Removed the Chord state getters `Session.documentState()`, `Conversation.viewState()`, and `Harness.taskGraph()`, and the `DocumentState` type; use `watchDoc()`, `Conversation.watch()`, and `Harness.watchTaskGraph()`. Durable no longer uses Chord's `replicatedState()`. -- Opening a SQLite storage migrates it to schema version 3 automatically, stamping conversations, tasks, and submissions with a commit sequence and indexing every stamp so a commit log can be read in ranges. A migrated database cannot be opened by 1.4.0 or earlier. +- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches and views see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first; a document the commit retires shows as its retirement only, without the operations stored before it), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()` or `Conversation.watch()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- The change type of `FileSystem.watch()` in `@earendil-works/pi-durable/env` is renamed from `WatchChange` to `FileChange` (same shape: `{ paths }`, `{ overflow: true }`, or `{ error }`), so `WatchChange` names only a watch listener's change. Migration: import `type FileChange` from `@earendil-works/pi-durable/env` where `WatchChange` was imported from it. +- `HostCallback` gains `"telemetry"`: a `host_error` report about a telemetry adapter that threw while a commit was published. Exhaustive switches over `HostCallback` need a case for it. +- `ConversationView.entries` is a window over the transcript instead of the active transcript: the fork-aware history `Conversation.entries()` scans, in entry ID order, with head markers (compaction summaries, resets, handoffs) appended in place instead of cutting the entries before them. A watch starts with the newest 50 entries; new entries are appends, and past 100 the oldest leave in one splice down to 50, which also sets the new `ConversationView.before`, the ID of the entry just before the window (absent when the window starts the transcript). First values and overflow replacements (`[["r", view]]`) hold at most `max` entries, never the whole transcript. Migration: read the active transcript the model sees from `Conversation.context()`; render a transcript longer than the window with `ConversationTranscript`, or page older entries with `Conversation.entries()`; a client that rebuilt the head cut from the view's operations no longer gets one ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `Conversation.watch(context, window?)` and `Harness.observeConversation(id, observer, context, window?)` take a `ConversationWindow` (`from`, `min` default 50, `max` default `2 * min`): `min`/`max` bound the window, and `from` starts the first value at that entry ID when at most `max` entries are at or after it, otherwise at the newest `min`; it picks only that first value, so every entry committed after the attachment is appended, and a `from` past the newest entry starts empty. Bounds that are not integers, or `max` below `min`, reject with a `RangeError`. Each watch and observer has its own window, so the observers of one conversation no longer share one change object or one value; they share the document values ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `pi.live` is no longer empty after a run that ended without an answer: it keeps `runFailure` (below) until a later run starts. Code that compares an idle `pi.live` with `{}`, or tests for idleness that way, checks `run === undefined` instead ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- Removed `watchEvents()` and its types `AgentEvent`, `AgentEventStream`, `SnapshotEvent`, `MessageChange`, and `ToolEventCall`, and spec §9.4. A renderer renders state instead of assembling it from events: everything the events carried that a renderer draws is in the conversation view, typed for producer and consumer, and new information becomes a field there rather than an event type every client must learn ([#11327](https://github.com/earendil-works/pi/issues/11327)). Migration, for a client of `watchEvents()`: + + Before: + + ```ts + const stream = await watchEvents(harness, conversation.id, context); + initialize(stream.snapshot); // entries, run, generation, tools, compactions, inbox, agent, usage + stream.start(async ({ value: events }) => { + for (const event of events) { + if (event.type === "message_update") applyChanges(event.changes); // streamed text + else if (event.type === "message_end") appendEntry(event.entry); + else if (event.type === "tool_execution_update") applyOutput(event.toolCallId, event.output); + else if (event.type === "run_end") showIdle(); + // ... and every other event type + } + }); + // After a drop: attach again; the new snapshot replaces everything held. + ``` + + After: + + ```ts + const transcript = new ConversationTranscript((query, limit, ctx) => conversation.entries(query, limit, undefined, ctx)); + const watch = await conversation.watch(context); + render(transcript.merge(watch.value)); + watch.start(async ({ value }) => render(transcript.merge(value))); + + function render(view: TranscriptView): void { + const live = view.docs["pi.live"] as LiveState; + drawItems(view.items); // entries, a "load more" row per gap, "load earlier" at the top + if (live.generation?.message !== undefined) drawStreaming(live.generation.message); + for (const slot of live.tools ?? []) drawTool(slot); // status, running output, details, result entry + if (live.runFailure !== undefined) drawFailure(live.runFailure); // why the last run ended without an answer + setBusy(live.run !== undefined); + } + // A "load earlier" or "load more" row scrolled into view: + render(await transcript.loadEarlier(context)); + render(await transcript.fillGap(gap, context)); + // After a drop: only what was missed when it fits the window, otherwise the newest entries and a gap. + const resumed = await conversation.watch(context, transcript.resume); + ``` + + The watch's operations carry the same deltas the events did (text and thinking appends, tool-call argument appends, output trims and appends), for a client that forwards them, for example to a remote replica. Where each event went: + + | event | state | + |---|---| + | `snapshot` | the watch's first value, `watch.value`, its window bounded by `max` | + | `run_start`, `run_end` | `pi.live.run` appears, disappears | + | `turn_start`, `turn_end` | `pi.live.run.taskId` moves to the next generation; the task graph's `pi.generation` nodes, `completing` at their hold | + | `message_start`, `message_update` | `pi.live.generation.message`, the streamed answer, with its `usage` | + | `message_end`, `entry_appended` | the entry appended to `entries` | + | `tool_execution_start`, `tool_execution_update`, `tool_execution_end` | the slot in `pi.live.tools` or `nestedTools`: `status` pending, running, done; `output`, `details`, `diagnostics`; `entry` once done (absent after a fault); a nested call's `arguments`, its `summary`, and its result in the caller's `NestedCallDoc` | + | `auto_retry_start`, `auto_retry_end` | `pi.live.generation.retry` set, removed | + | `deferred_poll` | `pi.live.generation.deferred.pollAt` | + | `compaction_start`, `compaction_end` | a status in `pi.live.compactions` appears, disappears; `pi.live.compactionFailure` when it failed; the summary entry when it is placed | + | `inbox_update` | `pi.inbox` | + | `agent_changed`, `usage_changed` | `pi.agent`, `pi.usage` | + | `submission` | `Submission.status()` and `wait()`, `Harness.submission(id)`; queued items in `pi.inbox`, the run's inputs in `pi.live.run.inputs` | + | `task_failed` | the task graph (`completing` with the outcome's status), `Harness.getTask()`, and `onReport`'s `task_faulted`; for a run, `pi.live.runFailure`; for a compaction, `compactionFailure`; for a tool, a done slot without `entry` | + | a run that ended without an answer (`run_end` with an `unanswered` `submission`) | `pi.live.runFailure`: `reason` (`model_error`, `no_model`, `aborted`, `reset`, `faulted`, `orphaned`), `detail`, `taskId`, `inputs` | + + Some event data has no state in the view. A tool's arguments after repair and `beforeTool` at its start: in the nested slot's `arguments` for a nested call, only in the tool task's checkpoint for a model-issued one. A submission's settlement: `Submission.status()`/`wait()` and `Harness.submission(id)`; an input settled without a run (withdrawn, `stale`, aborted while queued, `run_failed`) just leaves `pi.inbox`. A failed task's error: `Harness.getTask()` and `onReport`; the task graph keeps only live tasks, not terminal outcomes. A nested call's full result: its caller's `NestedCallDoc` (`watchDoc()`), which retires when the caller settles. Exact turn boundaries: the run's `taskId` and the task graph, with no explicit event. Whether a compaction did anything: its `CompactionResult` (`waitForTask()`), and the summary entry when it is placed. +- Restart-limited tasks no longer park: a task cut off `settings.restartLimit` times in a row ends instead, visibly. A tool call ends with an error result (diagnostic code `restart_limit`) and its task fails with detail `{ reason: "restart_limit" }`, a compaction fails into `pi.live.compactionFailure`, a generation ends its run with `pi.live.runFailure` reason `restart_limit` (its inputs settle `unanswered` with that reason), and a task of an extension's kind ends `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit`. The built-in tasks are reported as `task_interrupted` (below). An abort handler cut off at the limit still faults its task, but a run someone aborted now ends with `runFailure` reason `aborted`, the cut-off as `detail`, and an aborted compaction leaves no `compactionFailure`. `Harness.retryTask()`, the `restart_limit` blocked reason of `TaskInspection` and `TaskGraphNode`, and the `task_parked` report (and its telemetry value) are removed. Migration: a parked task kept its conversation busy until `retryTask()`; it now ends, so show `runFailure`, `compactionFailure`, the tool's error result, or the `task_interrupted` and `task_faulted` reports instead, and to run the work again, submit it again (a new input, `compact()`, or a new task); raise `restartLimit` before the task is next reserved to let it run longer ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- A generation whose model request was in flight when the host died no longer resends it on open: the attempt counts as failed under `settings.retry`. Its partial becomes an aborted entry, as before; the next attempt waits for the backoff, shown in `pi.live.generation.retry`, and with no retry left the run ends with `pi.live.runFailure` reason `interrupted` and the inputs settle `unanswered` with that reason. A deferred poll whose fetch was in flight is charged the same way, even when the replacement host's clock is behind, and its next attempt polls the same deferred response again instead of submitting a new request; a run that ends instead cancels it. A close still resends. `restartLimit` still bounds these phases: a charge that itself keeps being cut off ends the run `restart_limit`, past the limit without converting the partial ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- Watch listeners and synchronous observers take Chord's tagged watch shapes. Durable re-exports `WatchCall` and `NativeWatchCall` instead of `WatchChange` and `NativeWatchChange`; a listener gets `{ kind: "value", value, ops, retired }`. Chord's `{ kind: "failed", error }` call, for a source that recovers, never comes from Durable, whose failures end the watch. `WatchEnd` is tagged by `kind`, and a watch whose acquisition context is aborted ends `aborted` (was `cancelled`). Before: `watch.start(async ({ value, ops }) => render(value, ops))`, `if (end.reason === "cancelled")`. After: `watch.start(async (call) => { if (call.kind === "value") render(call.value, call.ops); })`, `if (end.kind === "aborted")`. +- `Observer` (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`) has the watch's shapes: `advance(call: WatchCall, context)` (always a `value` call) and `end(end: WatchEnd)`: `ended` at close or after a task document's retirement with its task (was `undefined`), `failed` with the error that failed the Session (was `{ error }`), and `listener_error` with the error for an observer that threw (was `{ error }`), now also for one whose `advance` returns a promise that rejects. Before: `{ advance: ({ value }) => push(value), end: (failure) => failure && fail(failure.error) }`. After: `{ advance: (call) => call.kind === "value" && push(call.value), end: (end) => end.kind === "failed" && fail(end.error) }`. +- `subscribeCommits(listener)` is `observeCommits(observer)` with the observer shape: `{ advance(publication, context), end(end) }`, returning `{ detach() }`. Its observation has no `value`: commits are events, and the Session does not know Storage's newest `seq` before its own first commit. `end` comes once, `ended` after the last publication at close or `failed` when the Session fails; an observer that throws (or whose `advance` rejects) is reported, detached, and ended `listener_error` (a throwing listener stayed subscribed). Before: `const unsubscribe = harness.subscribeCommits((publication) => apply(publication)); unsubscribe();`. After: `const observation = harness.observeCommits({ advance: (publication) => apply(publication), end: () => {} }); observation.detach();`. +- `HarnessOptions.onReport(report)` takes one `Report`: the fields of `ReportInfo`, tagged by `kind` (was `type`), and the `error` (was the first argument). `session_failed` is `storage_failed`, and `task_outdated`'s reason `incompatible_task` is the scheduler's own word for why the replacement cannot take the task: `task_too_old` (older than the task) or `migration_failed` (newer without a `migrate`), as a blocked task shows. The `pi.durable.report` event and `pi.durable.reports` metric keep their `type` attribute, with the new values. Before: `onReport: (error, info) => { if (info.type === "session_failed") restart(error); }`. After: `onReport: (report) => { if (report.kind === "storage_failed") restart(report.error); }`. +- `DocumentCommitChange` is tagged: a document change is `{ type: "document", kind: "change", record, conversationId, version, value, ops }` and a retirement `{ type: "document", kind: "retire", record, conversationId }` (was `value: null` and `version: undefined`). Before: `if (change.type === "document" && change.value !== null) use(change.value)`. After: `if (change.type === "document" && change.kind === "change") use(change.value)`. +- `CompactionStatus.reason` and `CompactionFailure.reason` in `pi.live` are `trigger` (what started the compaction; a run failure's `reason` is why it failed). `pi.live` is version 2: its `migrate` renames the field in a stored version 1 value when it is read, and the next commit that writes `pi.live` stores it at version 2, so code built against version 1 cannot read a conversation this version wrote to. Before: `live.compactionFailure?.reason`. After: `live.compactionFailure?.trigger`. +- `TaskInspection.state` uses `TaskState`'s word: `{ kind: "pending", migrates }` (was `ready`). Before: `state.kind === "ready"`. After: `state.kind === "pending"`. +- `FileChange` (`FileSystem.watch()` in `@earendil-works/pi-durable/env`) is tagged by `kind`: `{ kind: "paths", paths }`, `{ kind: "overflow" }`, or `{ kind: "failed", error }` (was `{ paths }`, `{ overflow: true }`, `{ error }`), and `FileWatcher.close(context)` is `stop(context)`. Before: `if ("overflow" in change) rescan(); else if ("paths" in change) reload(change.paths); await watcher.close(ctx);`. After: `if (change.kind === "overflow") rescan(); else if (change.kind === "paths") reload(change.paths); await watcher.stop(ctx);`. +- `SessionEnd` (`Session.closed`, `Harness.closed`) is tagged by `kind`: `{ kind: "closed" }` or `{ kind: "failed", error }` (was `reason`). Before: `harness.closed.then((end) => { if (end.reason === "failed") reopen(end.error); })`. After: `harness.closed.then((end) => { if (end.kind === "failed") reopen(end.error); })`. +- `ReportFrom`, the second argument of a task's `runtime.report(error, from)`, is tagged by `kind`: `{ kind: "extension", extension? }` or `{ kind: "host", callback }` (was `type`). Before: `runtime.report(error, { type: "host", callback: "env" })`. After: `runtime.report(error, { kind: "host", callback: "env" })`. +- `afterTool` runs on every result a tool task writes, not only on an executed call's: also on the error results the Harness writes itself, for a tool that does not resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent them), a block (`blocked`), an interruption (`interrupted`), the restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`); the diagnostic code tells them apart. Such a result is bounded with the tool's output limits after the hooks, never keeps a `structuredOutput`, and applies `control` only when the call completes; a nested call a hook turns into a success gets its bounded output as value unless the tool declares a schema. Once a tool task was cut off more than `settings.restartLimit` times its hooks no longer run, nor in the last attempt the scheduler gives its abort handler, and an abort's hooks delay `Conversation.abort()`. A non-replay-safe tool interrupted that often now ends with `restart_limit` too. Hooks that assumed the tool ran (cast `call.arguments`, read `structuredOutput`) check `result.diagnostics` first ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- A call to a tool its request did not offer gets a `pi.tool` task like any call, flagged `unoffered: true` in its input and its `pi.live.tools` slot, instead of a result the generation wrote without one: the task resolves nothing (a deferred tool the agent has still never runs), settles `tool_unavailable`, and runs `afterTool`, whose `control` then counts toward the round (one such call no longer prevents `terminate`). Its slot starts `pending` with a `taskId` instead of `done`, and its result entry lands in a later commit than the tool-calling answer. Controls are applied in call order. `pi.tool` is version 3 and `pi.generation` version 2 (a sequential round keeps such calls pending until their turn): an older Harness leaves both blocked (`task_too_old`) instead of resolving the flagged name ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `SubmissionRecord` gains a `type: "continue"` variant (`placed` without `entry`, `done` with `answer`, `unanswered`), so narrowing a record by `status` alone no longer guarantees an `entry`: check `type === "input"` too. A generation whose context does not end in a user message or tool results, system messages aside, now ends its run `unanswered` with the new `RunFailureReason` `nothing_to_answer` without a request; an empty conversation no longer faults its generation. `ConversationHandle.submit()` takes any `SubmissionDraft` (was `InputSubmissionDraft`), as `Conversation.submit()` does ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer` and `turn_limit`, also an `unanswered` submission's `reason`: exhaustive switches over it need cases ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). `pi.live` is version 3: its `migrate` adds `turns: 0` to a stored run without it, version 1 or a version 2 an earlier unreleased build wrote, whose run in flight otherwise failed its next tool round with "Value contains a non-finite number" ([#11043](https://github.com/earendil-works/pi/issues/11043)) + +- A Session's close ends synchronous observers (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, `observeCommits()`) `failed` with the error "session closed" instead of `ended`, so an adapter following a Session (Chord's `self.follow`) fails and follows the next one; `ended` stays for an observed owner that ended (a task document's task), and watches still end `ended`. Before: `end: (end) => { if (end.kind === "ended") done(); }`. After: `end: (end) => { if (end.kind === "ended" || end.kind === "failed") done(); }`. ### Added -- Storages rebuild their commit log from what they store: every built-in storage implements `ReplayableStorage`, with `log(after)` (the changes after a sequence as `{ seq, writes }` records that replay to the same reads, historical document reads included, or the whole state without `after`), `replay(record)` (a commit at an explicit sequence, gaps allowed), and `head()`. The log holds what survived: each task and submission once, at its last write, and current-only documents from their newest base on. `LogRecord` and `LogWrite` describe it; fork copies appear as the creations they became. -- `@earendil-works/pi-durable/sync` moves sessions between devices through a remote store. Every device continues a session as its own: `uploadSession()` uploads the log after the last sequence the copy shares with the remote as one gzip batch into a slot the store creates once per start sequence, and reports a copy as diverged when another device's upload created that slot first. `pullSession()` catches a copy up, resumes an interrupted pull, or restores; `foldSession()` with `compactRecords()` folds the history into a snapshot, keeping folded slots taken. `FlushScheduler` and `isUrgentCommit()` decide when to upload, retrying failed uploads with backoff; A store creates a slot only where the session's history ends (`not_head` otherwise, which the uploader treats as diverged), lists the sessions changed after a cursor with their heads and metadata, and keeps a metadata object per session for the host; `FileSystemSyncRemote` is a store in a directory; `FileSyncStateStore` keeps sync state in a directory. `uploadSession()` reports a copy whose head fell below what it had uploaded as `behind` and uploads nothing; `pullSession()` brings it back. `LogInterruptedError` marks a log read that a commit interrupted. +- `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and stores each `commit()` call in one store transaction (`ChordStore.commitSeries`), one seq and outbox row per batch, publishing each commit in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). +- Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. +- Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit when it is published after its storage call settled, and for a deferred one when memory adopts it, before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). +- `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `pi.live.runFailure` (`RunFailure`, `RunFailureReason`) records why the newest run ended without an answer, `model_error`, `no_model`, `aborted`, `reset`, `faulted`, or `orphaned`, with the run's task and inputs and a `detail` where one exists (the error text, the missing model, the fault, the orphan reason). The commit that ends the run sets it; a later commit that starts a run removes it ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `RunFailureReason` gains `interrupted` (the host stopped while the model was answering, with no retry left) and `restart_limit` (the run's task was cut off `settings.restartLimit` times in a row) ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- `onReport` gets `task_interrupted` (`reason` `restart_limit` or `interrupted`, `restarts`, and the task) when crashes end a built-in task: a tool call, a compaction, or a generation at the restart limit, or a run that ends `interrupted`. The telemetry report attributes gain the type `task_interrupted` and the reason `interrupted` ([#11139](https://github.com/earendil-works/pi/issues/11139)). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. +- Continue submissions: `submit({ type: "continue" })` starts a run on the context as it is, without a user message, for example after a run that ended without an answer (`pi.live.runFailure`). Only an idle conversation admits one (a busy one throws `ConversationBusy`; it never queues); items a failed run left queued are placed first and join the run. A context that ends in an answer, also a text answer cut off at its output limit, has nothing to continue (`nothing_to_answer`); a cut-off answer with tool calls is continued, as its calls end in error results. Tasks and tools submit one through `ConversationHandle`. The `pi.durable.submit` span starts a root trace for it, as for an input ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- Per-conversation turn limits: `AgentState.maxTurns` (`configure({ maxTurns })`, `null` clears; a `RangeError` for anything but an integer 1 or more) bounds the tool rounds one run takes. The round that reaches it runs its tools, then the run ends `unanswered` with the new `RunFailureReason` `turn_limit`, like a failed run, so `settings.afterFailedRun` decides the queued inputs and a continue resumes it on the round's results. The count is `pi.live.run.turns`; every run starts at 0, and an `onYield` continuation starts it over. Task-owned children and forks inherit the limit; `Agent.maxTurns` exposes it ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `whenIdle: "reject"` on input submissions: an idle conversation rejects the input with the new `ConversationIdle` and writes nothing, so it never starts a run. With `whenBusy: "steer"` it steers the current run or is refused, in one admission instead of a host's check and a separate submit that can race the run's end. A known request ID still returns its submission ([#11149](https://github.com/earendil-works/pi/issues/11149)) +- `Conversation.submit(tx, draft)` admits a submission inside the caller's commit (`Harness.commit()`, `Conversation.commit()`) with the same rules as `submit(draft, context)`, and returns its ID: a host checks its own documents and `pi.live`, admits, and writes its own state in one commit, which stores all of it or none. `ConversationHandle.submit(tx, draft)` does the same inside a task's or tool's own commit. The Harness supplies the clock, queue modes, the `pi.durable.submit` span (covering the admission only; its `result` is provisional until the commit is stored), and the request for progress. It must come before the commit's first table write, once per commit, and throws `ReadAfterWrite` otherwise on every path ([#11149](https://github.com/earendil-works/pi/issues/11149)) +- `HookApi.entry()` and `entries()`, and `TaskRuntime.entries()`: a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and on a rerun ([#11141](https://github.com/earendil-works/pi/issues/11141)) +- Run-end tasks: a `GenerationHooks.afterRun(run, api)` hook returns `{ tasks }`, built with the typed `runEndTask(task, input)`, after a run answered (a final answer or a `terminate` round; not when an `onYield` continuation is taken, not after a failure or an abort). The answer is committed and the run's inputs settle `done` first, so `Submission.wait()` resolves at the answer; the generation then creates the tasks as its children and waits for them, keeping `pi.live.run` (with `run.afterRun`) and the conversation busy, so idle waits include them, Stop aborts them, and a crash resumes them. Input queued meanwhile, steers included, is placed when they are terminal and starts the next run; a task admits a model-visible write or a fix-run follow-up in its own commit with `submit(tx, draft)`. A crash before the tasks exist asks the hooks again; a broken request (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) is reported and skipped. `run` names the run's entries: `first` (`pi.live.run.first`, carried into `runFailure.first` so a continue covers a failed run's history), `answer`, and `last` ([#11141](https://github.com/earendil-works/pi/issues/11141)) + +- `registryReader(value, context)`: a `RegistryReader` over a Chord v2 registry value whose composition is a registry snapshot (`RegistryValue`): `snapshot()` reads the current composition, `subscribe` observes its compositions synchronously inside Chord's batch, so a Harness opened in a Chord object takes its extensions from a Chord registry. +- `GenerationHooks.beforeProviderRequest({ model, payload }, api)` replaces the provider request body of a generation request, for that request only, e.g. to set a provider-specific field. The body is in the provider API's format; the hook returns `{ payload }` or `undefined`. Hooks chain in extension order and run each time pi-ai builds a body; a throw is reported and the chain continues. Compaction requests do not use it ([#10804](https://github.com/earendil-works/pi/issues/10804)) ### Changed -- Document, conversation view, task graph, and event watches use Chord's watch handles (`Watch`, and `NativeWatch` for event batches). +- Document, conversation view, and task graph watches use Chord's watch handles (`Watch`). +- Group commit: a Session no longer holds its line while a commit is stored. Commits that arrive while one storage call is in flight go out together in the next call, each with its own seq and its own outcome, so the conversations of one Session share storage round trips instead of queueing behind each other's. A job runs while earlier commits are being stored until it touches a key one of them wrote (a record, a document, a conversation's entries, a task's membership or cancellation state); then it waits until that commit is stored and published. Outcomes are delivered in line order, and every job after a commit that fails the Session gets `SessionFailed`. One Harness serving 128 conversations over storage with a 4 ms round trip went from about 5 to about 440 agent runs per second (median run 25 s to 0.28 s); with local SQLite or the Chord store, throughput at 128 conversations rises by about 30%, while with in-memory storage and no latency each commit costs somewhat more CPU than before (up to 25% fewer runs per second). At most 256 jobs wait for their outcome. See `docs/group-commit.md` and spec §4. +- A stored commit is published after its storage call settled, in line order, the commits of one call a macrotask apart (inside the storage's `publishing()` scope if it has one): the read caches and memory adopt it, then its telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run, so two views never show one commit apart. `observeCommits()` observers run one microtask later, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. A call that settles after something else failed the Session publishes nothing; what it stored is durable and appears after reopen. +- A normal agent run makes no storage reads: the Session keeps recently read conversation, task and submission records (bounded by count, and by live work for tasks), the newest entry of each conversation, and the task documents of live tasks, and the scheduler keeps the context range of a running conversation. Read results (`tx.task()`, `tx.conversation()`, `getTask()`, `runtime.entry()`, published entries) are frozen objects shared with these caches; mutating one throws. +- A tool slot's `running` status and a nested call's final arguments, and a generation's attempt, are stored with the next commit, like streamed progress: a run with one tool call stores 12 commits instead of 14. After a crash a slot may show `pending` while its task's checkpoint is at execution; the rerun sets it again. +- A wait whose commit is still being stored when the Harness closes ends with the closed error; the commit is still stored and published before close completes. +- Cancellation is checked when a job starts, which can be earlier than before: an invocation-bound submit queued behind an abort mark that is not stored yet lands, and the abort's cascade then aborts its run. +- The scheduler's per-commit work no longer grows with the number of conversations: after a commit it re-checks idleness only for the conversations above the tasks that became live or ended, expires idle context ranges from an expiry-ordered index, and `wake()` takes the earliest sleep from the sleeping tasks and scans live tasks only while nothing runs. Before, every commit re-walked every conversation read in the last `contextRetentionMs` (default 10 minutes). One Session with 2048 conversations went from 100 to 805 agent runs per second (4096: 46 to 772; local SQLite at 2048: 88 to 459), with CPU per run flat at about 1.6 ms instead of growing linearly. +- `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between publication and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. +- The abort signal of a newly marked running task fires after `subscribeCommits()` listeners instead of before them. +- A throwing Durable commit observer fails the Session after `Storage.commit()` returned, after the other observers and the host listeners got the commit; the commit still stands. When Storage also rejects that commit after adopting it, the storage error fails the Session and the observer's error is reported as `listener_error`. ### Fixed -- Fixed a conversation watcher (`watchEvents()`, conversation views) holding a copy of a streamed text or tool call per partial commit, through Chord append ops that kept the whole string alive; a 140 KB answer followed by a 140 KB tool call held well over 100 MB ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed a tool round with `control.handoff` ending the run on the tool-calling answer: the handoff's `pi.reset` entry is written and the same run continues with a new generation in the new context, so the submission settles with the eventual answer. Queued writes and selected steers are placed after the handoff text, follow-ups wait for the run's final boundary, and a queued reset still ends the run `unanswered` with reason `reset`. `terminate` remains the way to end a round without another model request. +- Fixed a conversation watcher (conversation views) holding a copy of a streamed text or tool call per partial commit, through Chord append ops that kept the whole string alive; a 140 KB answer followed by a 140 KB tool call held well over 100 MB ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed a nested call's error summary keeping the call's whole output alive through a slice of it ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed task span error messages and codemode call previews keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed stored assistant messages lacking `thinkingLevel`: a generation records the Pi thinking level its request asked for on the response, also after polling a deferred one, as pi-agent-core's agent loop does +- Fixed partial commits polluting `Object.prototype` when a streamed tool call's arguments gained a `__proto__` member after an earlier commit: the merge read the missing member through to `Object.prototype` and wrote into it +- Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. +- Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. +- Fixed a host that crashes during a model request resending it forever, adding an aborted entry each time: the interrupted request is a failed attempt under the retry policy, and the run ends with `runFailure` reason `interrupted` once no retry is left; other work cut off `restartLimit` times ends instead of keeping its conversation busy ([#11139](https://github.com/earendil-works/pi/issues/11139)) +- Fixed the built-in `read` resolving to `""` for an image in Code Mode scripts, so `image(await tools.read(...))` threw: `read` declares a `structuredOutputSchema`, a text file's text or an image's `ImageContent`, as the coding agent's `read` does ([#11015](https://github.com/earendil-works/pi/issues/11015)) +- Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254)) +- Fixed `tool_search` and Code Mode's `searchTools()` ranking a tool below tools that only mention its name, as the coding agent did before [#11440](https://github.com/earendil-works/pi/pull/11440) and [#11625](https://github.com/earendil-works/pi/pull/11625): `thread_read` with `limit: 1` returned `queue_read`, and `peekaboo app` missed `mcp__peekaboo__app`. Both now rank like the coding agent: a query that is exactly a tool's name ranks it first, a tool matching more of the query's words ranks above one matching fewer, a query word in the name outweighs any number of mentions in the description, and a tool whose whole name is in the query gets a large bonus. `searchTools()` also searches namespace instructions, as a low-weight field that does not lower a tool's score ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and Code Mode's `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index fe0adb7e8..b9e54a068 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -2,7 +2,7 @@ > **Experimental.** The API changes without notice between releases. -A durable agent harness. Conversations, model turns, tool calls, and your own state are committed to storage before anything is shown. If the process dies mid-turn, reopening the storage picks the work up where it stopped. +A durable agent harness. Conversations, model turns, tool calls, and your own state are committed to storage before anything is shown. If the process dies mid-turn, reopening the storage picks the work up where it stopped; a model request it died during counts as a failed attempt and is retried, so a request that keeps killing the process ends (see [Persist and Resume](#persist-and-resume)). Built on [`@earendil-works/pi-ai`](../ai/README.md) for model access and `@earendil-works/chord` for document state. @@ -81,10 +81,10 @@ Every async call takes a Chord `Context`, which carries cancellation and its cur `HarnessOptions.telemetry` accepts a host-owned `TelemetryContext` from `@earendil-works/pi-telemetry`, with optional `propagation` and `metrics` capabilities; omitted, it is a no-op. `DURABLE_TELEMETRY_SCHEMA`, exported from the package root, documents its typed spans, events, and metrics. Attributes contain IDs and control metadata, never messages, arguments, or results; IDs point to the transcript. -- `pi.durable.submit` covers the public submit call and its commit. Each input starts a new root trace with a link to its caller's current span; a write is a child of its caller. Duplicate `requestId` preserves the original submission and carrier. Internal admissions, such as compaction writes, emit events on their phase span without a submit span. +- `pi.durable.submit` covers the public submit call and its commit; for `submit(tx, draft)` it covers the admission only, its `result` is provisional until your commit is stored, and the commit's events go to your commit's context. Each input starts a new root trace with a link to its caller's current span; a write is a child of its caller. Duplicate `requestId` preserves the original submission and carrier. Internal admissions, such as compaction writes, emit events on their phase span without a submit span. - `pi.durable.phase` covers one run or abort phase of every task, including its decision commit, never the next phase, a durable wait, or a restart. Its attributes identify the task, phase, attempt when present, `resumed`, and outcome. Tool phases also identify the tool and call. Nested tasks and background compactions inherit the creating phase; model requests receive that phase context. Hooks add no spans. - Failed phase outcomes set error status with a fixed category (`task_faulted`, `task_failed`, or `task_orphaned`) and the outcome's diagnostic message, truncated to 500 characters (the reason for an orphaned task). Diagnostics can contain caller content; no error detail or exception name is recorded. An `aborted` outcome keeps status `ok`, including interruption by close or cancellation. -- Every `onReport` report also emits a `pi.durable.report` event on its current span. Without a span, the NOOP span drops the event; no root report span is created. Only `type`, optional fixed `reason`, `extension`, and `task.id` are recorded; no error payload is included. `pi.durable.reports` counts all reports by `type` only, including those without a span. Clock failures report without a phase context. The `onReport` callback is unchanged. +- Every `onReport` report also emits a `pi.durable.report` event on its current span. Without a span, the NOOP span drops the event; no root report span is created. Only `type` (the report's `kind`), optional fixed `reason`, `extension`, and `task.id` are recorded; no error payload is included. `pi.durable.reports` counts all reports by `type` only, including those without a span. Clock failures report without a phase context. The `onReport` callback is unchanged. - Committed entries and settled submissions emit `pi.durable.entry` and `pi.durable.submission`. Committed running phase changes emit `pi.durable.checkpoint`; a nested call answered from its stored result emits `pi.durable.replay`. - Propagation stores optional opaque `trace` carriers in submission and task JSON without a storage migration. Phase spans extract their task's stored carrier, or start at the adapter root when absent or unknown. `TaskOptions.trace` inherits the creator's carrier when absent, accepts an explicit carrier override, or uses `"root"` to start at the adapter root without storing a carrier. A run started for several queued inputs takes the first input's trace; its first invocation's phase spans link to the other inputs through immutable task `links`. A steer stays in the running trace and its submit span links to that run's stored carrier. A reopened task's first invocation has `resumed: true` in the same trace. - Metrics record `pi.durable.phase.duration` (milliseconds; task, phase, outcome) and `pi.durable.tasks.active` (live tasks; task name), without record IDs. Closing the Harness releases its active counts. @@ -213,11 +213,11 @@ The default `setTimer` declines the timers that only free memory in Workers, so ### Failures and Recovery -**Crash loops.** Work that deterministically crashes its process, by exhausting memory or CPU before its next checkpoint, would otherwise be resumed and crash again forever. Open counts each task a crash, eviction, or reset cut off; a close is not counted, and a new state the task commits starts the count over. Once the count reaches `settings.restartLimit` (default 5; `Infinity` never parks), the task is parked: `inspect()` and the task graph show it as blocked with reason `restart_limit`, `wake()` does not count it as active, and `onReport` gets it once as `task_parked`. Every task running when a process dies is counted, so show parked tasks and offer `harness.retryTask(id)`, which runs one again. `abortTask()` still runs its abort handler; an abort handler cut off that many times ends the task `faulted`. +**Crash loops.** Work that deterministically crashes its process, by exhausting memory or CPU before its next checkpoint, would otherwise be resumed and crash again forever. A model request the host died during is a failed attempt under `settings.retry`: its partial answer is kept as an aborted entry, the next attempt waits for the backoff (shown in `docs["pi.live"].generation.retry`), and with no retry left the run ends with `docs["pi.live"].runFailure` reason `interrupted`. A deferred response that was being fetched is polled again as the next attempt, not requested again, and cancelled when the run ends. A close is not a crash: a request it stopped is sent again. Other work is counted: open counts each task a crash, eviction, or reset cut off, and a new state the task commits starts the count over. Once the count reaches `settings.restartLimit` (default 5; `Infinity` never ends a task for it), the task ends instead of running again: a tool call with an error result (diagnostic code `restart_limit`), a compaction failed into `compactionFailure`, a run with `runFailure` reason `restart_limit` (also when charging an interrupted request keeps crashing), and a task of your own kinds `faulted`, reported as `task_faulted` with reason `restart_limit`. The built-in ones are reported as `task_interrupted`, as is a run that ends `interrupted`, so a crash loop always reaches `onReport`. Every task running when a process dies is counted, not only the one that killed it. An abort handler cut off that many times ends its task `faulted` too; a run someone aborted still ends `aborted`, with the cut-off in `detail`. -**Reports.** `onReport(error, info)` says what happened in `info.type`: `session_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_parked`, `migration_failed`, `task_outdated`, or `task_error` for a task, with `info.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `info.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `info.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. +**Reports.** `onReport(report)` gets one `Report`: the `error` and what happened in `report.kind`: `storage_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_interrupted` (crashes ended it, with `report.reason` and `report.restarts`), `migration_failed`, `task_outdated` (`reason` `missing_task`, `task_too_old`, or `migration_failed`, the words a blocked task shows), or `task_error` for a task, with `report.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `report.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `report.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. -**Queued inputs after a failed run.** A run that fails with a model error leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. +**Queued inputs after a failed run.** A run that fails with a model error, a missing model, nothing to answer, its turn limit, or crashes (`interrupted`, `restart_limit`) leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. **Blocked tasks.** Tasks blocked on a missing or unmigratable definition show their reason in the task graph too, and a registry change that unblocks them makes `onWake` report the Session active, so a host need not poll them. @@ -257,15 +257,15 @@ registry.install(createCodingTools({ images: await createNodePhotonImages() })); A model without image input sees a placeholder instead of the image, and `read` says so in a diagnostic. `/images/node` prepares images in a worker thread, so a large photo (a second or two of work) never blocks the event loop; the worker exits after 30 s without images, freeing Photon's WebAssembly memory (`createNodePhotonImages({ worker: false, idleMs })`). Elsewhere, `createPhotonImages()` and `/images/cloudflare` work on the calling thread, and decoding takes the image's full size in WebAssembly memory, which does not shrink again; leave the processor out on memory-constrained hosts such as Cloudflare Workers unless images stay small. -Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: +Define your own tool with a JSON Schema, built here with `@earendil-works/pi-schema`. `defineTool()` types `args` from `parameters` (through pi-schema's `Static`; other JSON Schema gives `unknown`), which the Harness coerces and validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: ```typescript -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, str } from "@earendil-works/pi-schema"; const count = defineTool({ name: "count", description: "Count from 1 to n", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (args, api) => { for (let i = 1; i <= args.n; i++) api.output(`${i}\n`); return {}; @@ -274,7 +274,7 @@ const count = defineTool({ registry.install(defineExtension({ name: "count", tools: [count] })); ``` -Each call runs as its own durable task. Its intent is committed before `execute()` runs. If the process dies mid-call, the tool reruns on reopen only when it is declared `replay: "safe"`; otherwise the model gets an `interrupted` error result with the output committed so far. Throwing from `execute()` gives the model an error result. A result can also return `usage`, which is added to the conversation's [usage](#usage-and-cost). It can also return `control: { terminate: true }`: when every result of the round asks for it, the run ends without another model request. +Each call runs as its own durable task. Its intent is committed before `execute()` runs. If the process dies mid-call, the tool reruns on reopen only when it is declared `replay: "safe"`; otherwise the model gets an `interrupted` error result with the output committed so far. A replay-safe call cut off `settings.restartLimit` times in a row ends with a `restart_limit` error result instead of running again. Throwing from `execute()` gives the model an error result. A result can also return `usage`, which is added to the conversation's [usage](#usage-and-cost). It can also return `control: { terminate: true }`: when every result of the round asks for it, the run ends without another model request. A result has three channels: `output` for the model, `structuredOutput` for programs that call the tool (see [Calling tools from tools](#calling-tools-from-tools)), and `details` for UIs. A tool that declares `structuredOutputSchema` returns a matching `structuredOutput`, which the Harness validates. Programs calling a tool without one get its bounded output: one text item as a string, one image as its `ImageContent`, nothing as `""`, and anything else as the content list, for errors too. `structuredOutput` is always the tool's own value: a schema tool's error result keeps the value it returned, and the text of one without is in a `tool_error` diagnostic, as a thrown error's message is. The transcript stores only `output` and `details`: @@ -282,8 +282,8 @@ A result has three channels: `output` for the model, `structuredOutput` for prog const lines = defineTool({ name: "lines", description: "Count the lines of a file", - parameters: Type.Object({ path: Type.String() }), - structuredOutputSchema: Type.Object({ lines: Type.Number() }), + parameters: obj({ path: str() }), + structuredOutputSchema: obj({ lines: num() }), execute: async (args, api, context) => { const text = await readText(api, args.path, context); const count = text.split("\n").length; @@ -292,7 +292,7 @@ const lines = defineTool({ }); ``` -A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, and their `diagnostics` say what went wrong. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. +A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. The built-in `read` returns a text file's text as a string and an image as its `ImageContent`, which code mode's `image()` shows. A later extension's tool with the same name replaces an earlier one where both are selected, and `wrapTool()` decorates whichever tool won: @@ -312,7 +312,7 @@ A tool calls another tool with `api.executeTool()`, as a code mode script or an const testsPass = defineTool({ name: "tests_pass", description: "Run the test suite and say whether it passed", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, context) => { const run = await api.executeTool("bash", { command: "npm test" }, context, { progress: false }); // Absent when bash did not finish, for example on a timeout. @@ -324,7 +324,7 @@ const testsPass = defineTool({ - The caller gets the nested call's `taskId`, `structuredOutput`, `details`, `diagnostics`, `usage`, and `isError`, not the `output` meant for the model. A blocked, invalid, failed, or aborted nested call returns an `isError` result. `executeTool()` itself rejects once `execute()` has returned, when the caller is aborted, or when the Harness closes; cancelling its `context` stops only the wait. - Hooks see a nested call's `call.parent`, so a guard can treat calls from scripts differently. -- While the caller runs, its nested calls show in `docs["pi.live"].nestedTools`, with the arguments they run with and a `summary` (error, duration, usage) once done, and as tool events with `parentToolCallId` and `parentTaskId`. `progress: false` keeps a nested call's running output out of `pi.live`. Before the caller settles, nested calls it left running are aborted, also those an earlier attempt made. The caller's result message does not list its nested calls; their task records keep them. +- While the caller runs, its nested calls show in `docs["pi.live"].nestedTools`, with their caller's `parentCallId` and `parentTaskId`, the arguments they run with, and a `summary` (error, duration, usage) once done; a nested call's full result is in its caller's `NestedCallDoc` while the caller runs. `progress: false` keeps a nested call's running output out of `pi.live`. Before the caller settles, nested calls it left running are aborted, also those an earlier attempt made. The caller's result message does not list its nested calls; their task records keep them. - Each nested call has a key, its position among the caller's nested calls unless the caller passes `{ key }`. A replay-safe caller that reruns after a crash and calls in the same order, or passes the same keys, gets the nested calls it already made back, finished or still running, instead of starting them again. A nested call the crash interrupted before its result was committed follows its own `replay` policy, so a replay-safe tool with external effects still needs its own idempotency. A caller that is not replay-safe never reruns, so after a restart its unfinished nested calls are abandoned, with everything they own, before they run again. - A nested call's `usage` is counted under its own tool; passing it on in the caller's result counts it twice. - Nested results are not kept: they live in the caller's task documents (`NestedCallDoc`, one per nested call, keyed by its key, holding `{ taskId, result? }`) only until the caller settles. The nested call's task record keeps only a small receipt. A stored result is exactly what `executeTool()` returns, without the model's `output`, so an image or a `bash` tail is stored once. @@ -391,6 +391,7 @@ await root.configure( modelTools: [readTool], // of those, offer the model only read; tools may still call bash instructions: "Only read; never edit files.", cwd: "/work/repo", + maxTurns: 50, // tool rounds one run takes at most; unset: no limit }, context, ); @@ -400,6 +401,8 @@ const agent = await root.agent(context); // resolved: model, extensions, tools, Extensions and tools are passed as objects and stored by name, so a stored name outlives its code: after an extension is uninstalled, conversations that select it just stop getting it until it is installed again. `createConversation()`, `fork()`, and `root()` take the same change as `agent`. A task-owned conversation, such as a subagent's, starts as a copy of its owner's conversation's agent. A fork starts with the agent its parent had at the fork entry. The model, prompt, and offered tools of a request are fixed when it is prepared; a change applies from the next request. Tool calls and hooks use the agent as their task phase resolves it, and the environment is built from the current `cwd` at each use, so a `cwd` or extension change can reach calls the model already made. +**Turn limits.** `maxTurns` bounds the tool rounds one run takes: the round that reaches it still runs its tools, then the run ends `unanswered` with `turn_limit`, like a failed run (`runFailure`, and `settings.afterFailedRun` decides the inputs queued behind it). An answer always ends the run, so it is never cut off. The count is `docs["pi.live"].run.turns`; it survives restarts, and every run starts at 0, also one a continue starts (see [Busy Conversations](#busy-conversations)), so a host or tool that continues automatically gets a fresh budget each time and must bound that itself. An `onYield` continuation also starts the count over: an extension that keeps a run going bounds its own loop, reading `pi.live.run.turns` and `pi.agent.maxTurns` with `api.snapshot`. `configure()` throws a `RangeError` for anything but an integer 1 or more (or `null`); a change applies at the end of the current round, and children and forks inherit it like the rest of the agent. + ## Settings Run policy shared by every conversation is passed as `settings`. It is read at every use and never stored, so getters make it live, for example backed by a settings file: @@ -413,7 +416,7 @@ const harness = await Harness.open(storage, { stream: { timeoutMs: 120_000 }, retry: { maxRetries: 3 }, compaction: { reserveTokens: 16384 }, - progress: { partialIntervalMs: 100, outputIntervalMs: 100 }, + progress: { partialIntervalMs: 100, outputIntervalMs: 100, persistWithinMs: 1_000 }, toolExecution: "parallel", contextRetentionMs: 600_000, // idle conversations keep their context in memory this long (checked lazily where background timers are declined, as in Workers) documentRetentionMs: 600_000, // loaded documents nothing observes stay in memory this long after their last use; observed ones always @@ -478,9 +481,12 @@ Everything a UI needs is committed state. `watch()` returns a watch of the conve ```typescript const watch = await root.watch(context); render(watch.value); // the state at attachment -watch.start(async (value, ops) => { - // value.entries: the active transcript - // value.docs["pi.live"]: the running generation (streamed partial, retry, deferred) and tool calls (output, details) +watch.start(async (call) => { + if (call.kind !== "value") return; // Durable's watches end on a failure; they never send a failed call + const { value, ops } = call; + // value.entries: a window over the transcript, newest last; value.before: the entry just before it + // value.docs["pi.live"]: the run, the running generation (streamed partial, retry, deferred), tool calls (output, + // details), compactions, and why the last run ended without an answer (runFailure) // value.docs["pi.inbox"], value.docs["pi.usage"], value.docs["pi.agent"], value.docs["pi.provider"] render(value); await send(ops); // for example to a remote client that applies them @@ -488,11 +494,37 @@ watch.start(async (value, ops) => { // later: await watch.stop(); ``` -Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `retired` (a document watch whose incarnation retired, after it delivered `[["r", null]]`), `ended` (the Harness closed), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. +The view's `entries` are a window, so neither the first value nor a reconnect costs the whole transcript. A watch starts with the newest 50 entries; new entries are appended, and once the window would hold more than 100, its oldest leave in one splice, leaving the newest 50. `watch(context, { min, max })` changes the bounds. `from` picks where the first value starts (as `transcript.resume` does after a drop); every entry committed after the watch attached is appended, so a `from` past the newest entry starts empty and receives what comes next. Compaction summaries and resets are entries like any other, in place; a summary's `head` says where the entries it summarizes end. `conversation.context()` gives the active transcript the model sees. -Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead. A client that joins late or reconnects starts from the current view; nothing is replayed. +`ConversationTranscript` keeps what a client has seen beyond the window: call `merge()` with every value of the view, and render the `items` it returns, the client's whole transcript with a "load earlier" marker at the top and a "load more" gap wherever the client missed entries, after a stall or a reconnect: -Partial answers and tool output are committed at most every 100 ms by default, so a crash loses at most that window. `settings.progress` changes the intervals; a host whose storage is remote can commit less often, for example `{ partialIntervalMs: 500, outputIntervalMs: 500 }`. +```typescript +import { ConversationTranscript } from "@earendil-works/pi-durable"; + +const transcript = new ConversationTranscript((query, limit, ctx) => root.entries(query, limit, undefined, ctx)); +const watch = await root.watch(context); +render(transcript.merge(watch.value)); +watch.start(async (call) => { + if (call.kind === "value") render(transcript.merge(call.value)); +}); + +// When a marker scrolls into view: +render(await transcript.loadEarlier(context)); // the page before the oldest entry +render(await transcript.fillGap(gap, context)); // the next page of a gap item + +// Reconnect after a drop: only what was missed, when it fits the window, otherwise the newest entries and a gap. +const resumed = await root.watch(context, transcript.resume); +``` + +Entries never change once committed, so a merge only adds; one that adds nothing returns the same `items` array. See `test/examples/31-transcript.ts`. + +Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one `WatchCall` per call, tagged by `kind`, and the newest commit's context: `{ kind: "value", value, ops, retired }`. Chord's watches also have `{ kind: "failed", error }` calls for failures a source recovers from; Durable's failures are final, so a Durable watch never sends one and ends instead. A watch ends once, and `closed` (or `stop()`) gives a `WatchEnd` tagged by `kind`: `stopped`, `aborted` (its acquisition context was aborted), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. + +Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(call, context)` with every later change as a `WatchCall` (always a `value` call), unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Harness closes or, for a task document, right after the retirement of the commit that ends its task, `failed` when the Harness fails. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws (or returns a rejected promise) is reported, detached, and ended `listener_error`. Watches are built on these observers. + +Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead, its window included. A client that joins late or reconnects starts from the current view; nothing is replayed. + +Partial answers and tool output are committed at most every 100 ms by default, and watches see each commit at once. Storage gets them later and together, as deferred commits (see [Your Own State](#your-own-state)): with the next commit that reads `pi.live`, such as the end of the answer or of a tool, and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). A crash loses at most about that window of an answer or of tool output; a stream that crashes in its first second may leave no aborted entry, and that attempt's usage is not recorded. `settings.progress` changes the intervals; a host whose storage is remote can commit less often, for example `{ partialIntervalMs: 500, outputIntervalMs: 500 }`. ## Busy Conversations @@ -502,16 +534,45 @@ A conversation is busy while a run is working on an input. Submitting to a busy await root.submit({ type: "input", content: "Also run the tests" }, context); // follow-up (default) await root.submit({ type: "input", content: "Use pnpm, not npm", whenBusy: "steer" }, context); await root.submit({ type: "input", content: "Only if idle", whenBusy: "reject" }, context); // throws ConversationBusy +await root.submit({ type: "input", content: "Only if busy", whenBusy: "steer", whenIdle: "reject" }, context); // throws ConversationIdle when idle await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened a file" } }, context); ``` - **Steers** are placed after the current tool round and join the running work. - **Follow-ups** are placed when the run answers, and start the next run. +- **`whenIdle: "reject"`** refuses an input when no run is working, so it never starts a run of its own. With `whenBusy: "steer"`, it steers the current run or throws `ConversationIdle`. A run that answered and waits for its [after-run tasks](#work-at-the-end-of-a-run) is still working, so such an input queues and starts the next run. +- **`submit(tx, draft)`** admits inside your own commit, with the same rules, and returns the submission's ID. Your checks, the admission, and your writes are stored together or not at all: + + ```typescript + const id = await harness.commit(async (tx) => { + const chat = await tx.doc(MyChat, root.id); // your own conversation document + if (chat.muted.includes(userId)) throw new Error("Muted"); + const id = await root.submit(tx, { type: "input", content: text, whenBusy: "steer", whenIdle: "reject" }); + chat.sent.push({ id, userId }); // your own state, same commit + return id; + }, context); + ``` + + Call it before the commit's own table writes (`appendEntry()`, `createTask()`, another admission), once per commit: otherwise it throws `ReadAfterWrite`, on every path. Documents may be read and written before and after it. A rejection (`ConversationBusy`, `ConversationIdle`) throws before any write, so you may catch it. Until your commit is stored the admission is provisional: if your callback throws later, no submission exists. Tasks and tools do the same in their own commits through `api.conversation(id)`; entries the admission places carry the task's `byTaskId`. - **Writes** append an entry without asking the model anything. A write of only `kind` and `data` changes nothing the model sees, so it is appended at once, even mid-run, and is never queued: a chat message shows in the transcript as soon as it is sent. It can therefore appear before items submitted earlier that are still queued, except behind a queued reset, where it waits so the reset does not hide it. A write with `model`, `head`, or `edits` waits for the next boundary like a steer. - `await submission.abort(context)` withdraws a queued submission. - The [settings](#settings) `steeringMode: "all"` and `followUpMode: "all"` place every queued item at once instead of one per turn. -If a run fails, queued items stay in the inbox until the next input or queued write places them, oldest first. +If a run fails, queued items stay in the inbox until the next input, queued write, or continue places them, oldest first. + +When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on), `turn_limit` (a tool round reached the conversation's `maxTurns`), or `nothing_to_answer` (a continue, below, found nothing to answer). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). + +To resume such a run without typing a message, submit a continue. It starts a run on the context as it is, so the model answers the user message or tool results the stopped run left: + +```typescript +const live = await harness.snapshot(LiveDoc, root.id, context); +if (live?.run === undefined && live?.runFailure !== undefined) { + const resumed = await root.submit({ type: "continue" }, context); // throws ConversationBusy while busy + await resumed.wait(context); // done with the answer, or unanswered like any run +} +``` + +A continue is never queued: it throws `ConversationBusy` while a run is working, since whether there is anything to continue depends on how that run ends. Items a failed run left queued are placed first and join the run. A context with nothing to answer (empty, or ending in an answer, also a text answer cut off at its output limit) ends the run without a request, `unanswered` with `nothing_to_answer`. A cut-off answer with tool calls can be continued: its calls never ran, so the context ends in their error results. Tools can continue the conversations they own through `api.conversation(id)`, as they submit to them. An input is placed as a `pi.user` entry. To show it as something else, give it an `entry` with your kind and data; the model still sees `content` as the user message: @@ -538,7 +599,9 @@ await root.reset(undefined, context); // start await root.reset("We were fixing the flaky login test. Continue.", context); // start from a handoff note ``` -While busy, the reset is queued like a write. When it is placed during a tool round, the current run ends. A tool can request the same with `control: { handoff: "..." }`. +While busy, the reset is queued like a write. When it is placed during a tool round, the current run ends: its inputs settle `unanswered` with reason `reset`, and queued inputs run in the new context. + +A tool can return `control: { handoff: "..." }` to write the same reset with its text, but the run continues: the next model request starts in the new context, and the submission settles with that eventual answer. Queued writes and steers land after the handoff text, follow-ups wait for the run's end, and a queued `reset()` still ends the run. When every call of the round also returns `terminate: true`, the run ends after the handoff instead. ## Compaction @@ -578,23 +641,7 @@ When a provider rejects a request because the context is too long, generation co When a compaction that a generation waits for at the blocking threshold fails, the generation fails too, without sending a request: its inputs settle `unanswered` with `model_error` and detail `Compaction failed: `. One that finds nothing to compact or is declined lets the request go ahead. -Running compactions are listed in `docs["pi.live"].compactions` with their reason, attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, reason, error) until a compaction produces a summary. The agent events add `compaction_start` and `compaction_end` (with `error` when the compaction failed), and `compactions` and `compactionFailure` fields in the snapshot. - -## Agent Events (Experimental) - -For consumers that want coding-agent style events (`message_start`, `message_update`, `tool_execution_start`, ...) instead of structural state: - -```typescript -import { watchEvents } from "@earendil-works/pi-durable"; - -const stream = await watchEvents(harness, root.id, context); -initialize(stream.snapshot); // entries, run, in-flight generation, tools, compactions, inbox, agent, usage -stream.start(async (events) => { - for (const event of events) console.log(JSON.stringify(event)); -}); -``` - -Events are derived from commits, one batch per commit, and apply on top of the snapshot. Message and tool updates carry deltas: text and thinking appends, appended tool-call argument text, and output trims and appends. When a consumer falls more than 100 batches behind, it receives a fresh `snapshot` event instead. See `test/examples/18-json.ts` for the full stream of one run. +Running compactions are listed in `docs["pi.live"].compactions` with their `trigger` (`manual`, `threshold`, or `overflow`), attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, trigger, error) until a compaction produces a summary. ## Hooks @@ -612,11 +659,47 @@ const Guard = defineExtension({ }); ``` -- **Generation:** `beforeRequest` (replace the messages of one request), `afterResponse`, `onYield` (continue the run with another user message), and `afterTools` (runs once a round's tools are done). -- **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). +- **Generation:** `beforeRequest` (replace the messages of one request), `beforeProviderRequest` (replace the provider request body of one request, e.g. to set a provider-specific field), `afterResponse`, `onYield` (continue the run with another user message), `afterTools` (runs once a round's tools are done), and `afterRun` (tasks to run before the run ends, below). +- **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). `afterTool` sees every result a tool task writes, also the error results the Harness writes itself (unavailable, invalid, blocked, interrupted, restart limit, aborted), told apart by their diagnostic code; past `settings.restartLimit` it no longer runs. On an abort it runs inside the abort handler, so a slow hook delays `abort()`. To limit a hook to some conversations, select its extension only there, for example with `configure({ extensions: { add: [Guard] } })`. +Hooks read committed documents (`api.snapshot()`) and the conversation's raw history (`api.entry()`, `api.entries()`), which compaction never changes. + +### Work at the end of a run + +An `afterRun` hook starts durable work after a run answered, without a tool call: for example the project's tests after a run that edited files. It returns tasks, built with `runEndTask()`, and the generation creates them as its children and waits for them before the run ends: + +```typescript +import { defineExtension, GenerationTask, hook, runEndTask, ToolResultEntry } from "@earendil-works/pi-durable"; + +const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tasks: [RunTests], // a task of yours, defineTask(): runs the tests, then commits its result + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, context) => { + // The run's own entries, also those a compaction summarized. + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, context); // page with `next` for longer runs + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && message.toolName === "edit"; + }); + return edited ? { tasks: [runEndTask(RunTests, { command: "npm test" })] } : undefined; + }, + }), + ], +}); +``` + +- **When:** after a final answer or a `terminate` round, not while an `onYield` continuation keeps the run going, never after a failure or an abort. +- **The answer first:** the answer is committed and the run's inputs settle `done` before the hooks are asked, so `Submission.wait()` resolves at the answer. +- **Busy until done:** `pi.live.run` stays, with `run.afterRun` listing the tasks, so the conversation is busy and `waitForIdle()` includes them; Stop aborts them; a crash resumes them. A crash before they exist asks the hooks again, never twice once they do. +- **Input meanwhile:** steers and follow-ups queue and start the next run once the tasks are done; `whenBusy: "reject"` and a continue throw `ConversationBusy`. +- **Results for the model:** a task gets its conversation's handle before its commit, `const handle = (await runtime.conversation(runtime.conversationId, context))!`, and admits a write or a follow-up (say "the tests failed: ...") inside it with `handle.submit(tx, draft)`, before its own table writes. Both are placed when the run ends: a write before any queued user input, a follow-up in queue order with the rest. A follow-up starts a fix run, which asks `afterRun` again with a fresh turn budget, so cap such loops in your extension. +- **Broken requests** (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) are reported through `onReport` against the hook's extension and skipped; the run still ends. + ## More Conversations and Forks ```typescript @@ -639,7 +722,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", // a rerun after a crash finds the same child and submission execute: async (args, api, context) => { const child = await api.commit(async (tx) => { @@ -706,8 +789,9 @@ decide: async (task, runtime, context) => { ```typescript const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { - for (const node of Object.values(value.tasks)) { +graph.start(async (call) => { + if (call.kind !== "value") return; + for (const node of Object.values(call.value.tasks)) { const status = node.state.status === "waiting" ? `waiting on ${node.state.on.join(", ")}` : node.state.status; console.log(`${node.id} ${node.kind} ${status}`, node.owner ?? `conversation ${node.conversationId}`); } @@ -739,7 +823,21 @@ await root.commit(async (tx) => { console.log(await harness.snapshot(Todos, root.id, context)); ``` -`harness.watchDoc()` watches one document like the view above. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). +A commit whose changes are cheap to lose, such as your own streamed progress, can be shown at once and stored later: + +```typescript +await root.commit( + async (tx) => { + (await tx.doc(Todos, root.id)).items.push("draft"); // a document like Todos: latest, fork "initial" + }, + context, + { persistence: { kind: "deferred", withinMs: 1_000 } }, // without withinMs: settings.progress.persistWithinMs +); +``` + +Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `observeCommits()` sees it then, with exactly what was stored (a document the commit retires shows as its retirement only). A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. + +`harness.watchDoc()` watches one document like the view above. A document watch follows its address: a retirement is a call with `retired: true`, the last value the watch was given (a change made in the retiring commit itself is not shown), and no ops, and a document created there later arrives as `[["r", value]]` on the same watch. A task document's watch ends `ended` with its task: once the retirement call of the commit that ends the task has returned, or at the Harness's close if that call was never delivered (a watch never started); an observer of it gets `end({ kind: "ended" })` right after the retirement. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). ## Usage and Cost @@ -791,46 +889,7 @@ await database.transaction(async (transaction) => { }); ``` -### Replication - -Every built-in storage is a `ReplayableStorage` and can rebuild its commit log from what it stores. `log(after)` reads -the changes committed after a sequence as `{ seq, writes }` records; replayed with `replay(record)` into a storage that -holds the same state at `after`, they produce the same current state. `log(undefined)` reads everything, for an empty -storage. `head()` is the last sequence. A host that syncs a session keeps only the sequence the other side has: - -```typescript -const records = source.log(await replica.head(context), context); // or the head the remote reported -for await (const record of records) await replica.replay(record, context); -``` - -The log holds what the storage kept, not every commit: each task and submission appears once, at its last write, and a -current-only document such as `pi.live` only with what followed its newest base. Fork copies appear as the creations -they became. A commit while a log is read makes it reject; read it again. Only the device that drives a session opens -a Harness on it: `Harness.open` commits, so a replica that keeps replaying records must be read through its storage. - -`@earendil-works/pi-durable/sync` moves sessions between devices through a remote store. Every device continues a -session as its own. Its uploads are batches, the log after the last sequence it shares with the remote, in gzip JSON -lines. A store keeps one slot per start sequence and creates it once, and only where the history ends (a slot inside a -batch would branch the chain), so the device whose upload creates a slot continues the session from there. A device that finds the slot taken by another device has diverged: its commits after -that point are a different history, so the host renames its copy to a new session (forked at that point) and the -session restores fresh. The remote folds the history into snapshots, keeping folded slots taken. - -The host keeps `{ syncedHead }` per copy (`SyncStateStore`; `FileSyncStateStore` keeps it in a directory), gives each -device a stable ID, and implements the store (`SessionSyncStore`); `FileSystemSyncRemote` is one in a directory, for -development and tests. The store also lists its sessions and keeps a metadata object per session for the host (title, -location, origin device), outside the history: the last write wins. - -```typescript -const result = await uploadSession({ sessionId, device, source: storage, store, state, context }); // after commits -if (result.type === "diverged") forkLocalCopy(sessionId, result.at); // host: rename, upload under the new ID -await pullSession({ sessionId, store, storage, state, context }); // catch up, resume, or restore into an empty storage -``` - -Pulling refuses a copy with commits it has not uploaded (`UnsyncedCommitsError`): upload first. Uploading refuses a copy that lost commits it had uploaded, as to a power loss (`behind`): its next commits would reuse their sequences, so it pulls them back first. A pull needs the -Harness closed; a device that falls behind while its Harness is open closes it, pulls, and reopens, or commits first and -diverges on its next upload. `FlushScheduler` and `isUrgentCommit()` decide when to upload. - -A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption of the batch, so memory never shows a partial log. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: +A custom backend's `commit(batches, receive, context)` stores `batches` in order, each as one atomic commit with its own seq, checked against the state the earlier ones left, in as few round trips as it can. It stops at the first batch its own checks refuse (global IDs, document actions, copy resolution): the batches before it stay stored, it and the later ones are not. Once they are durable it calls `receive(index, seq)` per stored batch, in order and synchronously, then resolves with `{ stored, refused? }`; it rejects for an unknown outcome (connection lost, a failed statement or `COMMIT`). `receive` is a durability receipt: the Session publishes each batch later, one at a time, so memory and every view never show a partial log. `receive` calls no Storage method and never throws. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: ```typescript import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; @@ -846,13 +905,13 @@ registerStorageConformance({ describe, expect, it }, "My Storage", async (use) = }); ``` -The package root loads TypeBox, because the tool task validates arguments with pi-ai's `validateToolArguments()`. That costs about 23 MB of peak RSS unbundled, about 4 MB in a tree-shaken bundle. +The package root loads pi-schema, because the tool task validates arguments with pi-ai's `validateToolArguments()`. ## Errors Three rules decide what an error does: -1. **A storage error ends the Harness.** The call that hit it rejects with the error. Everything after it, and every pending wait, rejects with `SessionFailed` (its `cause` is that error); watches and event streams end with `failed`; running tasks get their abort signal; `onReport` gets the error once. The Harness then closes itself. Nothing is retried; a custom storage retries its own transient failures. Reopen to recover: commits are atomic, so the store is consistent, and resubmitting with the same `requestId` finds a submission that was already admitted. +1. **A storage error ends the Harness.** The call that hit it rejects with the error. Everything after it, and every pending wait, rejects with `SessionFailed` (its `cause` is that error); watches end with `failed`; running tasks get their abort signal; `onReport` gets the error once. The Harness then closes itself. Nothing is retried; a custom storage retries its own transient failures. Reopen to recover: commits are atomic, so the store is consistent, and resubmitting with the same `requestId` finds a submission that was already admitted. 2. **An invalid read fails only that call:** a storage read throws `StorageRequestError` for a bad cursor, an unknown conversation, or history a document does not keep. A read you cancelled fails only itself too. The exemption covers reads only: any error from `commit()` or `mintId()`, `StorageRequestError` included, ends the Harness as in rule 1. 3. **Your code fails only its own unit:** a throwing task phase faults that task, a throwing tool gets an error result, a throwing hook is reported and skipped (`beforeTool` blocks the call), and a throwing commit callback rolls back that commit. Listeners that throw, or return a rejected promise, are reported and the others still run. The exception is a host `RegistryReader` whose `snapshot()` throws: the Harness cannot schedule without it, so it fails the Harness like a storage error. @@ -860,7 +919,7 @@ Three rules decide what an error does: ```typescript harness.closed.then((end) => { - if (end.reason === "failed") reopen(end.error); + if (end.kind === "failed") reopen(end.error); }); ``` @@ -880,10 +939,10 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [15-real-model](test/examples/15-real-model.ts) | Streaming an answer from OpenAI | | [16-coding-tools](test/examples/16-coding-tools.ts) | A tool-using turn on JSONL storage | | [17-print](test/examples/17-print.ts) | Print mode: submit a prompt, print the answer | -| [18-json](test/examples/18-json.ts) | JSON mode: agent events or raw view operations, on SQLite, JSONL, or memory | +| [18-json](test/examples/18-json.ts) | JSON mode: raw view operations, or appended entries and `pi.live` per change, on SQLite, JSONL, or memory | | [19-inbox](test/examples/19-inbox.ts) | Steers, follow-ups, writes, and withdrawal while busy | -| [20-late-join](test/examples/20-late-join.ts) | Attaching a view and an event stream mid-run | -| [21-subagent-foreground](test/examples/21-subagent-foreground.ts) | A replay-safe subagent tool whose child the call owns, with the child's events under the call | +| [20-late-join](test/examples/20-late-join.ts) | Attaching a view mid-run: the running tool's output, then the streamed answer | +| [21-subagent-foreground](test/examples/21-subagent-foreground.ts) | A replay-safe subagent tool whose child the call owns, with the child's transcript under the call | | [22-subagent-background](test/examples/22-subagent-background.ts) | Persistent subagents: spawn, steer, stop, list, answers reported back, restart-safe | | [23-child-tasks](test/examples/23-child-tasks.ts) | A checkout that owns and waits for four payments: failFast, abort, restart | | [24-compaction](test/examples/24-compaction.ts) | A long chat compacted in the background, manually, and after a context overflow | @@ -893,6 +952,12 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [28-sandbox-per-conversation](test/examples/28-sandbox-per-conversation.ts) | An environment per conversation, looked up from an app document | | [29-tool-override](test/examples/29-tool-override.ts) | A same-name bash for some conversations, and a wrapper that times whichever bash won | | [30-reload-and-restart](test/examples/30-reload-and-restart.ts) | Reloading an extension mid-call, and stored choices surviving a restart | +| [31-transcript](test/examples/31-transcript.ts) | A client's transcript over the windowed view: live appends, a stall with a gap, loading it and earlier pages, and resuming with `from` | +| [32-tests-after-edits](test/examples/32-tests-after-edits.ts) | An `afterRun` hook that runs the tests after every run that edited files, a fix run when they fail, and input waiting for them | +| [33-turn-limit-and-continue](test/examples/33-turn-limit-and-continue.ts) | A per-conversation turn limit, `runFailure`, and continuing without a user message | +| [34-submit-in-commit](test/examples/34-submit-in-commit.ts) | Admitting a visitor's steer inside the host's own commit with `submit(tx, draft)` and `whenIdle: "reject"` | +| [35-durable-on-the-store](test/examples/35-durable-on-the-store.ts) | A Harness on `StoreStorage` in a Chord host's store, next to a Chord object's own table: one commit queue, one id allocator, one export | +| [36-durable-views](test/examples/36-durable-views.ts) | A Chord object following a Durable view with `self.follow`: its frames and a member reading the followed value | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/docs/group-commit.md b/packages/durable/docs/group-commit.md new file mode 100644 index 000000000..4af775d71 --- /dev/null +++ b/packages/durable/docs/group-commit.md @@ -0,0 +1,296 @@ +# Group commit for the Durable Session + +Status: design, revision 2 (after three reviews). Branch `durable-perf` (worktree `~/pi-perf` on slayer), on top of the +uncommitted prerequisites (record cache, conversation tails, kept context ranges advanced at publish, task-document +index, deferred tool-slot `running` and generation attempt). Read `docs/spec.md` §1, §4, §4.1, §5, §8, §10.1 alongside. + +## 1. Problem + +A Session has one mutation line (`SessionImpl.#tail`). Each commit runs its callback, settles, calls `Storage.commit()`, +and adopts while holding the line, so every storage round trip of every conversation is serialized. After the +prerequisites the line makes no storage reads in a normal run; a run that calls one tool stores 12 commits. Measured on +slayer (one Harness, faux model, MemoryStorage with a simulated round trip per call, 32 concurrent conversations): +26.8 runs/s at 4 ms, 84 at 1 ms, 895 at 0 ms (CPU, one core). Throughput is fixed by the line. + +Group commit sends the commits that accumulate while one write is in flight as one storage call. Modelled with result +fences (each job on a run's critical path waits about one round trip): ~250 runs/s at 4 ms with 32 conversations, +rising toward the CPU ceiling with more conversations. A conflict probe (`/tmp/conflict`) measured, per run, 0 writing +jobs that would wait for one tool call or a streamed answer, 0.13 for three parallel tools. + +## 2. Semantics + +### Unchanged + +- Every logical commit keeps its own `seq`; history (`snapshotAsOf`, fork `asOf` copies, `createdAt`/`retiredAt`) + keeps per-commit boundaries. +- A commit's promise resolves only once it is durable and published; outcomes are delivered in line order (§5.1). +- Publication is in line order, deferred commits included (§5.2): telemetry events, Session listeners, the + `observeCommits()` dispatch, waits and aborts per commit; host observers of commit A run, and the continuations of + public waits A resolves start, before commit B is published (§5.3). +- No callback observes unstored state (§4). Memory adopts a commit at its publication. +- A commit adopted by Storage and then rejected stays published and rejects, as today (§3.4). +- Storage failure is fatal; callers are never told something storage does not hold. + +### Changed (accepted) + +1. A callback can run while an earlier commit's storage call is in flight. It runs until its first touch of a key an + unpublished commit wrote, waits until that commit's storage call settled and it is published, then continues. Code + before that touch runs early, assignments to caller variables included; acting on such values before the commit + resolves was already unsafe (a callback's own writes can fail to store today). +2. Cancellation is checked when a job starts; a job can start earlier. +3. If a batch fails fatally, later callbacks may already have run; their outcomes are `SessionFailed` (§5.1). +4. On a path that fails the Session, which commits survive differs: a stored prefix survives. Every caller's outcome + matches storage. On a bug path (adoption or an internal listener of part i throws) parts after i can be durable but + unpublished; their commits reject with `SessionFailed`; reopening recovers them. Likewise, once another error has + failed the Session, a call that settles afterwards, and the parts of a call not yet published when it fails, are + durable but unpublished and reject with `SessionFailed` (the first error); an `observeCommits()` consumer, such as a + replica, misses them until reopen. +5. Read results (`tx.task()`, `tx.conversation()`, `getTask()`, `runtime.entry()`, entries in published commits) are + frozen shared objects; mutating one throws (documented on `Tx`). +6. Prerequisite presentation changes (spec §8.2, §8.4 updated): a tool slot's `running` and a nested slot's final + arguments, and the generation's attempt, are stored like progress; after a crash a slot may show `pending` while + the task's checkpoint is `execute` (a safe rerun sets it again). +7. A wait whose commit is in flight when close begins ends with the closed error; the commit is still stored and + published by the drain (today this already happens when close lands before adopt). +8. Spec text to update: invariant 7, §4 (host listeners run while the line is held), §4.1 (holds leave at the last step + before `Storage.commit()`: now at logical admission), §10.1 (per-part outcomes), the `observeCommits` doc, the + `SessionImpl` class doc, the scheduler class invariant (decisions stay serialized; their storage overlaps). + +## 3. Storage + +### 3.1 `commit` + +```ts +/** + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Stops at the first batch Durable's own checks refuse: + * the batches before it are durable, it and the ones after are not. Calls `receive(index, seq)` once per durable batch, + * in order, synchronously, after all of them are durable and before the next commit starts. Resolves with the number of + * durable batches and the refusal, if any. Rejects for an unknown outcome (connection lost, statement or commit + * failed); `receive` calls made before the rejection still stand. + */ +commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, +): Promise<{ readonly stored: number; readonly refused?: unknown }>; +``` + +`receive` is a durability receipt, not publication: the Session publishes later (§5.3). It is Storage's only write +method (it replaced the earlier one-batch `commit(writes, adopt, context)`, and took its name): every backend implements +it natively. Tests and suites that write one batch directly use `commitWrites()` (`@earendil-works/pi-durable/testing`), +which throws the refusal. + +Example: `[A, B, C, D]`, C refused. A and B are durable and published; C rejects with the refusal; D rejects with +`SessionFailed`; the Session fails. This is today's outcome (A and B stored, C fails, D never reaches storage). + +### 3.2 Backends + +A refusal is one of Durable's own checks (`checkGlobalIds`, `checkDocumentActions`, copy resolution), run against the +transaction's state before that batch writes anything. Any other error is an unknown outcome. No savepoints (Durable +Objects' SQL API does not allow them). + +- SQLite: one transaction; per batch: read `next_seq`, run the checks, apply, advance; on a refusal stop and commit the + prefix. `prepareDocumentActions` and `candidateNextId` per batch. +- MemoryStorage: per batch `prepareCommit()` (checks and detaches, changing nothing) then apply; stop at the first + refusal (prefix semantics allow it). +- JSONL: check batch by batch against the in-memory state, append the prefix's commit markers in one write; a torn append + recovers a shorter complete prefix, which prefix semantics allow. +- IndexedDB: one `readwrite` transaction per call (metadata read once, written once at the end). Per batch every read + and check runs first (`checkGlobalIds`, `checkDocumentActions`, then `resolveCopies`, which reads each copy's source + against the state the earlier batches left), and only then its writes. A refusal stops issuing writes; the transaction + commits the prefix. An error of IndexedDB itself (a `DOMException`, or a failed request or transaction without an + error, also as a check's cause) aborts the transaction: the call rejects with nothing stored. +- StoreStorage (p3's primary storage): one series of the Chord store's commit queue per call + (`ChordStore.commitSeries`): one backend transaction for all batches, one seq and one outbox commit per batch, the + store's metadata (`last_seq`, `next_id`) written once per call. A batch's `DurableCommit.check(step)` (ids, + documents, copies through the step's reader, so it sees the earlier batches; the same reads a single commit makes) + runs before its writes; a throw refuses it and the series commits the prefix, unless one of the check's statements + failed (the store watches the check's executor), which rolls the series back and rejects, as does a throw of the + writes; a failed `COMMIT` is an unknown outcome that stops the store. Receipts run after `COMMIT`, before the next + step of the queue; the Session then publishes each commit in its own host batch (`publishing`). Memory per call is + the call's staged commits and one receipt entry per batch; the catalog is copied only if a batch changes a table + record. + +A Storage may also implement `publishing?(publish: () => void, context: Context): void`: the Session calls it for each +part's publication and the Storage runs `publish` inside its own scope. StoreStorage implements it with its attached +host's `batch.within(...)`, so Chord sources over Durable's views stay marked in one batch per commit. + +### 3.3 Wrappers + +- `CachedStorage` applies write-through at publication (§5.3), not at the receipt. Its read rule becomes per key: a read + records `version(key)` at its start and keeps its result only if no commit published since wrote that key. Versions + exist only for keys with a read in flight (registered at read start, removed at read end), so the map is bounded by + reads in flight. +- `GuardedStorage.commit` guards the call but does not fail the Session itself on rejection; the writer does + (§3.4), after publishing the received parts. + +### 3.4 Outcomes per logical commit + +The writer keeps a ledger per call: which parts were received. + +- Received: published (§5.3), then resolved through the fences. +- `commit` resolved with a refusal at part k: parts before k as received; part k rejects with the refusal; parts + after k and every queued job: `SessionFailed`; the Session fails after the received parts are published. +- `commit` rejected: received parts are published first, then reject with the error (adopted-then-rejected, as + today); unreceived parts reject with the error; the Session fails. +- Adoption or an internal listener of part i throws: part i as today (the Session fails after its storage call); + parts after i are not published and reject with `SessionFailed`. + +## 4. The line + +### 4.1 Mechanics + +- The line runs one job at a time; callbacks never interleave with each other's synchronous code. +- A job with writes, after its callback and settlement succeed, takes the held changes it will store (`takeHeld`) and + joins the pending batch (logical admission). Its outcome is parked (§5.1). +- The writer runs off the line: it sends the pending batch with `commit` when no call is in flight and the previous + call's parts are published. A job waiting on a gate never blocks the writer. +- Backpressure: at most `MAX_OUTSTANDING` (initially 256) admitted jobs whose outcome is not delivered, writing or not; + beyond it the line waits. +- Unpublished write set: per key, the newest unpublished writer. Registered at logical admission (immediate commits) or + at callback end (deferred commits, which stay in it until their publication); removed at publication only if the key + still maps to that writer. +- Touch: every read or write a job makes checks its key. On a hit the job awaits the gate of the newest writer of that + key, then proceeds. A gate opens when that commit is published and its storage call settled (holds released). + +### 4.2 Keys + +| Key | Read by | Written by | +|---|---|---| +| `task:` | `tx.task`, committed-task reads, every task record a scheduler region reads (decided task, wait members and their `on`, cycle walks) | task create or write | +| `owned:` | every node a walk over live children expands (`#hasOwnedLive`, `#ownedLive`, `#walkDown`, `#waitCycle`, `ownedTasks`, `#idle` in a line job) | task creation and a task becoming terminal: every node from the task's parent up to and including the nearest live task or the root conversation (the nodes `#relist` lists or unlists), chain loaded at admission if needed; conversation creation under a task: that task's chain | +| `roots` | global idle in a line job (`#idle(undefined)`) | every task creation whose chain reaches an ownerless conversation (directly or through terminal tasks and conversations), and every task becoming terminal there (conservative: whether the root is listed can change under pending commits) | +| `intent` | every cancellation check (`#belowCancelled`, `#cancellingOwner`, including the empty-intent fast path, `#inScope`, `#nearestLiveAbove`) | abort marks (set, upgraded, cleared), a non-completed held or terminal outcome, restart marks | +| `conv:` | `tx.conversation`, `#requireConversation`, edge loads | conversation creation | +| `entry:` | `tx.entry(id)`, `storage.entry(conv, id)` | entry append | +| `entry-append:` | `latestEntryId`, `latestHeadMarker`, entry scans, context bounds | entry append (blind) | +| `sub:` | `tx.submission`, settlement/placement resolution | submission create or write | +| `req::` | `submissionByRequest` | submission creation with a request ID | +| `doc:
` | `tx.doc`, `retireDoc`, cold loads, `conversationDocumentOnLine`, fork copies of the source | document create/change/retire/copy at the address; held changes the commit takes | +| `docscope:` | `scanDocuments`, task-document index reads | document creation/retirement in the scope (blind) | +| `*` | scans that cannot be keyed (`scanTasks`, `scanSubmissions` by status, `scanConversations`); `unloadDocuments`, close, version reload | conflicts with every unpublished write | + +Blind writes conflict with reads of the key, not with each other (appends with distinct minted IDs, creations in one +scope). Created records' keys are ordinary keys: a minted ID can leak through a caller variable before publication, and a +read of it waits. + +### 4.3 Liveness-only reads (not keyed) + +A stale read that only delays work needs no key: a reservation pass not seeing a task a pending commit made runnable; +a waiter's check whether its dependencies are still live; the projection of immutable ownership fields (`owner`, +`conversationId`, `background`) during chain loading (not the rest of `#loadChain`, which installs settled nodes and +null edges after the check); a reservation pass skipping a task with an invocation without reading its record. +Cancellation decisions and terminalization are never liveness-only. + +### 4.4 Line reads outside `tx` + +`readOnLine(job)` becomes `readOnLine((storage) => job)`: the job gets a gated Storage view whose reads record keys and +wait on gates. All current `readOnLine` call sites (harness.ts, scheduler.ts runtime reads, submissions.ts, task-graph.ts, +view.ts, `captureContextBounds`) use it; the Session's line helpers (`latestEntryId`, `latestHeadMarker`, +`conversationDocumentOnLine`, cold loads) are gated too. Off-line reads keep using the ungated Storage. + +## 5. Outcomes and publication + +### 5.1 Outcome fences + +Every job's outcome (commits with and without writes, `readOnLine` jobs, deferred commits, callback errors) is +delivered in line order. A Session-fatal failure makes every later outcome `SessionFailed`; a nonfatal rejection (a +callback's own error, cancellation, validation, `ReadOnlySession`) affects only its job. Fences hold outcomes, not the +line. Exemption: a scheduler-internal job without writes whose outcome goes only to scheduler code and whose touch set +is clean (no hit) delivers at once; it is equivalent to having run before every unpublished commit. A job with writes +(a reservation, a step that faults, a reconcile pass) is never exempt: its outcome waits for its own publication, so no +invocation starts on a reservation that is not durable. + +### 5.2 Deferred commits + +A deferred commit's callback runs and settles like any job; its memory adoption and publication are fenced like an +immediate commit's publication: they happen when every earlier unpublished commit is published. Its keys stay in the +unpublished write set until then. Holds are taken at logical admission, so a commit never stores a hold adopted after +it in line order. + +### 5.3 Publishing a call's parts + +After `commit` settles (or the received prefix is known, §3.4): publish part 0 (cache write-through, conversation +tails, adoption, Session listeners, host dispatch, waits), then a macrotask hop (`setImmediate` where available, else a +`MessageChannel` message), then part 1, and so on. The hop lets public wait continuations (`Submission.wait()`, +`waitForTask()`, `commit()`) of part i start before part i+1 is published. Tests use the public APIs. + +### 5.4 Close, unload, holds, eviction + +- Close: seal admission; run close listeners; signal every invocation (scheduler); let admitted callbacks finish, + gates included; drain pending and in-flight calls and their publication; join invocations and run `writeAtClose()` + through the writer; store remaining holds through the writer; end observers; close Storage. +- `unloadDocuments()`, version reloads (also nested inside a callback), hold timer stores: `*` barriers through the + writer; holds captured after the barrier. +- Documents with unpublished changes stay pinned. + +## 6. Scheduler + +Scheduler state changes at publication, as today. Inside a callback, decisions read it synchronously. Each decision is +restructured as a pure plan: + +1. A decision region runs with a synchronous "recording" flag set; helpers record keys (§4.2) only while it is set + (the same helpers run at publication inside `#observe`, unrecorded). +2. The region reads, records, and computes a plan or an error, staging nothing and mutating no scheduler state. +3. If a recorded key has an unpublished writer: discard the plan (or error), await the gate, reload scopes + (`#loadScopes`), rebuild the overlay (`overlayOf`, `#links`), and run the region again. A waiting job holds the line, + so unpublished writes only shrink; re-runs converge. +4. After a clean check: apply the plan: stage writes, mutate scheduler sets, create or end invocations, run `migrate`, + take registry snapshots, report. None of these runs inside a region, so none repeats. + +Sites to restructure (hardest first): `#reserve` (runnable iteration, abandoned marks, cut-off faults, `#resolve` with +`migrate` and reports, invocation creation); `#step`/`#decide` (registry refresh, `task_outdated` reports, +cancellation, `#end`); `#finalize` (rounds, overlay mutation, awaited `#settleOutcome` between rounds: one region per +round); `#validateWait`/`#waitCycle` (async member loads, cycle errors gated like plans); `abort`; `#commitState` +terminal (chain loads per overlay task, walk); `#reconcile` (intent walks, marks; it captures and clears work lists +before its commit: move the take inside, after the check). `abortConversation` and cascades scan with `*`. + +Sweep: `#sweep` and `#scheduleSweep` do not drop nodes loaded by a job whose commit is unpublished; chain loads record +the nodes they install for their job, released at that commit's publication or failure. + +Later, the ownership index can move into the Session behind `tx` reads with the same keys. + +## 7. Bounds + +- Outstanding jobs: at most `MAX_OUTSTANDING`, a count, not bytes: pending writes are bounded by what one commit can + hold (tool output is bounded by output limits), times the count. +- Unpublished write set: one entry per key per newest writer, removed at publication. +- Cache versions: only keys with a read in flight. +- Fences, gates, ledgers: settled at publication or Session failure; nothing outlives close. +- Existing: record cache (conversations 4096, submissions 256; tasks: every live one, the ended ones a waiting or just-resumed task names, plus 256 ended), conversation tails (4096 IDs, 256 markers), + task-document index (live tasks and their documents), kept context ranges (busy or within retention; an idle range + past its expiry is dropped when an entry would extend it; expiry fixed at idle start, the timer uses the current + setting: document and test), document cache (retention). + +## 8. Tests + +- `commit` per backend (conformance for every backend): prefix stored, refusal index, seq per batch, copies + against earlier batches' state, JSONL torn append; IndexedDB: a copy refused mid-call keeps the prefix, an IndexedDB + error in a check or a write rejects with nothing stored; StoreStorage: one backend transaction per call; Chord's + `commitSeries`: refusal, a failed check statement and a throwing write roll back, a throwing receipt stops the store. +- Gating per key class, with deterministic interleavings: records; ancestor chain membership (live P → terminal B → + terminal A → empty C, creation in C vs P terminal); global idle with empty roots; pending abort mark vs child step + (empty-intent fast path); concurrent opposite waits; leaked entry ID read by `tx.entry`; request IDs; absent + addresses; held documents (hold created after a reader prepared); `*` barriers; `readOnLine` read after an + un-awaited commit. +- Outcome fences: no-write, read, deferred, callback errors in line order; nonfatal rejections do not poison later jobs; + fatal failure poisons later outcomes. +- Deferred publication behind a pending immediate commit, and when that commit fails. +- Failures per part: refusal, rejection after receipts (received parts published, then rejected), adoption failure. +- Publication order through public waits: A's `Submission.wait()`/`waitForTask()` continuation sees B unpublished. +- Close during in-flight calls with an invocation waiting on its signal; unload, version reload, hold timer during + flight; backpressure on all outstanding jobs. +- Scheduler retries: `migrate`, registry snapshots, reports and cleanup run once across re-runs; sweep vs chain loads. +- Randomized interleavings: random commits over overlapping keys, random latency and refusals; storage equals the stored + commits replayed in seq order, no callback observed unpublished state (instrumented oracle), outcomes match storage. +- Prerequisites: tool crash at both sides of the intent commit (sibling flushing presentation first); kept range + installed after a publication between return and install; live retention change; cache eviction and close; tails + eviction, fork markers. +- Bounds: outstanding, write sets, versions, fences return to zero when idle. +- Existing suites: durable (both retention modes), workers. + +## 9. Benchmark + +Conc probe at 0/1/4/20 ms with 1/8/32/64/128 conversations, one tool, three parallel tools, streaming; dpetrou's repro +as is and with warm documents. Report throughput, p50/p95 run latency, batch sizes, gates per run, fence delay. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index f3614a7ce..8f6b04982 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -9,6 +9,7 @@ import type { Draft, JsonRepresentation, JsonValue, + WatchCall, WatchEnd, WatchHandle, } from "@earendil-works/chord"; @@ -41,7 +42,8 @@ type once the PR lands. The core rule is: > A Session atomically commits immutable entries, full task records, and -> Chord-tracked documents. Only committed state is observable. +> Chord-tracked documents. Only committed state is observable. A commit marked +> deferred may be observed before it is stored (section 4.1). ## 1. Terms and invariants @@ -63,18 +65,36 @@ The core rule is: Required invariants: 1. One Session commit is atomic across all record and document writes. -2. A document update is published only after its storage commit succeeds. -3. All visible progress is durable. There is no volatile publication path. +2. A stored commit is published once Storage made it durable, after the + storage call that stored it settled, in line order (section 4), also when + that call then rejects; a commit that fails before it is durable publishes + nothing. Once the Session has failed, nothing more is published: a commit + whose call settles after another error failed the Session, or whose turn to + be published comes after that, is durable but unpublished, and reopening + recovers it (section 10.1). A deferred commit (section 4.1) is published to watches and the + conversation view when memory adopts it, once the commits before it are + published, and to `observeCommits()` once it is stored. +3. All visible progress is durable, except what a deferred commit holds: what + watches see can be ahead of Storage by up to that commit's window + (`withinMs`), and a crash loses it. A commit is deferred only when its caller + marks it so, and only when it changes nothing but existing latest, + initial-fork conversation documents (section 4.1). Every stored document is + a state memory had. 4. External effects do not run inside the Session mutation transaction. 5. Entries and IDs are immutable and never reused after a committed write. 6. Document drafts are fully revoked when their transaction callback settles: the Session synchronously prepares or aborts every open change at that point. Values assigned into a draft are copied by value and must be strict JSON. -7. The mutation line remains held through storage settlement and committed-state - adoption. Commit/close observers run synchronously only to capture immutable - state; document-state and watch user callbacks run later, off the line. -8. A storage error is fatal to the open Session (section 10.1). It publishes - nothing and must be reopened. Preparation and checkpoint failures occur before +7. The mutation line runs one commit callback at a time; their storage + overlaps (group commit, section 4). A callback never observes what an + unpublished commit wrote: at its first read or write of such a key it waits + until that commit is stored and published. Commits are published in line + order, and every outcome is delivered in line order. Commit/close observers + run synchronously only to capture immutable state; document-state and watch + user callbacks run later, as microtasks the line never waits for. +8. A storage error is fatal to the open Session (section 10.1), which must be + reopened. A commit it fails publishes nothing, unless Storage had already + made it durable (a durability receipt). Preparation and checkpoint failures occur before storage admission and roll back normally. ## 2. Core records @@ -226,6 +246,32 @@ type SubmissionRecord = readonly reason: string; readonly detail?: JsonValue; } + )) + /** A run without a user message (section 6): no entry, never queued. */ + | (SubmissionRecordBase & { + readonly type: "continue"; + } & ( + | { + readonly status: "placed"; + readonly entry?: never; + readonly answer?: never; + readonly reason?: never; + readonly detail?: never; + } + | { + readonly status: "done"; + readonly entry?: never; + readonly answer: EntryId; + readonly reason?: never; + readonly detail?: never; + } + | { + readonly status: "unanswered"; + readonly entry?: never; + readonly answer?: never; + readonly reason: string; + readonly detail?: JsonValue; + } )); type SubmissionSettlement = @@ -301,8 +347,10 @@ Context derivation: kind ahead of the baseline therefore keeps the baseline from leading. Stored entries and `contributions` keep the committed order. -Views carry raw active entries. UI reduction and model-context reduction are -separate consumers. Older stored history is available through the owning +The conversation view (section 9.3) carries a window over the visible +entries in ID order, head markers in place; `Conversation.context()` derives +the active transcript. UI reduction and model-context reduction are separate +consumers. Entries outside the window are available through the owning `Conversation` object's cursor-based `entries()` scan. ### 2.2 Public Harness surface @@ -324,6 +372,8 @@ type SubmissionDraft = { readonly type: "input"; readonly content: UserInput; readonly whenBusy?: "steer" | "followUp" | "reject"; + /** `reject`: an idle conversation rejects the input with `ConversationIdle` (section 6). */ + readonly whenIdle?: "reject"; readonly entry?: never; } | { @@ -331,6 +381,15 @@ type SubmissionDraft = { readonly entry: EntryDraft; readonly content?: never; readonly whenBusy?: never; + readonly whenIdle?: never; + } + /** Start a run on the context as it is, without a user message; idle only (section 6). */ + | { + readonly type: "continue"; + readonly content?: never; + readonly entry?: never; + readonly whenBusy?: never; + readonly whenIdle?: never; } ); @@ -388,14 +447,15 @@ type HarnessOptions = { ) => (() => void) | undefined; /** * Replaces the timer for sleeping tasks (section 5.6): called with `Harness.wake()` after open and whenever it - * changes, off the Session line; the host arms its own wakeup and calls `resume()` when it fires. + * changes, in a microtask after the commit that changed it was adopted; the host arms its own wakeup and calls + * `resume()` when it fires. */ readonly onWake?: (wake: Wake) => void; /** - * Failures that do not fail the calling operation, and the Session's failure, once; `info` says where each came - * from (section 10.1). + * Failures that do not fail the calling operation, and the Session's failure, once; `report.kind` says where each + * came from (section 10.1). */ - readonly onReport?: (error: unknown, info: ReportInfo) => void; + readonly onReport?: (report: Report) => void; }; /** Curated pi-ai request options; absent fields use pi-ai defaults. */ @@ -460,7 +520,7 @@ type HarnessSettings = { readonly documentRetentionMs?: number; /** What a failed run does with the inputs queued behind it (section 6). */ readonly afterFailedRun?: "keep" | "continue" | "withdraw"; - /** Restarts in a row without progress after which a task is parked (section 5.4); `Infinity` never parks. */ + /** Restarts in a row without progress after which a task ends instead of running again (section 5.4). */ readonly restartLimit?: number; }; @@ -497,6 +557,8 @@ type AgentState = { instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ cwd?: string; + /** Tool rounds one run takes at most (section 8.5); an integer, `1` or more; unset, no limit. */ + maxTurns?: number; }; /** Built-in rewindable agent document; see below. */ @@ -517,6 +579,8 @@ type AgentChange = { | null; readonly instructions?: string | null; readonly cwd?: string | null; + /** `configure()` throws a `RangeError` unless it is an integer `1` or more, or `null`. */ + readonly maxTurns?: number | null; }; /** One change in any commit. A given field replaces the stored one; `null` clears it; `undefined` changes nothing. */ @@ -537,6 +601,7 @@ type Agent = { readonly sections: readonly PromptSection[]; readonly instructions?: string; readonly cwd?: string; + readonly maxTurns?: number; }; /** Entry whose `data` has type `D`; `never` means the kind carries no data. */ @@ -568,6 +633,10 @@ type SettledSubmissionRecord = SubmissionRecord & { interface Submission { readonly id: SubmissionId; status(context: Context): Promise; + /** + * Resolves once the input is answered or unanswered: at the answer, also while the run still waits for its after-run + * tasks (section 8.3), so the conversation may still be busy. + */ wait(context: Context): Promise; abort(context: Context): Promise<"aborted" | "already_placed" | "settled">; } @@ -580,7 +649,7 @@ type TaskInspection = { readonly record: TaskRecord; readonly state: | { readonly kind: "running" } - | { readonly kind: "ready"; readonly migrates: boolean } + | { readonly kind: "pending"; readonly migrates: boolean } | { readonly kind: "waiting"; readonly on: readonly TaskId[] } /** A sleep whose `until` the clock has not reached (section 5.6). */ | { readonly kind: "sleeping"; readonly until: number } @@ -588,7 +657,7 @@ type TaskInspection = { | { readonly kind: "completing" } | { readonly kind: "blocked"; - readonly reason: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; readonly error?: unknown; }; }; @@ -598,32 +667,40 @@ type Wake = { readonly active: boolean; readonly at?: number }; /** What a report is about (section 10.1). */ type ReportTask = { readonly id: TaskId; readonly kind: string; readonly conversationId: ConversationId }; -type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; type ReportInfo = - | { readonly type: "session_failed" } + | { readonly kind: "storage_failed" } | { - readonly type: "task_faulted"; + readonly kind: "task_faulted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "threw" | "restart_limit"; } - | { readonly type: "task_parked"; readonly task: ReportTask; readonly extension?: string } - | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "task_outdated"; + readonly kind: "task_interrupted"; readonly task: ReportTask; readonly extension?: string; - readonly reason: "missing_task" | "incompatible_task"; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; } - | { readonly type: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "extension_error"; + readonly kind: "task_outdated"; + readonly task: ReportTask; + readonly extension?: string; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; + } + | { readonly kind: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { + readonly kind: "extension_error"; readonly extension?: string; readonly task?: ReportTask; readonly conversationId?: ConversationId; } - | { readonly type: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } - | { readonly type: "listener_error" }; + | { readonly kind: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } + | { readonly kind: "listener_error" }; +/** One report: what it is about, and the error. */ +type Report = ReportInfo & { readonly error: unknown }; type HarnessInspection = { readonly scheduling: "paused" | "running" | "closing"; @@ -641,6 +718,8 @@ type HooksOf = K extends Task interface Conversation { readonly id: ConversationId; submit(submission: SubmissionDraft, context: Context): Promise; + /** Admit inside the caller's commit, with the same rules; stored with that commit (section 6). */ + submit(tx: Tx, submission: SubmissionDraft): Promise; /** Resolved with the current registry snapshot and settings. */ agent(context: Context): Promise; /** `configure()` in its own commit. */ @@ -649,6 +728,8 @@ interface Conversation { commit( change: (tx: Tx) => T | Promise, context: Context, + /** As for `Session.commit()` (section 4.1). */ + options?: CommitOptions, ): Promise; /** With `at`, the context as of that visible entry, as `fork(at)` would start with. */ context(context: Context, options?: { readonly at?: EntryId }): Promise; @@ -668,8 +749,8 @@ interface Conversation { /** `{ background: true }` also aborts background work under the conversation (section 5.4). */ abort(context: Context, options?: { readonly background?: boolean }): Promise; waitForIdle(context: Context): Promise; - /** The structural view (section 9.3) as a watch. */ - watch(context: Context): Promise; + /** The structural view (section 9.3) as a watch; `window` bounds its entries. */ + watch(context: Context, window?: ConversationWindow): Promise; } interface Harness extends Session { @@ -693,14 +774,15 @@ interface Harness extends Session { conversationId?: ConversationId, ): Promise<"aborted" | "already_placed" | "settled" | "not_found">; abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">; - /** Run a task parked by the restart limit again (section 5.4). */ - retryTask(id: TaskId, context: Context): Promise; waitForTask(id: TaskId, context: Context): Promise>; waitForIdle(context: Context): Promise; /** Session total of every conversation's `pi.usage` (section 8.6). */ usage(context: Context): Promise; - /** Live tasks as a structural view (section 9.5), as a watch. */ + /** Live tasks as a structural view (section 9.4), as a watch. */ watchTaskGraph(context: Context): Promise; + /** Synchronous observers of a conversation view and of the task graph, for adapters (section 9.1). */ + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; } declare const Harness: { @@ -904,7 +986,7 @@ replace its model configuration without reopening the Harness. Settings are sync modes are read on the Session line. `stream` is forwarded to generation requests; `stream.maxRetries` are provider retries inside one request, while `retry` governs durable generation attempts (section 8.3). Settings changes make -no commit and emit no event. +no commit and change no view. ```ts // The user's settings, read live through getters. @@ -1005,7 +1087,7 @@ wait aborts only that waiter. Conversation handles are stateless; compare them by `id`. Hosts discover conversations through lookups and scans. There is no creation listener; the task -graph view (section 9.5), `inspect()`, and the idle waits report live work. +graph view (section 9.4), `inspect()`, and the idle waits report live work. `submit()` returns after durable admission, not settlement. An input submission creates a user message with the Harness clock's timestamp at placement, which is @@ -1036,7 +1118,7 @@ under the current registry: `running` with an active invocation; `completing` for a held outcome; `waiting` with the live tasks it still waits for, the live part of its `on`, or, when abort-marked, its live ordinary owned work (section 5.5); `sleeping` with its `until` while the clock has not reached it (section -5.6); `ready` when the next scheduling pass would reserve it, with `migrates` +5.6); `pending` when the next scheduling pass would reserve it, with `migrates` when its definition is newer and has `migrate`; or `blocked` (section 5.4). A migration shows as failed only after the scheduler tried it, or when the newer definition has no `migrate`; inspection never runs one to find out. Blocked reasons are derived, never stored. Queued and placed @@ -1284,14 +1366,55 @@ without storage-assigned lifetime fields. There is no mutable `session.document()` API. ```ts -type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +type SessionEnd = { readonly kind: "closed" } | { readonly kind: "failed"; readonly error: unknown }; + +type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; +type CommitOptions = { readonly persistence?: CommitPersistence }; + +/** One stored commit as `observeCommits()` publishes it (section 4.1); change order is unspecified. */ +type CommitPublication = { readonly seq: Seq; readonly changes: readonly CommitChange[] }; +type CommitChange = + | Extract + | DocumentCommitChange; +/** A creation or update (`kind: "change"`), a retirement (`kind: "retire"`), or a definition-free copy. */ +type DocumentCommitChange = + | { + readonly type: "document"; + readonly kind: "change"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId | undefined; + readonly version: number; + readonly value: JsonObject; + /** Empty for a creation. */ + readonly ops: readonly Op[]; + } + | { + readonly type: "document"; + readonly kind: "retire"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId | undefined; + } + | { + readonly type: "document.copy"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId; + readonly source: { readonly id: DocumentId; readonly at: DocumentPoint }; + }; +type CommitObserver = { + advance(publication: CommitPublication, context: Context): void; + end(end: WatchEnd): void; +}; +type CommitObservation = { detach(): void }; interface Session extends DocumentObserver { - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** `options.persistence` may defer storing the commit (section 4.1). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; close(context: Context): Promise; - /** Settles once closed, Storage included: `{ reason: "closed" }`, or `{ reason: "failed", error }` (section 10.1). */ + /** Settles once closed, Storage included: `{ kind: "closed" }`, or `{ kind: "failed", error }` (section 10.1). */ readonly closed: Promise; - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; + observeCommits(observer: CommitObserver): CommitObservation; subscribeClose(listener: () => void): () => void; snapshot(token: SessionDocToken, context: Context): Promise | undefined>; @@ -1303,6 +1426,14 @@ interface Session extends DocumentObserver { snapshotAsOf(token: RewindableConversationDocToken, conversationId: ConversationId, at: EntryId, context: Context): Promise | undefined>; snapshotAsOf(token: RewindableConversationDocFamilyToken, conversationId: ConversationId, key: string, at: EntryId, context: Context): Promise | undefined>; + + /** A synchronous observer of one document address, for adapters (section 9.1); the scope arguments as for `snapshot()`. */ + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocToken, conversationId: ConversationId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocToken, taskId: TaskId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: SessionDocFamilyToken, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocFamilyToken, conversationId: ConversationId, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocFamilyToken, taskId: TaskId, key: string, observer: Observer>, context: Context): Promise> | undefined>; } interface Tx { @@ -1332,7 +1463,7 @@ interface Tx { ): Promise>; /** Raw submission record; no admission rules (busy check, inbox, placement). Hosts use `Conversation.submit()`. */ createSubmission(create: SubmissionCreate): Promise; - /** Settle a queued or placed submission; only a placed input can be answered. A settled one stays unchanged. */ + /** Settle a queued or placed submission; only a placed input or continue can be answered. A settled one stays unchanged. */ settleSubmission(id: SubmissionId, settlement: SubmissionSettlement): void; /** Newest visible entry of the conversation that carries a `head`. */ latestHeadMarker(conversationId: ConversationId): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; @@ -1481,12 +1612,25 @@ callback fails with no pending acquisition callback succeeds with no pending acquisition prepare every open change -> immutable next revision + self-contained Chord Op[] Session evaluates each required/ordinary document write exactly once - Storage.commit persists the atomic batch while the Session line remains held -storage succeeds - adopt every prepared change by pointer swap and enqueue immutable revision/ops publication - commit listeners, including conversation view mounts (section 9.3), capture it synchronously - release the line; invoke listeners later -storage fails +commit deferred and holdable (section 4.1) + join the writer's queue unstored; release the line + once the commits before it are published: adopt every prepared change by pointer swap; + queue its ops per document; watches and view mounts capture it synchronously; + observeCommits() does not; invoke listeners later +otherwise, at logical admission (section 4) + take the held ops of every document the commit acquired and put them in front of its own + join the writer's pending batch; release the line +the writer's storage call makes it durable (a receipt) and settles + after the commits before it: apply it to the read caches, adopt every prepared change by + pointer swap; record its telemetry events + queue one microtask that calls observeCommits() observers with what was stored + the Session's own observers, including watches and view mounts (section 9.3), capture it + synchronously; the waits they resolve and the aborts they queue run after that microtask + fail the Session if an own observer threw (the commit stands, the rest of the call is not published) +the call rejects after it stored the commit + the commit is published; the storage error rejects it and fails the Session (section 10.1), + and an own observer's throw is reported +the commit is not stored (refused, rejected, or after a failure) abort every prepared change, fail the Session (section 10.1), and publish nothing ``` @@ -1552,8 +1696,11 @@ The Session evaluates this predicate exactly once after tracker preparation. base, excluding the change being evaluated. Storage reports it when it materializes the current value; the Session advances it after each adopted write and resets it after every base, so evaluation performs no Storage read. -Creation and version transitions require bases and do not call it. The Session -then gives Storage only the selected representation: +Creation and version transitions require bases and do not call it. A deferred +commit's predicate also runs once, when it is held, against the stored count; a +base it selects is stored with the held changes, as a store whose held +operations overflowed is (section 4.1). The Session then gives Storage only the +selected representation: ```text required or predicate true -> base with complete value @@ -1693,26 +1840,102 @@ granule, a tenth of the retention up to one second. Task documents also retire with their task (section 5.5). `createSession(storage, { documentRetentionMs, setTimer })` sets them for a plain Session; the Harness reads its settings at each use. -A Session commit callback may be asynchronous. It owns the Session mutation -line through callback execution, preparation, storage settlement, committed -baseline adoption, and publication enqueue. Adoption runs inside -`Storage.commit()`: the Session passes an `adopt(seq)` callback, which the -storage calls synchronously once the batch is durable and before its next -commit (section 10), so memory always shows a complete prefix of the log. The -callback keeps its own error: a failed adoption fails the Session once -`Storage.commit()` returns. A commit that rejects after its adoption fails the -Session too, and its adopted changes stay adopted. Commit observers run on the -line; watch listeners run later. -External model, process, tool, network, and human effects run outside it. -`subscribeCommits()` observes complete immutable publications synchronously on -the line after adoption. `subscribeClose()` observes close synchronously when it +A Session commit callback may be asynchronous. The Session mutation line runs +one job at a time: a commit's callback and its settlement, or a read on the +line. A commit with writes then joins the pending batch (logical admission), +and the line goes on with the next job. Storage overlaps the line (group +commit): a writer off the line sends every commit admitted while no storage +call is in flight as one `Storage.commit()` call (section 10), each with +its own `seq`, and publishes them one by one, in line order. Memory +always shows a complete prefix of the log, plus what deferred commits hold +(section 4.1). External model, process, tool, network, and human effects run +outside the line. + +No job observes what an unpublished commit wrote. Every read or write of a job +touches a key: a record (task, conversation, entry, submission, a +conversation-scoped request ID), a document address or incarnation, a scope's +documents, a conversation's newest entries, or, in the scheduler, the live work +below an ownership node, the ownerless roots, and the cancellation intent set; +a scan that cannot be keyed touches every key. While an admitted commit that +writes the key is unpublished, the job waits until it is stored and published, +then goes on; a job waiting holds the line, so the unpublished writes only +shrink. Code before a job's first such touch runs early: a callback can run +while an earlier commit is being stored, and cancellation is checked when the +job starts. The scheduler decides in regions that record the keys they read and +run again after such a wait (section 5.4). A commit's outcome is delivered once +it is published, and every job's outcome (commits with and without writes, +reads, deferred commits, callback errors) in line order; a scheduler commit +that writes nothing and waited at no gate delivers to the scheduler at once. A failure that fails +the Session makes every later outcome `SessionFailed`; a callback's own error, +a cancellation, or a refused read-only commit affects only its job. At most 256 +admitted jobs wait for their outcome; beyond that the line waits. The bound is a +count, not bytes: what waits is bounded by what one commit can hold (tool output +by its output limits), times the count. + +A commit is published after the storage call that stored it settled, so every +view changes in the same step as memory. A Storage with `publishing()` runs each +commit's publication inside a scope of its own: the store backend runs it in its +attached host's batch, so Chord sources over Durable's views are marked in one +batch per commit. Its publication applies it to the Session's read caches and +adopts it in memory; then come the commit's telemetry +events (`pi.durable.entry`, `pi.durable.submission`), then the Session's own +observers, synchronously: document watches, conversation view mounts, the task +graph, the scheduler, and submission waits, with the observers adapters attach +(`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, section 9.1). They +advance watches and resolve waits, but run no task code: the abort signal of a task the commit newly marks +follows in a microtask. Host `observeCommits()` observers run one microtask +after the publication, before every wait the commit resolves and every abort it +queues: those continuations are queued by the own observers, after the +dispatch. There is no order between host listeners and watch listeners, nor +`onWake`: they are independent observers of the commit, and a watch delivery +already queued for an earlier frame may deliver this commit first. Two commits +of one storage call are published a macrotask apart, so the host listeners of +the first, and the continuations of the waits it resolves (`commit()`, +`Submission.wait()`, `waitForTask()`), run before the second is published. + +```text +commit([.., 41, 42]) settles +publish 41: caches, trackers, telemetry events, own observers (views advance, waits resolve) +microtask: observeCommits() observers +microtasks after it: wait continuations, session.commit() resolves, abort listeners +microtasks, in no order relative to the above: watch listeners, onWake +macrotask: publish 42 +``` + +Publication never throws. A throwing telemetry adapter is reported (`host_error`, +callback `telemetry`; a plain Session drops it, as it drops every report) and +the commit resolves; a host listener's throw is reported (section 10.1), and an +adapter's observer that throws is also detached and ended (section 9.1). The +first throw of an own observer is kept, the other observers and the host +listeners still get the commit, and it fails the Session after the host +listeners got it; the commit stands and resolves, and the commits after it in +the same call are not published (section 10.1). A throw while memory adopts +the commit is different: nothing more of it is published, the Session fails, +and the commit rejects with that error. A commit Storage stored whose call then +rejects is durable: it is published, waits it resolved stay resolved, the +commit rejects with the storage error, and the Session fails with it; an own +observer's throw in that commit is then reported as a `listener_error`. + +`observeCommits(observer)` observes stored commits in one synchronous +`observer.advance(publication, context)` call per observer, on the line, each with exactly what that storage +transaction stored (section 4.1). It attaches synchronously and returns a `CommitObservation` with an idempotent +`detach()`; unlike the other `observeXXX()` it has no `value`: commits are events, not a state, and the Session does +not know Storage's newest `seq` before its own first commit. A publication +queues the one dispatch of a commit only when an observer is attached; the +dispatch calls the observers attached when it runs. So an observer attached +between the publication and the dispatch gets the commit only when some +observer was attached at the publication; one detached in between does not get +it; one attached after the dispatch ran misses it. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Session has closed, +after its last publication, or `failed` with the error that failed it; never after `detach()`. An observer whose +`advance` throws, or returns a promise that rejects, is reported (section 10.1), detached, and ended +`listener_error`; the others still get the commit. `subscribeClose()` observes close synchronously when it begins, after admission is sealed, also when a failure closes the Session; the Harness ends waits, watches, and streams there, and signals task invocations after every close listener has run, so what a task holds ends with the Session's -reason rather than as cancelled. Both return idempotent -disposers; their listeners must not block or call Session APIs, and one that +reason rather than as aborted. It returns an idempotent +disposer; its listeners must not block or call Session APIs, and one that throws is reported without affecting the others (section 10.1). Watch listeners -still run later, off the line. +still run later, as microtasks the line never waits for (invariant 7). ```ts await session.commit(async tx => { @@ -1726,9 +1949,15 @@ await session.commit(async tx => { ``` Mutation admission occurs on the Session line before a commit callback starts. Closing seals mutation admission and task -reservation. Already-admitted commits settle before storage closes. Once a -commit is admitted, caller cancellation does not interrupt storage settlement or -undo the commit. Cancelling a close wait does not reopen admission. +reservation, runs the close listeners, and signals every task invocation; the +callbacks already admitted finish, waits at their touches included, every +pending and in-flight storage call settles and is published, the writes at +close and the held changes go through the same writer, and then storage closes. +Once a commit is admitted, caller cancellation does not interrupt storage +settlement or undo the commit. Cancelling a close wait does not reopen admission. +`unloadDocuments()`, a reload for another definition version, and the store at a +held document's deadline wait until every admitted commit is published, then +store what is held at that point through the writer. Table rules: @@ -1737,6 +1966,10 @@ Table rules: - Any table read after the first table write throws `ReadAfterWrite`. - Document access and read-your-writes remain available after table writes. - Creation methods return their created ID/record; callers do not read it back. +- `Conversation.submit(tx, draft)` and `ConversationHandle.submit(tx, draft)` + come before the commit's first table write, another admission's included; + otherwise they throw `ReadAfterWrite`, whether or not the admission would read + a table (section 6). Storage ownership: @@ -1747,6 +1980,90 @@ Storage ownership: - Every storage read returns a detached JSON value. - Immutable strings may be shared; mutable arrays and objects may not. +### 4.1 Deferred commits + +A commit whose writes are only cheap to lose, such as a streamed answer or +running tool output, may be stored later than it is shown: + +```ts +await runtime.commit( + async (tx) => { (await tx.doc(LiveDoc, conversationId)).generation!.message = partial; }, + context, + { persistence: { kind: "deferred", withinMs: 1_000 } }, +); +``` + +`Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a +tool's `api.commit()` take the option; without it, or +with `{ kind: "immediate" }`, a commit is stored before it resolves. `withinMs` +is a number of milliseconds, `0` or more (`Infinity` waits for one of the other +triggers below); anything else rejects with a `TypeError`. Without it, a +Harness uses `settings.progress.persistWithinMs` (default 1,000) and a plain +Session its `persistWithinMs` option (default 1,000). + +The mark is a permission. A deferred commit is **held** only when, after its +callback, settlement, and the read-only check, every write it makes changes an +existing conversation document with `history: "latest"` and `fork: "initial"` +at its stored version. Anything else, such as an entry, a task state, a +submission, another document, a document's creation, migration, or retirement, +makes it immediate, a normal commit. A held commit is adopted in memory by +pointer swap, resolves, and is published to the Session's own observers: document +watches and the conversation view (section 9). Its stored +version and delta count stay; its operations wait, folded per incarnation +(Chord's `PendingOps`), and the incarnation is pinned in the cache, so idle +eviction skips it at any retention. + +What is held is stored by the first of: + +1. a normal commit that acquires the document (`tx.doc()`), whether it changes + it or only reads it, and stores anything: the held operations go in front of + its own in the one content write of that incarnation, in its storage + transaction. A stored delta of a held document therefore always begins with + its held operations; a base the commit writes covers them; +2. the document's deadline, the earliest `withinMs` deadline of its held + commits on the Session clock, kept by one foreground timer (section 5.6); + when it fires, every held document is stored in one storage transaction: + a deadline is the latest time, and the others cost little more there; +3. `close()`, after the writes at close and before the trackers are dropped; + a failure of that store fails the Session, and the backend still closes; +4. any drop of a tracker that holds them: `unloadDocuments()`, and a token of + another definition version, which reloads the document from Storage; + retirement is case 1. + +Held operations leave their queue only at the commit's logical admission +(section 4), after every fallible step: a callback, settlement, or checkpoint +failure of a normal commit keeps them, and a commit never stores a hold adopted +after it in line order. A held commit is adopted in memory and published when +the commits before it are published, and what it writes stays unpublished until +then: a commit that reads the document waits for it. A store fails like any +storage write: it fails the Session (section 10.1). Past Chord's +`MAX_PENDING_OPS` (10,000) held operations, or when a held commit's +`checkpointWhen` selected a base, the store writes a base of the current value. +A failed Session stores nothing; after a reopen nothing is held. + +Publication: the Session's own observers see each held commit once, when memory +adopts it, and of a normal commit that stores held changes only its own +operations. `observeCommits()` sees every storage transaction with exactly +what it stored, published once stored (section 4): nothing for a +held commit, and the stored operations, held first, for a commit or a deadline +that stores them. A document the commit retires is published as its retirement +only: the held operations and the commit's own change stored before the +retirement are not. A consumer that applies +`observeCommits()` publications to stored state, such as a replica, stays +equal to Storage in every live document. Its `seq` values have gaps where nothing was published. + +Consistency: every stored document is a state it had in memory, and a normal +commit that reads a held document through `tx` stores it with whatever it +derives. What a crash loses is a held commit's changes: up to `withinMs` plus the +line and I/O delay. A value read outside a commit (`snapshot()`, a watch) may be +ahead of Storage; nothing durable may be derived from it without reading it in +a commit. Other readers of Storage, such as another process, a replica, or an +export, see held changes once they are stored. + +The Harness defers exactly two kinds of commits: generation partials (section +8.3) and running tool progress (section 7.3). Every commit that consumes or +clears them reads `pi.live` and so stores them first (section 8.2). + ## 5. Tasks ### 5.1 Definition @@ -1853,6 +2170,8 @@ interface TaskRuntime extends DocumentObserver, Docum current: RunningTask, ) => NextTaskState | undefined | Promise | undefined>, context: Context, + /** May defer storing a commit that changes only documents (section 4.1); one that returns a state is stored. */ + options?: CommitOptions, ): Promise; memo(name: string, context: Context): Promise; @@ -1871,6 +2190,13 @@ interface TaskRuntime extends DocumentObserver, Docum /** Committed entry visible from the task's conversation. */ entry(id: EntryId, context: Context): Promise; entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** Committed raw history of the task's conversation, fork-aware, as `Conversation.entries()`. */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; /** Committed raw active transcript and model context, optionally cut off at `at`. */ context(conversationId: ConversationId, context: Context, options?: { readonly at?: EntryId }): Promise; /** The Harness clock. */ @@ -1879,7 +2205,7 @@ interface TaskRuntime extends DocumentObserver, Docum /** A `task_error`, or with `from` an `extension_error` or `host_error`, with this task (section 10.1). */ report( error: unknown, - from?: { type: "extension"; extension?: string } | { type: "host"; callback: HostCallback }, + from?: { kind: "extension"; extension?: string } | { kind: "host"; callback: HostCallback }, ): void; /** Commit `{ status: "sleeping", until, checkpoint }`; resolves once stored (section 5.6). */ sleepUntil(until: number, checkpoint: S, context: Context): Promise; @@ -1937,7 +2263,8 @@ Harness is closing, the task is terminal, or a run invocation's task carries an abort mark. Its `tx.createTask()` defaults to the task's conversation, and every entry it appends records the task as `byTaskId`. When the callback returns a state, the runtime replaces the task's state in the same -commit, so the checkpoint or outcome is atomic with the callback's entries, +commit (stored at once even when `options` defers it, section 4.1), so the +checkpoint or outcome is atomic with the callback's entries, documents, and child tasks and is type-checked against the task's checkpoint and result types. Returning nothing leaves the state unchanged. A terminal or completing state drops the memos. A `waiting` or `sleeping` state ends the @@ -2100,7 +2427,16 @@ or its `orphaned` settlement when its definition is unavailable. ### 5.4 Scheduler, abort, and ownership The scheduler serially reserves eligible tasks, then runs handlers off the -Session line. One in-memory `TaskInvocation` contains mode, abort controller, +Session line. Its decisions stay serialized on the line while their storage +overlaps (section 4): each reads the scheduler's memory, which changes as +commits are published, in a region that records what it reads (a task's +record, the live work below an ownership node, the ownerless roots, the +cancellation intent set) and stages nothing; when a commit that writes one of +those is unpublished, the region waits for it and runs again, so `migrate`, +registry snapshots, reports, and cleanup run once, after a decision stands. +Every commit writes the memberships its task changes alter: a task created or +ended changes the live work of every node from its parent up to the nearest +live task or the root conversation, and the chain is loaded at its admission. One in-memory `TaskInvocation` contains mode, abort controller, and completion promise. Abort protocol: @@ -2182,7 +2518,7 @@ Every run invocation start resolves the task's definition by `TaskRecord.kind` from the scheduler's current registry snapshot: ```text -restarts at restartLimit -> stay pending, blocked: restart_limit +restarts at restartLimit -> a kind of an extension: terminal faulted (built-in tasks end themselves) definition missing -> stay pending, blocked: missing_task stored version newer than it -> stay pending, blocked: task_too_old stored version older than it -> migrate(input, checkpoint, storedVersion) @@ -2198,20 +2534,43 @@ after joining the invocations it stopped, close writes those tasks back as reset leaves `running` records. A state the task commits itself that differs from its current one (a new checkpoint, a wait, a sleep, an outcome) is progress and removes the count; the same checkpoint committed again, -memos, and document writes are not. Once `restarts` reaches -`settings.restartLimit` (default 5; `Infinity` never parks), the task is parked: -blocked with reason `restart_limit`, reported once as `task_parked` at the open -that parks it, shown in `inspect()` and the task graph, and not active in -`wake()`. Work that deterministically exhausts memory or CPU before its next -checkpoint thus stops after a few attempts instead of running forever. Every -task running when a process dies is counted, not only the one that killed it, -so a long stream next to the culprit can be parked too; a host shows parked -tasks and offers `Harness.retryTask(id)`, which removes the count and runs the -task again, and does nothing for a task that is not parked. A run task that is -parked keeps its conversation busy. An abort mark starts the count over and -still runs a parked task's abort handler; an abort handler cut off -`restartLimit` times is not run again, and the task ends `faulted` with detail -`{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit`. +memos, and document writes are not. Work that deterministically exhausts memory +or CPU before its next checkpoint thus shows as a growing count, and once +`restarts` reaches `settings.restartLimit` (default 5; `Infinity` never ends a +task for it), the task ends visibly instead of running again: + +| task | at the limit | +|---|---| +| tool call (`pi.tool`) | ends with an error result, diagnostic code `restart_limit` ("Tool X was interrupted 3 times in a row; it is not run again"), built from its slot like an interrupted call's; its task fails with detail `{ reason: "restart_limit" }` | +| compaction | fails with that error and detail, recorded in `pi.live.compactionFailure` | +| generation | ends its run `unanswered` with `pi.live.runFailure` reason `restart_limit`, failing with that detail; a run that answered already, in `finish` or `drain`, ends without a failure and the generation completes (section 8.3) | +| a kind of an extension | the scheduler ends it `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit` | + +A built-in task that ends itself this way, and a generation whose run ends +`interrupted` (section 8.3), is reported as `task_interrupted` with that reason, +the task, and its `restarts`, so a host keeps a crash-loop signal. The built-in +tasks check the count when a phase starts, a tool call after reading its call; +reading the setting only when the task has a count, so the normal path does no +more work. The scheduler checks an extension's task when it reserves it, +reading the current setting, so a raised limit lets the task run; a task that +already ended stays ended. A generation's request or deferred poll in flight +when the host died is a failed attempt under the retry policy (section 8.3) +first; the limit still bounds it, so a charge that itself keeps cutting the +task off (converting a huge partial, say) ends the run. At the limit a task +still does its usual cleanup, such as converting a partial or cancelling a +deferred response; past it, after that cleanup was cut off too, it ends +without. Every task running when a process dies is counted, not only the one +that killed it, so a long stream next to the culprit can end too. A nested +call runs only while its replay-safe caller runs, so a crash counts both and +the caller reaches the limit no later than its nested calls; the caller's own +`restart_limit` result is what the model sees. An abort mark starts the count +over and still runs the abort handler; an abort handler cut off `restartLimit` +times is not run again, and the task ends `faulted` with detail +`{ reason: "restart_limit" }`, reported as `task_faulted` with reason +`restart_limit`. Someone aborted that task, so its conversation shows the +abort: a run ends with `runFailure` reason `aborted`, the cut-off as `detail`, +without converting its partial, and a compaction's status is removed without a +failure. An abort invocation resolves the definition the same way. When the definition can take the task, its abort handler runs; otherwise the task is orphaned as @@ -2225,10 +2584,9 @@ incompatible, neither at open nor later. A blocked task without an abort mark keeps its durable record unchanged, remains live, still blocks ordinary idle waits, and is reconsidered whenever the registry changes. The blocked reason is derived runtime state, not a persisted task status. A blocked task is not active in -`wake()` (section 5.6): only a registry change, or `retryTask()` for a parked -one, can let it run, and either makes `onWake` report it active, so a host need -not poll it. The task graph shows the reason and follows registry changes -(section 9.5). Document migration remains +`wake()` (section 5.6): only a registry change can let it run, and it makes +`onWake` report it active, so a host need not poll it. The task graph shows the reason and follows registry changes +(section 9.4). Document migration remains access-driven. Aborting a blocked task cannot run its abort handler, because that code is @@ -2322,7 +2680,9 @@ The **ordinary owned work** of a task `T` is every task that ordinary traversal reaches below `T`: the tasks `T` owns, the tasks in the conversations `T` owns, and, transitively, the same for each of those that is not background. A background task and everything below it are excluded. A conversation-owned task -may be background; a child task may not. +may be background; a child task may not. The tasks a generation creates for its +`afterRun` hooks (section 8.3) are such children, so Stop and idle waits include +them, and a cascade from the generation aborts them first. Two rules follow from the tree. @@ -2448,7 +2808,8 @@ commits and task changes. The default declines them in Cloudflare Workers, since a pending timer keeps a Durable Object from hibernating or being evicted even when `nodejs_compat` gives it an `unref()`, and wherever timers cannot be unreferenced. Foreground timers pace or resume work: throttled -progress commits through `TaskRuntime.setTimer()`, and, without `onWake`, the +progress commits through `TaskRuntime.setTimer()`, the deadline of deferred +commits (section 4.1), and, without `onWake`, the timer for the earliest sleep while scheduling runs. That one keeps a Node.js process alive, since a sleeping task is work left to do. A timer should not fire early and may fire late; every caller checks its clock again when it fires. The @@ -2469,12 +2830,12 @@ paused, when the Session next needs a running process: Blocked tasks (section 5.4), `completing` tasks, and waits on live tasks count for neither; the work they wait for does. With `HarnessOptions.onWake`, the Harness arms no timer for sleeps. It calls `onWake` with `wake()` after open and -whenever the value changes, off the Session line; a throw is reported. The host +whenever the value changes, in a microtask the line never waits for; a throw is reported. The host arms its own wakeup and calls `resume()` when it fires, which makes due sleeps runnable. A host whose process can be evicted, such as a Durable Object with its one alarm, arms the alarm at `at`, or at `now + heartbeat` while `active` if that is earlier, and deletes it when neither is set. `onWake` runs after the -commit that changes the value, so a host call that can make an idle Session +commit that changes the value was adopted (section 4), so a host call that can make an idle Session active arms the recovery wakeup first: `submit()`, `compact()`, `abortTask()`, `Conversation.abort()`, a registry change, and `resume()` after open. An early wakeup is harmless; a missing one leaves work stranded. A failed alarm write @@ -2517,7 +2878,7 @@ An input is placed as one entry whose `model` is the user message of its `content`: a `pi.user` entry, or, with `entry: { kind, data }`, an entry of that kind carrying that data. Only `kind` and `data` are taken from `entry`; admission rejects a kind that is not a non-empty string or starts with `pi.`. Nothing in -the Harness reads input entries by kind; context, compaction, events, and run +the Harness reads input entries by kind; context, compaction, and run accounting go by `model` and the submission records. A write of only `kind` and `data`, with no `model`, `head`, or `edits` (present @@ -2539,30 +2900,82 @@ inputs, including while its tool round runs, then the next generation. Tool tasks never own the run; the current round's tool tasks are listed in `pi.live.tools`. The ID list is mutable state because a boundary adds placed steering inputs to an active run; every terminal path -settles exactly the listed inputs. +settles exactly the listed inputs. A run that answered while `afterRun` hooks +are selected (section 8.3) settles its inputs `done` at the answer and keeps +`run`, with an empty ID list and `run.afterRun` set, until the tasks the hooks +asked for are terminal: the conversation stays busy, and `Submission.wait()` +has already resolved. Admission and terminal transitions: | action | submission state | other writes | |---|---|---| | input, idle with empty inbox | `placed`, with user entry | create run and generation | +| continue, idle | `placed`, no entry | place queued items (final boundary); create run and generation with them and itself | +| continue, busy | none | reject with `ConversationBusy` | +| input with `whenIdle: "reject"`, idle | none | reject with `ConversationIdle` | | display-only write, no queued reset | `done`, with entry | append entry; no run | | write, idle with empty inbox | `done`, with entry | append entry; no run | | input or other write, busy or non-empty inbox | `queued` | append inbox item | | boundary places user item | `placed`, with entry | add ID to current or successor run | | boundary places write | `done`, with entry | append entry | -| run answers | input `done`, with required answer entry | remove `run`, or hand it to a successor | +| run answers | input or continue `done`, with required answer entry | remove `run`, or hand it to a successor; with `afterRun` hooks selected, keep `run` until the tasks they ask for are terminal | | run fails or its task aborts | input `unanswered`, with reason | remove `run`; inbox per `afterFailedRun` below | +| tool round reaches `maxTurns` | input `unanswered`, reason `turn_limit` | as a failed run | | withdraw queued item | `unanswered`, reason `aborted` | remove inbox item | | stale head write | `unanswered`, reason `stale` | remove inbox item | `requestId` deduplicates within one conversation before any write; reusing one for the other submission type rejects. A busy input with `whenBusy: "reject"` -writes no record and reports `ConversationBusy`. An idle input queues with mode -`steer` when `whenBusy` is `steer` and `followUp` otherwise. +writes no record and reports `ConversationBusy`. An idle input with +`whenIdle: "reject"` writes no record and reports `ConversationIdle`, also when +the inbox holds queued items, so the input never starts a run of its own: with +`whenBusy: "steer"` it steers the current run or is refused, except as below. Idle means +`run === undefined`, the inverse of `busy` above. A run that answered and waits +for its after-run tasks is busy, so such an input queues there; it starts the +next run when that run ends, as every item queued behind a final answer does. An idle input queues with +mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. -`settings.afterFailedRun` decides what a run that fails with `model_error` or -`no_model` does with the inputs queued behind it, in the commit that settles its +`Conversation.submit(tx, draft)` runs the same admission inside the caller's +commit instead of one of its own and returns the submission's ID; a task or tool +does so through `ConversationHandle.submit(tx, draft)` in its own commit. The +Harness supplies the clock, queue modes, the `pi.durable.submit` span (same +parent, links, and trace carrier as `submit(draft, context)`), and the request +for progress a started run needs. The admission's writes, a started run +included, are stored with the caller's other writes in that commit, or not at +all: a host can check its own documents and `pi.live`, admit, and record the +submission in one commit. A rejection throws before any write. Admission must +precede the commit's first table write, another admission's included: it throws +`ReadAfterWrite` otherwise, on every path, also where it would read no table (a +busy input without request ID or trace). Entries it appends inside a task's or +tool's commit carry that task's `byTaskId`, as the commit's own appends do. The +span covers the admission only: it ends before the caller's commit is stored, +its `result` is provisional (a later throw in the callback or a storage failure +leaves no submission), and the commit's events (`pi.durable.entry`, +`pi.durable.submission`) go to the caller's commit context. A first argument +that is not a draft is taken as a transaction; one of another Session throws. + +A continue starts a run on the context as it is, without a user message, for +example after a run that ended without an answer. It is admitted only while the +conversation is idle and never queues: whether a run can continue depends on how +the current one ends, so a busy conversation rejects it with `ConversationBusy` +before any write. Its record is `placed` without an entry; an idle conversation's +queued items are placed first by a final boundary in the same commit, and its +selected user items join the run, ahead of the continue. Whether the context has +anything to answer is checked when the run's generation prepares (section 8.3), +not at admission: a context that does not end in a user message or tool results, +system messages aside, ends the run without a request, `unanswered` with +`nothing_to_answer`. An answer cut off at its output limit (`length`) is an +answer, so a text answer is not continued; one with tool calls is, because its +calls never ran and the context ends in their synthesized `missing_result` +error results (section 2.1). A host offers continue while the conversation is +idle and `pi.live.runFailure` is set (section 8.2); a stale failure (one a later +run started in its commit kept) settles `nothing_to_answer`. Each continue starts +a run with its own turn budget. + +`settings.afterFailedRun` decides what a run that fails with `model_error`, +`no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, or +`turn_limit` does with the inputs queued behind it, in the commit that settles its own inputs: `keep` (the default) leaves them queued; `continue` applies the final boundary, so the next run starts with them, as after an answer; `withdraw` settles them `unanswered` with reason `run_failed`, keeping queued @@ -2622,8 +3035,11 @@ A reset placed at a `final` boundary follows the answer, so the inputs are already `done`. At ordinary `postTools`, generation continues even with no queued trigger; -selected steer IDs join that continuation. A terminating or handoff tool round -(section 8.5) uses final behavior instead. At `final`, the current run's placed +selected steer IDs join that continuation. A handoff tool round (section 8.5) +first appends its reset and then applies this ordinary `postTools` boundary, so +queued writes and selected steers land in the new context and follow-ups wait +for the run's final boundary. A terminating tool round uses final behavior +instead. At `final`, the current run's placed input submissions settle first, unless an `onYield` continuation keeps them open (below); selected user IDs start one successor generation with those IDs as the new run's inputs. Writes never trigger generation by @@ -2631,10 +3047,16 @@ themselves. A final boundary without user triggers leaves the conversation idle, except for an `onYield` continuation (section 8.3), which applies only when the boundary selected no user item and no reset. -Only successful run ends apply the final boundary: an answer, `terminate`, or -`handoff`. Failure, a run task's abort handler, fault, and orphan settle the -run's inputs `unanswered` and leave the inbox alone (`Conversation.abort()` -separately withdraws queued user items); the queued items stay visible in +Only successful run ends apply the final boundary: an answer or `terminate`. +With `afterRun` hooks selected, it applies when the run ends after its +after-run tasks, not with the answer, so whether items queued before the answer +land right after it depends on the conversation's extension selection. Items +queued meanwhile wait as well, steers included: there is no next model request +in the run for them to join. A run whose after-run phase was cut off +`settings.restartLimit` times ends without this boundary, as a failed run does. +Failure, a run task's abort handler, fault, and orphan settle the run's inputs +`unanswered`, unless the run answered already and waits for its after-run tasks +(its inputs are `done`), and leave the inbox alone (`Conversation.abort()` separately withdraws queued user items); the queued items stay visible in `pi.inbox` until the next input or queued write runs a boundary, or their withdrawal. Selected and stale items are removed positionally while retained item order is @@ -2889,7 +3311,7 @@ await conversation.configure({ extensions: { add: [Venv] } }, context); |---|---|---| | generation `prepare` | agent (tools, sections, model, thinking level), stream options, compaction thresholds for that model | once per request; model, stream options, and offered tools are fixed in the request checkpoint (section 8.3) | | every task phase | extension selection for hooks; `runtime.agent()` | at most once per phase handler, at its first hook dispatch or `agent()` call, from that phase's snapshot and the committed `pi.agent`, off the Session line, with the invocation's context; fixed for the rest of the phase | -| tool task | the implementation of an accepted call | at `call` and `execute`: the name among the phase agent's `tools`, the current selection after the tools filter; no such tool, for example one removed after preparation: `tool_unavailable`. Replay policy on reopen uses the same lookup (section 7.3) | +| tool task | the implementation of an accepted call | at `call` and `execute`: the name among the phase agent's `tools`, the current selection after the tools filter; no such tool, for example one removed after preparation, or a call flagged `unoffered` (section 8.3): `tool_unavailable`. Replay policy on reopen uses the same lookup (section 7.3) | | generation, when a tool round starts | tool execution mode | once per round (section 8.3) | | generation, compaction | retry policy | at each attempt's classification | | generation, on a context overflow | compaction policy for the request's model | at classification | @@ -2897,8 +3319,8 @@ await conversation.configure({ extensions: { add: [Venv] } }, context); | boundaries and idle admission, on the Session line | queue modes | per boundary (section 6) | | tool task, `prepare`, `runtime.env()` | environment | at each call (section 2.2) | -Registry installs and settings changes make no commit, so they emit nothing -(section 9.4). A conversation on the default selection changes when the +Registry installs and settings changes make no commit, so no view changes +(section 9.3). A conversation on the default selection changes when the settings or the installed extensions change; its next preparation appends the system delta (section 7.4). A UI showing resolved tools or a resolved model resolves again through `Conversation.agent()`. @@ -2917,11 +3339,13 @@ consuming commit may rerun them. Abort errors always propagate. | hook | composition | ordinary throw | |---|---|---| | `beforeRequest` | replacement chain | report, continue | +| `beforeProviderRequest` | replacement chain | report, continue | | `afterResponse` | all observers | report, continue | | `onYield` | first continuation wins | report, continue | | `beforeTool` | argument replacement chain; first block wins | block tool with error text | | `afterTool` | result replacement chain | report, continue | | `afterTools` | all observers | report, continue | +| `afterRun` | all observers; requested tasks concatenated in extension order | report, continue | | `beforeCompact` | first decision wins | report, continue | Hooks use task memos for durable first-writer-wins decisions. There is no public @@ -2934,11 +3358,20 @@ type HookResult = T | undefined | Promise; interface HookApi extends DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; - /** `HarnessOptions.models`. */ /** The current `HarnessOptions.models`, read at each access. */ readonly models: Models; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; + /** As `TaskRuntime.entry()` (section 5.1). */ + entry(id: EntryId, context: Context): Promise; + entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** As `TaskRuntime.entries()` (section 5.1). */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; } interface GenerationHooks { @@ -2948,14 +3381,37 @@ interface GenerationHooks { api: HookApi, context: Context, ): HookResult<{ readonly messages: readonly Message[] }>; + /** After pi-ai builds the provider request body, before it is sent; replaces the body for that request only. */ + beforeProviderRequest( + request: { readonly model: Model; readonly payload: unknown }, + api: HookApi, + context: Context, + ): HookResult<{ readonly payload: unknown }>; /** Every terminal provider message, before classification. */ afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise; /** A final answer; `continue` appends a user message and continues the run. */ onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>; /** After every tool of the round is terminal; `results` are the round's result entries in call order. */ afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; + /** After a run answered: the tasks to run before it ends (section 8.3). */ + afterRun(run: RunSummary, api: HookApi, context: Context): HookResult<{ readonly tasks: readonly RunEndTask[] }>; } +/** How a run ended, for `afterRun`. */ +type RunSummary = { + /** A final answer, or a tool round whose every call asked to `terminate`. */ + readonly reason: "answer" | "terminate"; + /** The run's first entry (`pi.live.run.first`, section 8.2), or the answer for a run without one. */ + readonly first: EntryId; + /** The answer, or the terminating round's tool-calling answer. */ + readonly answer: EntryId; + /** The run's last entry: the answer; for a `terminate` round its newest result, or the reset of its handoff. */ + readonly last: EntryId; +}; + +/** Built by `runEndTask(task, input)`, which types `input` by the task. */ +type RunEndTask = { readonly task: AnyTask; readonly input: JsonValue }; + /** `parent` is set for a nested call (section 7.3). */ type ToolHookCall = ToolCall & { readonly parent?: { readonly taskId: TaskId; readonly callId: string } }; @@ -2966,7 +3422,10 @@ interface ToolHooks { api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result is committed; replaces the result. */ + /** + * Before the result is committed; replaces the result. Runs on every result the tool task writes, also the error + * results the Harness writes itself (section 8.4), except past the restart limit. + */ afterTool( call: ToolHookCall, result: ToolExecutionResult, @@ -3007,6 +3466,16 @@ after a first decision, does so inside its own `invoke`. Hooks receive the task' runtime as their `HookApi`; hook memos and the task's own memos share one namespace, so hook authors prefix their memo names. +Hooks read committed state as it is when they run, off the line, so a hook a +crash reruns may see more. Entries never change and compaction never removes +them (section 8.7): a range bounded by entry IDs, such as an `afterRun` hook's +`entries({ minEntryId: run.first, maxEntryId: run.last })`, reads the same +before and after a compaction and in a rerun. `HookApi` has no `context()`: +the active transcript changes in the middle of a run, when a compaction summary +or a handoff's reset lands, so a hook asking what a run did reads its history +instead. A range also holds entries of others, such as display-only writes, +compaction summaries, and entries of other tasks; hooks filter by `kind`. + Selection decides which behavior a conversation has; the extension's own document holds that behavior's state: @@ -3032,6 +3501,53 @@ await root.commit(async (tx) => { }, context); ``` +An `afterRun` hook starts durable work at the end of a run, without a tool +call, and the run waits for it (section 8.3). Here the project's tests run after +every run that edited files, and a failure starts a fix run: + +```ts +const RunTests = defineTask<{ command: string }, { phase: "run" }, string>({ + name: "app.run-tests", version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, context) => { + // The app's helper over `ExecutionEnv.exec()`: { passed: boolean; output: string }. A rerun after a crash runs it again. + const result = await runTests(await runtime.env(context), task.input.command, context); + const handle = (await runtime.conversation(runtime.conversationId, context))!; + await runtime.commit(async (tx) => { + // Admitted in this commit, before its table writes; placed when the run ends, in queue order with other input. + if (!result.passed) { + await handle.submit(tx, { type: "input", content: `Tests failed:\n${result.output}`, whenBusy: "followUp" }); + } + await tx.appendEntry(runtime.conversationId, { kind: "app.tests", data: { passed: result.passed } }); + return { status: "terminal", outcome: { status: "completed", result: result.output } }; + }, context); + }, + }, + abort: async (_task, runtime, context) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), context), +}); +export const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tasks: [RunTests], + hooks: [hook(GenerationTask, { + afterRun: async (run, api, context) => { + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, context); + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && ["edit", "write"].includes(message.toolName); + }); + return edited ? { tasks: [runEndTask(RunTests, { command: "npm test" })] } : undefined; + }, + })], +}); +``` + +A fix run starts another run, which asks `afterRun` again, with a fresh turn +budget (`AgentState.maxTurns`); the extension caps such a loop itself, for +example with a counter in its own document. + ### 7.3 Tools ```ts @@ -3062,7 +3578,13 @@ type ToolExecutionResult = { interface ConversationHandle { readonly id: ConversationId; - submit(submission: InputSubmissionDraft, context: Context): Promise; + /** Any submission, as `Conversation.submit()` takes: a task may also continue a conversation it owns. */ + submit(submission: SubmissionDraft, context: Context): Promise; + /** + * Inside the invocation's commit (`runtime.commit()`, a tool's `api.commit()`), as `Conversation.submit(tx, ...)`; + * appended entries carry the task's `byTaskId`. Rejects once the invocation is aborted or ended. + */ + submit(tx: Tx, submission: SubmissionDraft): Promise; abort(context: Context): Promise; waitForIdle(context: Context): Promise; } @@ -3087,7 +3609,8 @@ interface ToolExecutionApi extends Docum retainedOutput(): { readonly text: string; readonly truncated: boolean }; diagnostic(diagnostic: ToolDiagnostic): void; details(value: TDetails, context: Context): Promise; - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** A commit of the call's task; `options` may defer storing it (section 4.1). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; createTask( @@ -3249,8 +3772,12 @@ error diagnostic already says what went wrong, so programs get it, for example diagnostic too. So a hook that redacts a schema-less tool's output redacts what programs get. An error result the Harness writes itself (unavailable, invalid arguments, -blocked, interrupted, aborted, abandoned, or a scheduler fault) has no -`structuredOutput`; its diagnostics say what went wrong. A thrown error +blocked, interrupted, aborted, abandoned, ended at the restart limit, or a +scheduler fault) has no +`structuredOutput`; its diagnostics say what went wrong. When an `afterTool` +hook turns such a result into a success, programs get its bounded output for a +schema-less tool, or one that did not resolve, and no value for a schema tool; +the hook's own `structuredOutput` is dropped. A thrown error becomes a `tool_error` diagnostic with its message. A nested call whose result breaks this gets an error result with an `invalid_structured_output` diagnostic, without the broken value: a schema tool's then has no `structuredOutput`, a schema-less tool's its output; @@ -3281,8 +3808,14 @@ declares them, built from the resolved tools. Loaded tools follow the other tools in load order, so a load appends declarations to the transcript and the cached prefix holds. `@earendil-works/pi-durable/tool-search` provides the coding agent's `tool_search` (`createToolSearchExtension()`): it ranks -`Agent.deferred` with BM25 over the tools' names, descriptions, and parameter -names and descriptions, and loads the matches. The `codemode` section does not +`Agent.deferred` with the coding agent's tool ranking and loads the matches. +The ranking is BM25 over the tools' descriptions and parameter names and +descriptions, with the name as a separate field: a tool matching more of the +query's terms ranks above one matching fewer, a query term in the name +outweighs any number of mentions in the description, a tool whose whole name +after its last `__` is in the query gets a large bonus, and a query that is +exactly a tool's name ranks it first. Code mode's `searchTools()` ranks the +same way, with each tool's namespace text as a further, low-weight field. The `codemode` section does not list deferred tools; scripts find them with `searchTools()`. Tools reach files and processes only through `api.env`, never through an @@ -3424,7 +3957,8 @@ canonical path before opening it does not prevent a concurrent rename or symlink swap, so callers that restrict paths do so for hygiene, not security. The `details()` promise resolves after the corresponding or coalesced document -commit. During normal settlement the tool task stops its throttle and awaits the +commit, which memory adopts at once and Storage gets with the next commit that +reads `pi.live` or within `settings.progress.persistWithinMs` (section 4.1). During normal settlement the tool task stops its throttle and awaits the commit in flight; the terminal commit, which appends the result entry, is the final flush and settles any `details()` promise still pending. Abort and close obey invocation and Session admission gates: uncommitted buffered updates may be @@ -3443,7 +3977,10 @@ A tool call is accepted only if its request offered the tool. Generation checks this against the tool set replayed from the committed model context through the request's `cutoff`; `beforeRequest` replacements do not change it. It answers a call to a tool it did not offer with the `tool_unavailable` result described in -section 2.2, without a tool task. An agent change after preparation does not +section 2.2, through a tool task whose input is flagged `unoffered`: the task +resolves nothing, so even a tool the agent has (a deferred one not loaded yet) +never runs, and `afterTool` and `control` apply to the result as to any call's. +An agent change after preparation does not affect this check (section 2.2). The tool task resolves the called name among the `tools` of the phase's agent resolution, the tools filter applied. It uses that implementation @@ -3508,7 +4045,7 @@ that submission's result. Provider call IDs are not unique across a Session and must not serve as request IDs. Aborting or abnormally terminalizing the tool task cascades through the owned scope. The tool reports the child in its running details, for example `api.details({ conversationId })`, -so a UI that sees the call can attach to the child's view or events. +so a UI that sees the call can attach to the child's view. ```ts export const Subagent = defineExtension({ @@ -3593,10 +4130,13 @@ generation's `tools` phase (section 8.5) adds the named tools to the conversation's stored tools filter; they take effect at the next preparation. The round terminates only when every result of the round requests `terminate`, as in the pi agent loop; the `tools` phase then uses a final boundary. Any -`handoff` in the round, the last one in call order when several ask, ends the -run the same way after appending a `pi.reset` entry with `head: "self"` and the -handoff text as a user message (section 8.1), exactly what `reset(handoff)` -writes. +`handoff` in the round, the last one in call order when several ask, appends a +`pi.reset` entry with `head: "self"` and the handoff text as a user message +(section 8.1), exactly what `reset(handoff)` writes, and the run continues: the +next generation answers in the new context, and the run's inputs settle with +that eventual answer or error, not with the tool-calling answer. A later round +may hand off again. A round that also terminates writes the reset and then ends +the run as `terminate` does. A result's `usage` is stored on the tool-result message and added to the conversation's `pi.usage.tools[toolName]` in the result commit (section 8.6). A @@ -3613,7 +4153,8 @@ retain their tool-result entry ID for the generation's `tools` phase. Error results the Harness writes itself set `isError` and carry an `error` diagnostic with one of the codes `tool_unavailable`, `invalid_arguments`, -`blocked`, `interrupted`, `aborted`, or `tool_error` (a throw from `execute()`) +`blocked`, `interrupted`, `abandoned`, `restart_limit` (section 5.4), +`aborted`, or `tool_error` (a throw from `execute()`) and the error text as its message. Their content is the durable partial output, if any, and `details` is the tool's last reported value, if any. @@ -3964,8 +4505,12 @@ continuation calls `models.fetchDeferred()` and `models.cancelDeferred()` with that same model and signal. Missing models and synchronous/streamed pi-ai errors are classified into the durable generation outcomes below. -Generation and tool progress are throttled durable document commits. A crash may -lose only the uncommitted throttle window. Recovery converts committed partials +Generation and tool progress are throttled document commits, deferred +(section 4.1): watches see each at once, and Storage gets them with the next +commit that reads `pi.live` or within `settings.progress.persistWithinMs` +(default 1,000 ms). A crash may lose the throttle window plus what was not +stored yet: a stream that crashes before its first store leaves no aborted entry, +and that attempt's usage is not recorded. Recovery converts stored partials to normal interrupted/aborted transcript entries, clears presentation state, and then retries or terminates according to the task phase. Retry deadlines, attempts, compaction, and tool progress are current document state for late @@ -3988,7 +4533,7 @@ is exported as an `Entry` token. | `pi.user` | `[UserMessage]`, timestamp from the Harness clock at admission or placement | submissions, `onYield` continuations | | `pi.assistant` | `[AssistantMessage]` with any stop reason | generation | | `pi.system` | `[SystemMessage]` with `content: ""` (section 7.4) | generation preparation | -| `pi.tool-result` | `[ToolResultMessage]` | tool tasks; generation for calls to tools its request did not offer | +| `pi.tool-result` | `[ToolResultMessage]` | tool tasks; generation for calls an aborted round never started | | `pi.reset` | absent, or `[UserMessage]` with the handoff text; always `head: "self"` | `reset()`, generation `tools` phase for `handoff` | | `pi.compaction` | `[UserMessage]` with the wrapped summary; `head` is the first kept entry | compaction tasks (section 8.7) | @@ -4001,8 +4546,24 @@ messages out of later requests, so no separate usage or notice kind exists. ```ts type LiveState = { - /** Run control (section 6); present exactly while the conversation is busy. */ - run?: { taskId: TaskId; inputs: SubmissionId[] }; + /** + * Run control (section 6); present exactly while the conversation is busy. `turns` counts the run's tool rounds, + * which `AgentState.maxTurns` bounds (section 8.5); an `onYield` continuation starts it over (section 8.3). + */ + run?: { + taskId: TaskId; + /** Placed inputs, settled when the run ends; `[]` once it answered and waits for its after-run tasks. */ + inputs: SubmissionId[]; + turns: number; + /** + * The run's first entry, where `afterRun` hooks start reading (section 8.3): the first one the boundary that + * started it placed, a write's or an input's; a continue's `runFailure.first`; otherwise the run's first + * tool-calling answer. A run started before it was recorded may lack it and gets its answer instead. + */ + first?: EntryId; + /** The run answered and waits for the tasks its `afterRun` hooks asked for, listed once created (section 8.3). */ + afterRun?: { tasks?: TaskId[] }; + }; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -4018,7 +4579,7 @@ type LiveState = { * IDs ascend. */ nestedTools?: { - /** `/`: the call's ID in tool events, unique because keys contain no `/`. */ + /** `/`: the call's ID, unique because keys contain no `/`. */ callId: string; parentCallId: string; /** The calling tool task, model-issued or nested; the Harness relates slots by task ID. */ @@ -4040,11 +4601,10 @@ type LiveState = { tools?: { callId: string; name: string; - /** - * Absent for a call not started yet (sequential round, `pending`) and for a call its request did not offer, - * which starts `done` with the `entry` generation wrote. - */ + /** Absent for a call not started yet (sequential round, `pending`). */ taskId?: TaskId; + /** A call to a tool its request did not offer: its task settles `tool_unavailable` without running anything. */ + unoffered?: true; status: "pending" | "running" | "done"; /** Retained running output and what the bounds dropped. */ output?: string; @@ -4064,18 +4624,52 @@ type LiveState = { * after the task ended. */ compactionFailure?: CompactionFailure; + /** + * Why the newest run ended without an answer, so a client can show it after the run ended. Set in the commit that + * ends the run; removed when a later commit starts a run. + */ + runFailure?: RunFailure; +}; + +/** The newest run that ended without an answer. */ +type RunFailure = { + /** The run's task at its end. */ + taskId: TaskId; + /** The run's inputs, settled `unanswered`. */ + inputs: SubmissionId[]; + reason: + | "model_error" + | "no_model" + | "aborted" + | "reset" + | "faulted" + | "orphaned" + | "interrupted" + | "restart_limit" + | "nothing_to_answer" + | "turn_limit"; + /** + * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`); `interrupted` and `restart_limit`: + * what was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. + */ + detail?: string; + /** The failed run's `run.first`, which a continue takes over. */ + first?: EntryId; }; type CompactionFailure = { taskId: TaskId; - reason: CompactionReason; + /** What started the compaction that failed. */ + trigger: CompactionReason; /** The failed or faulted outcome's error message, or why the scheduler orphaned the task. */ error: string; }; type CompactionStatus = { taskId: TaskId; - reason: CompactionReason; + /** What started it: `compact()`, a threshold, or a context overflow. */ + trigger: CompactionReason; /** Whether a generation waits for it: a compaction the generation owns. */ blocking: boolean; attempt: number; @@ -4087,9 +4681,10 @@ type CompactionStatus = { | field | value | |---|---| | kind | `pi.live` | -| version | `1` | +| version | `3` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | +| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`; from versions 1 and 2: `run.turns` set to `0` where it is missing (section 3.6); `run.first`, `run.afterRun`, and `runFailure.first` are optional and need no step | | checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | @@ -4106,9 +4701,11 @@ its 64 KiB scan, the commit writes the retained window as one set. Either way each commit writes at most one window, and the throttle bounds the rate. Generation creates `tools` with one `pending` slot per call in the commit that -appends the tool-calling answer; a call its request did not offer starts `done` -with its result entry. A tool task sets its slot `running` in its intent commit, -publishes throttled output and details into it, and in its terminal commit sets +appends the tool-calling answer; a call its request did not offer is flagged +`unoffered`. A tool task sets its slot `running` (and a nested slot's final +arguments) with a deferred commit (section 4.1) right before its intent commit, +publishes throttled output and details into it with deferred commits, and in its +terminal commit sets `done` and `entry` and removes `output`, `droppedBytes`, `droppedLines`, `details`, and `diagnostics`, which the result entry now carries (section 7.3). The generation's `tools` phase removes `tools`. Slot updates apply only while a slot with the task's @@ -4132,11 +4729,51 @@ that outcome holds as `completing` (section 5.5), or the scheduler's cleanup It stays small: the summary text lives in the task's placement, never here. Partials are normalized to strict JSON before assignment. Every terminal path of a run task removes `run`, `generation`, and `tools` in the commit that settles the run's -inputs. `tx.settleSubmission()` stages each input's new status +inputs. A run that answered while `afterRun` hooks are selected is the one +exception: the answer's commit settles its inputs and removes `generation` and +`tools`, and `run` stays, with `run.afterRun` set, until the run ends after its +after-run tasks (section 8.3). + +A run that ends without an answer records why in `runFailure`, in the commit +that settles its inputs `unanswered`: + +| reason | the run ended because | input settlement | +|---|---|---| +| `model_error` | the response failed after any retries, or a blocking or overflow compaction it needed failed; `detail` is the error text | `model_error` with the text | +| `no_model` | no model is configured, or the configured one is not available; `detail` names it | `no_model` | +| `aborted` | its task was aborted (`Conversation.abort()`, `abortTask()`); `detail` says when its abort handler was cut off `restartLimit` times (section 5.4) | `aborted`, with that detail | +| `reset` | a queued reset started a new context at the post-tools boundary (a tool's handoff continues the run) | `reset` | +| `faulted` | the scheduler faulted its task (section 5.4); `detail` is the error message | `faulted` with the message | +| `orphaned` | its task was aborted while no definition could take it; `detail` is the reason | the reason | +| `interrupted` | the host stopped while the model was answering, with no retry left (section 8.3); `detail` names the attempt | `interrupted` with the text | +| `restart_limit` | its generation was cut off `restartLimit` times in a row (section 5.4), also while an interrupted request was charged; `detail` says in which phase | `restart_limit` with the text | +| `nothing_to_answer` | its generation found a context that does not end in a user message or tool results, system messages aside, such as a continue after an answer (section 8.3); no `detail` | `nothing_to_answer` | +| `turn_limit` | a tool round reached the conversation's `maxTurns` and ran its tools (section 8.5); `detail` is the run's count, `"1 turn"` or `"N turns"` | `turn_limit` with the count | + +The next commit that starts a run removes it. A run started in the same commit, +the queue's next run under `afterFailedRun: "continue"` (section 6) or after a +reset, keeps it, so the failure is shown until a later run starts. A run that +ends with an answer, including a round whose tools all ask to terminate, leaves +it alone; so does one that answered and then ends without its after-run tasks +(aborted, orphaned, or cut off): it answered. A run that fails copies its +`first` into `runFailure.first`, so a continue reads the failed run's history +too. Inputs settled without a run (withdrawn, stale, aborted while +queued, `run_failed`) are not run failures; their submission records say why. `tx.settleSubmission()` stages each input's new status and resolves the transaction's latest candidate submission record, falling back to committed state, during assembly, like task-document validation (section 3.3), so it is not a caller table read and works after the commit's first table write. +The streamed fields, `generation.message` and a running slot's `output`, +`droppedBytes`, `droppedLines`, `details`, and `diagnostics`, are written by +deferred commits (section 4.1). Every commit that derives anything from them or +clears them acquires `pi.live`: converting a partial into an aborted entry, +settling a tool from its slot, handing a generation over to its tool round, +ending a run, and clearing a safe rerun's progress. It therefore stores them +first, with its own changes, and after a crash Storage holds a `pi.live` its +memory had: control fields as of the last normal commit, streamed fields as of +the last store. Recovery's own conversions and clears are normal commits, so a +second crash never converts the same partial twice. + ### 8.3 Generation ```ts @@ -4179,18 +4816,40 @@ type GenerationCheckpoint = tools: TaskId[]; /** Calls of a sequential round not started yet, in call order. */ pending: string[]; + } + | { + /** The run answered; its `afterRun` hooks are asked for tasks, again after a crash. */ + phase: "finish"; + run: RunSummary; + } + | { + /** Waiting on the tasks the hooks asked for, which the generation owns; `waiting` holds this checkpoint. */ + phase: "drain"; + answer: EntryId; + tasks: TaskId[]; }; type GenerationResult = { entryId: EntryId }; ``` -`pi.generation` is version 1 and starts at `{ phase: "prepare", attempt: 1 }`. +`pi.generation` is version 2 and starts at `{ phase: "prepare", attempt: 1 }`; +`migrate` keeps a version 1 record. Version 2 keeps calls to tools not offered +in a sequential round's `pending`, flagged `unoffered` in their slots (section +8.5), which a version 1 Harness would start as tool tasks without the flag, so +it leaves version 2 blocked (`task_too_old`); the `finish` and `drain` phases +below are version 2 as well. The run's inputs live in `pi.live.run`, not in the task input. - `prepare` runs section 7.4 with the phase's agent resolution and resolves the settings once. When the agent has no model or `models.getModel()` does not know it, the task fails with `no_model`. When it resumes with `overflow` and its compaction did not complete with an `entryId`, the run fails with - `model_error` and the overflow text as detail. Otherwise one commit appends + `model_error` and the overflow text as detail. When the committed model + context does not end in a user message or tool results, system messages + aside (it is empty, or ends in an answer, as for a continue after one), the + run fails with `nothing_to_answer` before any section is rendered or request + sent. Only roles count: a `length` answer with only text ends the context, + while one with tool calls is followed by the error results synthesized for + its calls (section 2.1), so a continue answers those. Otherwise one commit appends the planned `pi.system` entries and moves to `request` with the new tail as `cutoff`, the agent's model and thinking level, and the settings' stream options. These stay fixed for this request attempt; a retry prepares again. `compacted` carries over to @@ -4229,15 +4888,37 @@ The run's inputs live in `pi.live.run`, not in the task input. `prepare`. `request` converts a leftover partial (below) before it resolves the model. - `request` first converts a committed partial left in `pi.live` by an - interrupted attempt into an aborted `pi.assistant` entry. It then streams the + interrupted attempt into an aborted `pi.assistant` entry, and sets + `pi.live.generation` to the attempt with a deferred commit (section 4.1), which + the conversion makes immediate. When the host died + while this request was in flight (the record's `restarts`, section 5.4), the + attempt failed: that commit also either sleeps at `retry` until + `now + retryDelayMs(policy, attempt)`, with `pi.live.generation.retry` saying + "The host stopped while the model was answering, on attempt N", when the + retry policy allows another attempt, or settles the inputs `unanswered` and + ends the run with `runFailure` reason `interrupted` and that text, failing the + task. A request a close stopped is not counted and is sent again, below. A + `poll` whose fetch was in flight is charged the same way ("The host stopped + while the deferred answer was fetched"), checked before its `pollAt`, which a + replacement host's clock may not have reached: its `retry` keeps the deferred + response and polls it again as the next attempt instead of sending another + request, and a run that ends instead cancels it, as the abort handler does. + A generation asleep in `retry` or `poll`, or waiting for its tools, is not + running and never charged. Past `restartLimit` the run ends `restart_limit` + instead (section 5.4). Otherwise it streams the model context through `cutoff` with the invocation signal, the thinking level as `reasoning` (omitted for `off`), the conversation's persisted provider `sessionId`, and the pinned `streamOptions`, committing throttled partials at - most every `settings.progress.partialIntervalMs` (default 100 ms). + most every `settings.progress.partialIntervalMs` (default 100 ms), deferred + (section 4.1), so Storage gets them at the next commit that reads `pi.live` + or within `settings.progress.persistWithinMs`. Before streaming, the `beforeRequest` chain may - replace the messages for this request only. Recovery resends the same - committed messages with the same pinned model, thinking level, and stream - options, and reruns `beforeRequest`. + replace the messages for this request only. Once pi-ai has built the provider + request body, the `beforeProviderRequest` chain may replace it, again for this + request only; it runs each time pi-ai builds a body. After a close, recovery resends + the same committed messages with the same pinned model, thinking level, and + stream options, and reruns `beforeRequest`; after a crash the attempt is + charged instead, as above. - `afterResponse` observes every terminal message, from `request` or `poll`, before classification; a still deferred result is not terminal. - Before classifying, the handler stops the partial throttle and awaits any @@ -4247,11 +4928,23 @@ The run's inputs live in `pi.live.run`, not in the task input. `{ continue }` wins. The commit appends the answer and applies the final boundary (section 6). With a continuation and no selected user item or reset, it appends a `pi.user` entry with the continuation content, creates a - successor generation, and hands it `pi.live.run`, keeping the inputs open. + successor generation, and hands it `pi.live.run`, keeping the inputs open and + setting its `turns` to `0`: each stretch a continuation starts gets the whole + turn budget, and bounding `onYield` loops is the extension's job (it reads + `pi.live.run.turns` and `pi.agent.maxTurns` with `api.snapshot`). The chain runs + whatever the count. Otherwise it settles the run's inputs `done`, removes `run` and `generation`, and starts a successor run for the selected user items, if any; a dropped continuation is not retried. Both complete with - `{ entryId }`. + `{ entryId }`. Whether a continuation applies is decided from the inbox + before the boundary places anything: it applies when the inbox holds no + steer, no follow-up, and no reset. + When the phase's agent selects an `afterRun` hook and no continuation + applies, the commit instead appends the answer, settles the run's inputs + `done`, keeps `run` with `inputs: []` and `afterRun: {}`, removes + `generation`, applies no boundary, and moves to `finish` with + `{ reason: "answer", first, answer, last }`: `first` is `run.first`, or the + answer when the run has none, and `answer` and `last` are the answer. - `toolUse` with at least one tool call: the commit appends the assistant entry and starts the tool round described below, moving to `waiting` in the `tools` phase. A `toolUse` message without calls is classified like `stop`. @@ -4281,19 +4974,45 @@ The run's inputs live in `pi.live.run`, not in the task input. it sleeps again. A still deferred result sleeps until a strictly later `pollAt`; any other result is classified as above. A model gone after reopen fails `no_model` once the poll is due. +- `finish` asks the `afterRun` hooks with its `run` (section 7.2) and checks + each requested task off the line: its kind must be an installed task that is + not built-in (`pi.*`), its input JSON, and the installed definition's + `initial()` must return a JSON checkpoint of one of its phases. A request + that fails a check is skipped and reported as an `extension_error` of the + hook that asked for it; the run never fails for one. The task is + created from the installed definition, not the hook's token, with the + checkpoint `initial()` returned, so `initial()` runs once. One commit creates + the tasks as children of the generation, lists them in `run.afterRun.tasks`, + and commits `waiting` on them with `allSettled` at `{ phase: "drain", answer, + tasks }`. Without tasks it ends the run at once, as `drain` does. A crash in + `finish` runs it again and asks the hooks again; nothing was created. Once the + tasks exist, the hooks are not asked again for this run, and a crash leaves + the wait and the tasks to resume. +- `drain` runs once every after-run task is terminal, whatever its outcome. One + commit applies the final boundary (section 6), removes `run`, starts a + successor run for the selected user items, and completes with + `{ entryId: answer }`. +- `finish` and `drain` check the restart count like every phase (section 5.4). + At the limit the run ends `completed` with the answer, without its tasks, + without a `runFailure`, and without the final boundary, so queued items stay + queued as after a failed run; the scheduler reports it `task_interrupted`. - The abort handler calls `models.cancelDeferred()` in `poll` when the model is known (a failure is reported), converts a committed partial, settles the inputs `unanswered` with - `aborted`, removes `run` and `generation`, and ends `aborted`. + `aborted`, removes `run` and `generation`, and ends `aborted`. In `finish` and + `drain` the run answered already: the handler runs after the after-run tasks + are terminal (section 5.5), removes `run`, and records no `runFailure`; the + inputs stay `done`. A tool round starts in the commit that appends the tool-calling answer: 1. The offered tools are replayed with pi-ai `getCurrentTools()` from the committed model context through `cutoff`: `request` already holds it, and - `poll` derives it again. A call to a tool not offered gets its - `tool_unavailable` result entry here, without a task. -2. Every other call gets a `pi.tool` task owned by the generation, with input - `{ assistant, callId }`. The round is sequential when the settings' - `toolExecution`, read as the round starts, is `sequential` or any called + `poll` derives it again. +2. Every call gets a `pi.tool` task owned by the generation, in call order, with + input `{ assistant, callId }`, plus `unoffered: true` (also on its slot) for a + call to a tool not offered, which settles `tool_unavailable` without resolving + the name (section 8.4). The round is sequential when the settings' + `toolExecution`, read as the round starts, is `sequential` or any offered called tool, resolved from the phase's agent as the tool task resolves it (section 7.3), has `executionMode: "sequential"`: then only the first call gets its task now and the rest wait in `pending`. Otherwise every call gets its task at @@ -4302,12 +5021,14 @@ A tool round starts in the commit that appends the tool-calling answer: `pending` shrinks. 3. The generation commits `waiting` on its tool tasks with `allSettled` and the `tools` checkpoint (section 8.5); `pi.live.run` stays with it. -4. `pi.live.tools` receives the round's slots (section 8.2), and `generation` is - removed. +4. `pi.live.tools` receives the round's slots (section 8.2), `generation` is + removed, and `pi.live.run.turns` grows by one; a run without `first` gets the + tool-calling answer as its first entry. -Input submissions settle `unanswered` with one of these reasons: `no_model`, -`model_error` (detail: provider error text), `aborted`, `faulted` (detail: error -message), or an orphaning blocked reason (section 5.4). Fault and orphan +Input and continue submissions settle `unanswered` with one of these reasons: +`no_model`, `model_error` (detail: provider error text), `nothing_to_answer`, +`turn_limit` (detail: the turn count, section 8.5), `aborted`, `faulted` (detail: error message), or an orphaning blocked reason +(section 5.4). Fault and orphan settlement convert a committed partial into an aborted `pi.assistant` entry, like the abort handler. @@ -4323,9 +5044,11 @@ type ToolTaskCheckpoint = type ToolTaskResult = { kind: "model"; entryId: EntryId; control?: ToolControl } | { kind: "nested" }; ``` -`pi.tool` is version 2 and starts at `{ phase: "call" }`; `migrate` gives a -version 1 input `kind: "model"`, and readers treat a terminal result without -`kind` as a model-issued call's. A model-issued call's input stays small +`pi.tool` is version 3 and starts at `{ phase: "call" }`; `migrate` gives a +version 1 input `kind: "model"` and keeps a version 2 input (version 3 added +`unoffered`), and readers treat a terminal result without `kind` as a +model-issued call's. A call flagged `unoffered` resolves no tool: `call` settles +it `tool_unavailable` and the abort handler does too. A model-issued call's input stays small because the terminal record keeps it; the call is read from the assistant entry. A nested call (section 7.3) carries its call, since no entry holds it, and settles differently: its result commit records the result's `usage` @@ -4339,17 +5062,22 @@ Below, `{ entryId }` is a model-issued call's result. phase's agent `tools` (section 7.3), validates the arguments with pi-ai `validateToolArguments()`, runs the `beforeTool` chain, and validates again - (section 7.3). One commit then records intent: it moves to `execute` with the - final arguments and the tool's replay policy and sets the slot `running`. The + (section 7.3). A deferred commit sets the slot `running`; one commit then + records intent: it moves to `execute` with the final arguments and the tool's + replay policy. After a crash the slot may still show `pending` while the task's + checkpoint is `execute`; a safe rerun sets it `running` again. The same handler executes the tool, runs the `afterTool` chain, and commits the result; it builds `api.env` with `HarnessOptions.env` right before executing (section 2.2). A tool that does not resolve, invalid arguments, or a block - commits the corresponding error result instead, without intent. + runs the `afterTool` chain on the corresponding error result and commits it + instead, without intent. - `execute` is reached only by recovery. When both the stored and the current policy, resolved as in `call`, are `safe`, it executes again with the stored arguments, without `beforeTool`, and settles like `call`. Otherwise it commits an `interrupted` error result from the slot's durable partial output, details, and - diagnostics, and ends `failed` with `{ entryId }`. + diagnostics, after the `afterTool` chain, and ends `failed` with `{ entryId }`. + Either way, a task cut off `settings.restartLimit` times ends with a + `restart_limit` result instead (section 5.4), also after the chain. - The result commit bounds the content, appends the diagnostics block (section 7.3), appends one `pi.tool-result` entry with `model: [{ role: "toolResult", toolCallId, toolName, content, details, isError, durationMs, timestamp }]` and @@ -4369,7 +5097,25 @@ Below, `{ entryId }` is a model-issued call's result. `completing` with the result entry already written (section 5.5); the generation resumes only when the tool task is terminal. - The abort handler commits an `aborted` error result from the slot's durable - partial output, details, and diagnostics and ends `aborted` with `{ entryId }`. + partial output, details, and diagnostics, after the `afterTool` chain, and ends + `aborted` with `{ entryId }`. The scheduler faults a task whose abort handler + was cut off `restartLimit` times, without a result (section 5.4), so the last + attempt it runs, at `restartLimit - 1` restarts, skips the chain. +- Every result the task writes passes the `afterTool` chain: the slot is read + before the result commit, since hooks run outside commits, and does not change + while the tool does not run. A result the Harness wrote itself is then bounded + with the tool's output limits (the defaults when it did not resolve), gets a + truncation diagnostic when that dropped content (a slot's truncation + diagnostic stays only while the hooks keep its output), and loses any + `structuredOutput`, except that a nested call the chain turns into a success + gets its bounded output as value, as a schema-less tool's success does + (unless the tool declares a schema, section 7.3); `control` applies only when + the task completes. Once the + task's restarts exceed `settings.restartLimit`, the chain no longer runs, + since it may be what keeps cutting the task off. Results that no tool task + writes never pass it: the generation's `aborted` results for calls a + sequential round never started (section 8.5), a nested call's fallback result + (section 7.3), and a missing result synthesized in context (section 2.1). ### 8.5 Tool rounds @@ -4388,19 +5134,35 @@ and then commits once: no effect for others. Names are written without consulting the registry, so a tool that is not registered now is offered once it is again. The next preparation offers the tool when it resolves and the model may call it. -- When every result of the round requests `terminate`, or any requests - `handoff`, it appends the handoff's `pi.reset` entry, if any, settles the - run's inputs `done` with the tool-calling answer, removes `run` and `tools`, - and applies the final boundary. +- When any result requests `handoff`, it first appends the `pi.reset` entry of + the last one in call order and advances the boundary's active range start to + it, so the boundary below places its writes and user items after the handoff + text, and a queued head write targeting an entry before it is stale. +- When every result of the round requests `terminate`, it settles the run's + inputs `done` with the tool-calling answer, removes `run` and `tools`, and + applies the final boundary. When the phase's agent selects an `afterRun` + hook, it keeps `run` instead and moves to `finish` (section 8.3) with + `{ reason: "terminate", first, answer, last }`: `answer` is the tool-calling + answer and `last` the newest of the round's result entries and the handoff's + reset; the final boundary waits for the run's end. +- Otherwise, when the run's `turns`, counted in the commit that started the + round, reached the phase agent's `maxTurns`, the run ends as a failed run + (section 6): its inputs settle `unanswered` with `turn_limit` (detail: the + count), `runFailure` records it, `afterFailedRun` decides the queued items, and + the generation still completes with `{ entryId }`. A continue resumes it on the + round's results (section 6). - Otherwise it removes `tools` and applies the `postTools` boundary. When that - boundary placed a reset, the run's inputs settle `unanswered` with `reset` and - selected user items start a successor run (section 6). Otherwise selected - steer IDs join `pi.live.run`, and it creates the next generation, owned by the - conversation, and hands it the run. + boundary placed a queued reset, which wins over a handoff written before it, + the run's inputs settle `unanswered` with `reset` and selected user items + start a successor run (section 6). Otherwise selected steer IDs join + `pi.live.run`, and it creates the next generation, owned by the conversation, + and hands it the run; after a handoff, that generation prepares and requests + in the new context. It completes with `{ entryId }` of the tool-calling answer. Its abort handler runs only after its tool tasks are terminal (section 5.5). For every call still -in `pending`, which never started, it appends an `aborted` error result; a +in `pending`, which never started, it appends an `aborted` error result +(`tool_unavailable` for an `unoffered` one), without `afterTool`; a started call whose task faulted or was orphaned keeps its `done` slot without an entry (section 8.2). It then settles the inputs `unanswered` with `aborted`, removes `run`, `generation`, and `tools`, and ends @@ -4481,9 +5243,10 @@ ways to start one; the task is the same, only ownership and placement differ: Only a blocking compaction runs while the conversation's run waits for it. The others never take run control: the conversation keeps working while they -summarize, and their summary is placed like any passive write. Compactions do -not coordinate with each other; several may run at once, and section 6 decides -between their summaries. +summarize, and their summary is placed like any passive write: while a run +that answered waits for its after-run tasks (section 8.3), at the boundary that +ends it. Compactions do not coordinate with each other; several may run at +once, and section 6 decides between their summaries. **Range selection** is a pure function of a `ContextView`, whose `contributions` give each active entry's model messages after every edit in the @@ -4543,7 +5306,7 @@ Phases: `sessionId`, and the pinned stream options without `deferred`, with `cacheRetention: "none"` and the pinned `maxTokens`. Providers such as Codex may suppress that identity when caching is disabled. Recovery resends the same - request. The response is classified in one commit + request, until the task was cut off `restartLimit` times in a row (section 5.4). The response is classified in one commit that adds its usage to `pi.usage` (section 8.6): - `stop` with non-empty text and no tool call: the text is the summary, placed as below. @@ -4600,7 +5363,7 @@ aborted with its generation, before the generation's abort handler runs conversation abort keeps it, and `Submission.abort()` withdraws it. **Failure.** The commit that fails a compaction (`model_error`, `no_model`) -removes its status and sets `pi.live.compactionFailure` to its task, reason, +removes its status and sets `pi.live.compactionFailure` to its task, trigger, and error message, replacing an older failure; the Harness cleanup does the same for a compaction the scheduler faults or orphans, with the fault's message or the orphan reason. Only a compaction that produces a summary @@ -4623,23 +5386,31 @@ re-exports both): ```ts type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "retired" | "ended" } - | { readonly reason: "listener_error"; readonly error: Error } - /** The source failed; for Durable, what failed the Session, usually a storage error (section 10.1). */ - | { readonly reason: "failed"; readonly error: unknown }; + | { readonly kind: "stopped" | "aborted" | "ended" } + | { readonly kind: "listener_error"; readonly error: Error } + /** The source failed for good; for Durable, what failed the Session, usually a storage error (section 10.1). */ + | { readonly kind: "failed"; readonly error: unknown }; + +/** + * One listener call: the newest value, the ops from the previous value, and a retirement since the previous value + * call; or a failure the source may recover from, after which the watch stays open. Durable sends no `failed` call. + */ +type WatchCall = + | { readonly kind: "value"; readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean } + | { readonly kind: "failed"; readonly error: unknown }; interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Installs the sole serialized asynchronous listener. */ - start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void; + start(listener: (call: WatchCall, context: Context) => Promise): void; /** Idempotently stops future callbacks and returns this watch's terminal result. */ stop(): Promise; /** Settles when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type DocumentWatch = WatchHandle | null>; +type DocumentWatch = WatchHandle>; interface DocumentObserver { watchDoc(token: SessionDocToken, context: Context): Promise | undefined>; @@ -4651,23 +5422,42 @@ interface DocumentObserver { } ``` -A watch ends once, with the first reason that applies: `stopped` by `stop()`; -`cancelled` when its acquisition `Context` is cancelled; `retired` when its -document incarnation retired; `ended` when the Session closed; `listener_error` -when its listener threw or rejected; `failed` when the Session failed. +A watch ends once, with the first end that applies, tagged by `kind`: `stopped` by `stop()`; +`aborted` when its acquisition `Context` is aborted; `ended` when the owner +of what it watches ends (the Session closed, or a task document's task became +terminal); `listener_error` when its listener threw or rejected; `failed` when +the Session failed. Durable's failures are final, so a Durable watch only gets +`value` calls; Chord's `failed` call, for a source that recovers, never comes. -Watch acquisition never creates; absent lookup returns `undefined`. Successful -acquisition binds one concrete incarnation. Retirement is an explicit event of -that incarnation, never inferred from a `null` value: it delivers a JSON `null` -root replacement `[["r", null]]` as the last frame, then ends the watch as -`retired`. A later recreation requires acquiring a new watch. +Watch acquisition never creates; absent lookup returns `undefined`. A document +watch follows its address (kind, scope, and key) from the incarnation found at +acquisition. A retirement does not end it: the next call has `retired` set, +with the last value the watch was given and no ops (Durable has no value for an +absent document; `null` never appears). A commit that changes a document and +retires it publishes the retirement only, so that call carries the value from +before the commit, not the content stored with the retirement. A document created at the address later arrives as a +root replacement `[["r", value]]` of the new incarnation, folded with what is +pending, so one call may report a retirement and carry the next incarnation: + +```text +commit: retireDoc(StateDoc); doc(StateDoc).value = 10 +call: { kind: "value", value: { value: 10, ... }, ops: [["r", { value: 10, ... }]], retired: true } +``` + +Task documents retire when their task becomes terminal and are never created +again, so a task document's watch gets the retirement of that commit as its last +call and then ends `ended`, once that call returned. Until then Session close or +failure still ends it at once, dropping the retirement if it was not delivered +yet, as for a watch never started. A watch of a session or conversation document that +is retired for good stays until it is stopped, its context is aborted, or the +Session ends; it pins nothing while its address is absent. Session close ends every watch at its seal: the watch keeps its last value and receives no frame from a commit that settles during close. Termination drops every reference the watch holds into its source: its listener, its subscriptions and the closures that detach, report, and replace, and its -cancellation hook. A stopped watch that is still referenced keeps only its last +abort hook. A stopped watch that is still referenced keeps only its last value alive, never the Session or the listener; a callback already running keeps what it needs until it settles. Watches retain immutable revisions independently of the loaded tracker cache. Reopen creates a new watch lifetime. @@ -4678,15 +5468,77 @@ acquisition, and every later commit is buffered until `start()`. This prevents a value from being paired with operations based on a newer unseen revision. +**Observers.** Watches are for application code. Adapters that must change in +the same step as Durable's memory, such as a Chord source over a Durable view, +attach a synchronous observer instead; every watch is built on one: + +```ts +/** The shapes of Chord's watch: `advance` gets a `WatchCall` (always a `value` call), `end` a `WatchEnd`. */ +type Observer = { + advance(call: WatchCall, context: Context): void; + end(end: WatchEnd): void; +}; +type Observation = { readonly value: T; detach(): void }; + +interface Session { + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + // ... and the other five scopes (section 3.3), the observer before the context +} +interface Harness { + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; +} +``` + +Attaching works like acquiring a watch, atomically on the Session line: the +observation's `value` includes every commit before it, and `advance` gets every +later change that a watch would get, unfolded, as memory adopts it: for a +stored commit at its publication (section 4), for a deferred one when +memory adopts it, before it is stored (section 4.1). A commit gives a view or +the graph at most one call, and a document observer one call per change at its +address: a commit that retires the document and creates it again gives two, the +retirement and then the new incarnation's `[["r", value]]`. The task graph also +advances without a commit, when a registry change or a failed migration changes +a task's `blocked` reason (section 9.4): that call runs on the Session line, +outside any commit's publication, with a context of its own. A document observer follows its +address as a document watch does, and `observeDoc()` gives `undefined` for an +absent one. `end` comes once: when the Session closes (`failed` with the error +"session closed": an adapter following the Session, such as Chord's `self.follow`, +fails and follows the next Session; watches end `ended`) or fails (`failed` with +the error), or, for a task document, right after the retirement change of the +commit that makes its task terminal, at that commit's publication (`ended`). +`detach()` is idempotent, and nothing follows it, not even the rest of the +commit underway; a detached observer is referenced by nothing of the Session. +`context` governs only the attachment: cancelling it while attaching rejects +the attachment and leaves nothing attached; cancelling it afterwards does +nothing, and `detach()` ends the observation. + +The attachment completes on the line, and the caller's continuation runs +later, so a commit adopted in between reaches `advance` before the promise of +`observeXXX()` resolves. Each change carries the whole new value, and its +operations apply to the value before it, starting from `observation.value`: an +adapter that gets a change before its observation records it, and applies it +once it has the observation, or takes `call.value`. + +`advance` runs inside the storage's commit, so an observer records the change +and returns: no awaits, no Session calls, no throws, and quick, as its time is +part of every commit; so does `end` when a task ends. One that throws anyway, or +returns a promise that rejects, is reported as `listener_error`, detached, and ended +`listener_error` with the error; the commit and +the other observers are not affected. The call object is plain data, shared by +every observer of a document or the task graph and by the watches built on it, +and must not be mutated; each observer of a conversation view gets its own, for +its own window (section 9.3), sharing the document values. + ### 9.2 `watchDoc` Tasks, hooks, and tools may observe any existing document for which their code has a token and owner/key. There is no additional subtree permission system inside trusted Session code. -On the Session line, `watchDoc()` resolves one existing concrete incarnation, +On the Session line, `watchDoc()` resolves the incarnation at the address, captures its current immutable tracker revision in O(1), and registers for every -later committed revision and its operation batch. It returns `undefined` when +later committed revision at that address and its operation batch. It returns `undefined` when absent. Before `start()`, `watch.value` remains the acquisition revision while later frames buffer. @@ -4695,8 +5547,8 @@ const watch = await api.watchDoc(JobOutputDoc, producerTaskId, context); if (watch === undefined) return; try { await initializeConsumer(watch.value, context); - watch.start(async (value, ops, deliveryContext) => { - await consume(value, ops, deliveryContext); + watch.start(async (call, deliveryContext) => { + if (call.kind === "value") await consume(call.value, call.ops, deliveryContext); }); } catch (error) { await watch.stop(); @@ -4730,10 +5582,12 @@ the trusted immutability contract. Consumers must not mutate them or retained descendants. The selected commit Context's values are preserved without inheriting the producer's cancellation. A callback owns its own asynchronous work. -A watch remains bound to its original incarnation. Retirement drops what is -pending and queues the terminal `[["r", null]]` with value `null`. After that -callback settles, the watch closes as `retired` and never -follows a replacement incarnation. +A watch follows its address (section 9.1). A retirement folds into the +pending call like any commit and sets its `retired` flag; its value is the last +one and its operations are those still pending. A later incarnation arrives as +`[["r", value]]`. A retirement that ends a task document's task is that watch's +last call: it is delivered, then the watch ends `ended` once the call returned. While its address is +absent a watch pins nothing; a new incarnation is pinned while it is watched. `stop()` is idempotent. It synchronously unregisters, discards pending frames, prevents another callback from starting, and returns the common terminal promise. @@ -4746,222 +5600,192 @@ rejection is observed. ### 9.3 Conversation view -The public view is a fixed structural mount of the conversation's active +The public view is a fixed structural mount of a window over the conversation's transcript and its built-in documents: ```ts type ConversationView = { readonly conversation: ConversationRecord; - /** Raw active entries, as `ContextView.entries` (section 2.1): the head marker, then the non-head entries from its head. */ + /** + * A window over the transcript, its fork-aware history (`Conversation.entries()`), in entry ID order. Head markers + * such as compaction summaries sit where they were written; their `head` tells where the entries they summarize end. + */ readonly entries: readonly EntryRecord[]; + /** ID of the entry just before the window; absent when the window starts the transcript. */ + readonly before?: EntryId; /** `pi.agent`, `pi.live`, `pi.inbox`, `pi.provider`, and `pi.usage`, keyed by kind; absent documents are absent. */ readonly docs: Readonly>; }; + +type ConversationWindow = { + /** + * Where the first value's window starts, inclusive, such as right after the newest entry a client has + * (`ConversationTranscript.resume`). It selects only that first value. + */ + readonly from?: EntryId; + /** Default 50; an integer, at least 1. */ + readonly min?: number; + /** Default `2 * min`; an integer, at least `min`. */ + readonly max?: number; +}; + +interface Conversation { + watch(context: Context, window?: ConversationWindow): Promise; +} +interface Harness { + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; +} ``` These kinds and their fields are public protocol. Third-party documents are -exposed through their own Chord objects, not mounted. +exposed through their own Chord objects, not mounted. Everything a renderer +draws is state in this value: the transcript in `entries`, the run and its +presentation in `pi.live` (`run`, `generation`, `tools`, `nestedTools`, +`compactions`, `compactionFailure`, `runFailure`, section 8.2), the queue in +`pi.inbox`, the agent in `pi.agent`, spend in `pi.usage`. A run that answered +and waits for its after-run tasks shows as `run` with `run.afterRun` and no +`generation`. A renderer renders the newest value; it never assembles state +from a stream of events. + +**The window.** Each watch and observer has its own window. It starts with the +newest `min` entries and the ID of the entry before them in `before`. With +`from`, it starts at `from` instead, when at most `max` entries are at or after +it; then `before` is the newest entry before `from`, and a window that resumes +right after the newest entry starts empty. Otherwise it starts at the newest +`min`, and `before` shows the client the entries it missed. `from` only picks +that first value: every entry committed after the attachment is appended, +whatever its ID, so a `from` past the newest entry, such as one a client +computed against another replica, gives an empty window whose `before` is the +newest entry, and then every entry that follows. The window has a stable start, +so new entries are appends. When it would hold more than `max` +entries, its oldest leave in one splice down to `min`, and `before` moves to +the last one that left: + +```text +min 3, max 5; the window holds 4..7, before 3 +commit appends 8: ["p", ["entries"], 4, 0, [8]] window 4..8 +commit appends 9: ["p", ["entries"], 5, 0, [9]], ["p", ["entries"], 0, 3, []], ["s", ["before"], 6] + window 7..9, before 6 +``` + +A commit that appends more than `max` entries at once keeps its newest `min`. +A first value, and an overflow replacement (section 9.2), therefore hold at +most `max` entries, never the whole transcript. Entry IDs are ordered but not +contiguous: a client never counts entries or infers ranges from IDs. `before` +tells whether a window continues what the client has, and +`Conversation.entries()` with `minEntryId`/`maxEntryId` and its cursor fetches +what lies in between; resuming from right after the newest entry and +`maxEntryId: oldest - 1` are exclusive bounds, the only arithmetic on IDs a +client does, and `ConversationTranscript` does both. A window with +bounds that are not integers, or `max` below `min`, rejects with a +`RangeError`. + +Head markers stay in place: a compaction summary, a reset, or a handoff is an +appended entry like any other. The view does not derive the active +transcript; `Conversation.context()` does (section 2.1). The Harness keeps at most one mount per conversation. The first `watch()` or -event attachment builds its revision on the Session line from the -committed active transcript and documents; the mount is dropped when its last +observer builds its documents on the Session line, and each attachment reads +its window there with bounded scans; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's -`subscribeCommits()` publications, like `watchDoc()`: every publication is -already durable, so the view shows only committed state. One publication that -touches the conversation yields one Chord batch: +commits as memory adopts them, for a stored commit at its publication +(section 4) and for a held one when memory adopts it (section 4.1), like +`watchDoc()`: it shows committed state, including deferred commits before they +are stored (section 4.1). One publication that touches the conversation yields +one Chord batch per observer: the document operations, applied once and +shared, then that observer's entry splices: ```text document op ["s", ["generation", "message"], value] -> view op ["s", ["docs", "pi.live", "generation", "message"], value] ``` -An appended entry without a head is a splice at the end of `entries`. An -appended head marker `H` makes the entries `H` followed by the current non-head -entries at or after `H.head`: the kept entries stay, and splices remove the -others and insert `H` at the front. Every kept entry is already mounted because -Harness writers never target a head before the current range: resets and -handoffs head themselves, compaction summaries target a kept entry of the -current range, and head writes that reach further back are stale (section 6). A raw head write that targets further back shows only the -mounted entries (section 12). Document creation and retirement set and delete the `docs` key. Parent entries through `parent.at` are immutable, so a child's mount follows only its own conversation. A publication that touches nothing mounted creates no revision; a redundant nonempty batch remains a real publication. Mounted document revisions -are structurally shared under the trusted immutability contract. The mount -performs no semantic projection and owns no persistence. +are structurally shared under the trusted immutability contract, and an entry +that stays in a window keeps its identity across revisions. The cost of a +commit is its operations plus, per observer that gets entries, one copy of its +window, at most `max` entries. The mount performs no semantic projection and +owns no persistence. `Conversation.watch()` exposes the mount through the serialized watch of -section 9.2, with folded pending commits and a full-value overflow replacement. +section 9.2, with folded pending commits and a full-value overflow replacement; +`Harness.observeConversation()` attaches a synchronous observer of the same mount +(section 9.1). A view never retires, so its changes never set `retired`. -### 9.4 Agent events (experimental) - -`watchEvents(harness, conversationId, context): Promise`, -exported from the package root, is an experimental adapter that -translates one conversation's committed publications into agent events shaped -like the coding agent's `AgentSessionEvent`s. It owns no tracker or persistence, -covers exactly one conversation (not its owned subtree), and emits an event only -after the commit that makes it true. Its protocol may change without notice. +**A client's transcript.** A window only ever adds to what a client has seen: +entries never change once committed, and those that leave the window stay with +the client. `ConversationTranscript` keeps them, as a layer over any view, +wherever it comes from (a watch, or a replica of it on another host): ```ts -/** - * Which call a tool event is about. `toolCallId` is the provider's ID for a model-issued call and - * `/` for a nested one; consumers match it and never parse it. `taskId` is the call's tool task, - * absent for a call that never got one. A nested call also names its caller by call ID and by tool task. - */ -type ToolEventCall = { - toolCallId: string; - toolName: string; - taskId?: TaskId; - parentToolCallId?: string; - parentTaskId?: TaskId; -}; +type TranscriptItem = + | { readonly kind: "entry"; readonly entry: EntryRecord } + /** Entries strictly between `after` and `upTo` are missing: render "load more". */ + | { readonly kind: "gap"; readonly after: EntryId; readonly upTo: EntryId } + /** The transcript continues above: render "load earlier". */ + | { readonly kind: "earlier" }; -type AgentEvent = - | { - type: "snapshot"; - entries: readonly EntryRecord[]; - run?: { inputs: readonly SubmissionId[] }; - /** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */ - generation?: { attempt: number; message?: AssistantMessage; retry?: { at: number; error: string }; deferred?: { pollAt: number } }; - tools: readonly ToolSlot[]; - /** `pi.live.nestedTools` (section 8.2). */ - nestedTools: readonly NestedToolSlot[]; - /** `pi.live.compactions` (section 8.2). */ - compactions: readonly CompactionStatus[]; - /** `pi.live.compactionFailure` (section 8.2). */ - compactionFailure?: CompactionFailure; - inbox: readonly { id: SubmissionId; mode: InboxItem["mode"] }[]; - /** `pi.agent`; `{}` when absent. */ - agent: AgentState; - usage: UsageState; - } - | { type: "run_start"; inputs: readonly SubmissionId[] } - | { type: "run_end"; inputs: readonly SubmissionId[] } - | { type: "turn_start" } - | { type: "turn_end" } - | { type: "message_start"; message: Message } - | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } - | { type: "message_end"; entry: EntryRecord } - | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) - | ({ - type: "tool_execution_update"; - /** A front trim and then an append of the retained window, or its replacement. */ - output?: { trimStart?: number; append?: string } | { set: string }; - details?: JsonValue; - diagnostics?: readonly ToolDiagnostic[]; - } & ToolEventCall) - /** A model-issued call ends with its `entry`, a nested call with its `result`; see below for when neither. */ - | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) - | { type: "inbox_update"; items: readonly { id: SubmissionId; mode: InboxItem["mode"] }[] } - | { type: "submission"; record: SubmissionRecord } - | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } - | { type: "auto_retry_end"; attempt: number } - | { type: "deferred_poll"; pollAt: number } - | { type: "entry_appended"; entry: EntryRecord } - | { type: "agent_changed"; agent: AgentState } - | { type: "usage_changed"; usage: UsageState } - | { type: "task_failed"; taskId: TaskId; kind: string; message: string } - | { type: "compaction_start"; taskId: TaskId; reason: CompactionReason; blocking: boolean } - /** The task's receipt tells whether it produced a summary; the summary entry has its own events. */ - | { type: "compaction_end"; taskId: TaskId; reason: CompactionReason; error?: string }; +/** The conversation view as a renderer sees it: `items` instead of the window. */ +type TranscriptView = Omit & { readonly items: readonly TranscriptItem[] }; -/** One change to the in-flight assistant message, relative to that message. */ -type MessageChange = - | { type: "text_start" | "thinking_start" | "toolcall_start"; contentIndex: number; block: AssistantMessage["content"][number] } - | { type: "text_delta" | "thinking_delta"; contentIndex: number; delta: string } - | { type: "toolcall_delta"; contentIndex: number; path: readonly (string | number)[]; delta: string } - | { type: "block"; contentIndex: number; block: AssistantMessage["content"][number] } - | { type: "message"; message: AssistantMessage }; +type EntryLoader = (query: Omit, limit: number, context: Context) => Promise>; -interface AgentEventStream { - /** The `snapshot` event at attachment. */ - readonly snapshot: Extract; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; - stop(): Promise; - readonly closed: Promise; +class ConversationTranscript { + constructor(load: EntryLoader, options?: { readonly pageSize?: number }); // default 50 + /** Call with every value of the view. */ + merge(view: ConversationView): TranscriptView; + /** The page before the oldest entry held. */ + loadEarlier(context: Context): Promise; + /** The next page of a gap, from its start. */ + fillGap(gap: Extract, context: Context): Promise; + /** The ID the transcript reaches: its newest entry, or a later empty window's `before`. */ + readonly newest: EntryId | undefined; + /** `{ from }` right after `newest`, `{}` for an empty transcript, undefined before the first merge. */ + readonly resume: ConversationWindow | undefined; } ``` -Every event is relative to the `snapshot` and the events before it, never to -Durable's document layout. `snapshot` replaces everything a consumer holds; every -other event applies on top of it. Attachment captures the snapshot and -registers for later publications atomically on the Session line. +```ts +const transcript = new ConversationTranscript((query, limit, ctx) => conversation.entries(query, limit, undefined, ctx)); +const watch = await conversation.watch(context); +render(transcript.merge(watch.value)); +watch.start(async (call) => { + if (call.kind === "value") render(transcript.merge(call.value)); +}); +// A marker scrolled into view: +render(await transcript.loadEarlier(context)); +// Reconnect after a drop: +const resumed = await conversation.watch(context, transcript.resume); +``` -Events derive from committed changes: +A window whose `before` lies past the newest entry the client has does not +continue it: after a stall, an overflow, or a reconnect, the entries in between +become a gap. While the client keeps up there is never one: the window only +appends, and its splice removes entries the client already holds. +`fillGap()` loads `pageSize` entries from the gap's start and shrinks it, or +removes it once nothing is missing; `loadEarlier()` loads the page before the +oldest entry held, and the `earlier` item disappears once a page reaches the +transcript's start. A window, or a page, that covers part of a gap fills that +part. Items are in ID order: `earlier` first, then entries, each gap before +the entry after it. -- `run_start`/`run_end`: `pi.live.run` appears, is removed, or is replaced by a - successor run whose first input differs. Steers joining the current run and - handovers between its tasks are not run events. The inputs' outcomes are - `submission` events. -- `turn_start`: a `pi.generation` task is created. `turn_end`: a generation's - outcome is committed, when it holds `completing` or becomes terminal, - whichever comes first, so a successor created at hold starts after it. -- `message_start`: the first committed partial of an attempt, or, for a message - entry without a committed partial, the entry itself; `message_end` follows for - every entry with `model` messages. Generation commits a partial only once it has - content, and every built-in path that clears one appends its entry, so each - started message ends with its entry. Entries without messages are - `entry_appended`. -- `message_update`: the Chord operations on the committed partial, translated: - an insertion into `content` starts blocks, an append to a block's `text` or - `thinking` is a delta, an append to a string inside a tool call's `arguments` - is a `toolcall_delta` with the path relative to `arguments`, any other change - inside a block sends that `block`, which already holds the batch's later - changes to it, and any other change to `content` sends the whole `message`. - Each update carries the partial's current `usage`, as the coding agent's JSON - mode does; a change of only `usage` sends no changes. Only deltas travel, so throttled - partials cause no write amplification on the wire. -- `tool_execution_start`: a slot becomes `running`; `args` come from the tool - task's intent checkpoint in the same commit. A slot that becomes `done` - without running (a call not offered, invalid arguments, a block, a fault - before intent) gets only `tool_execution_end`. `tool_execution_update`: output - appends, front trims, and replacements of the slot's retained window, and the - slot's current `details` and `diagnostics` when they change; removed ones, as - when a safe replay restarts the tool, send `null` and `[]`. - `tool_execution_end`: the slot becomes `done`, with its `pi.tool-result` - entry, which carries the diagnostics, or without one after a fault or - orphan. An unfinished slot that disappears because its run ended ends with the - result entry appended in the same commit, as for the unstarted calls of an - aborted round, directly before its `message_start`, or without an entry. - Nested slots produce the same events with `parentToolCallId`; a nested call - ends with its `result`, taken from the `NestedCallDoc` value its terminal - commit published, or without one after a fault or orphan or when its slot - disappears unfinished. Snapshots show done nested slots with their summary, - not their result; a late subscriber reads a live caller's results from its - `NestedCallDoc`, keyed by call key (the slot's `callId` after `/`). -- `inbox_update`, `agent_changed`, `usage_changed`: the document changed; a - retired one reads as its initial value, as in a snapshot. Registry installs - and settings changes make no commit and emit nothing; a UI showing resolved - tools or a resolved model resolves again through `Conversation.agent()`. -- `auto_retry_start`/`auto_retry_end`, `deferred_poll`: `pi.live.generation` - gains or drops `retry`, or gains `deferred` or moves its `pollAt`. -- `task_failed`: a task of the conversation settles `faulted` or `orphaned`. -- `compaction_start`/`compaction_end`: a status appears in or disappears from - `pi.live.compactions`. `compaction_end` carries `error` when the same commit - recorded the task in `pi.live.compactionFailure`. A retry backoff shows only - in the snapshot's `compactions`. A queued summary is placed later with its own - `message_start`/`message_end` and `submission` events. +`items` is a plain array. A merge costs O(items), never re-sorts or +re-fetches, and one that adds nothing returns the previous `items` array; with +the same `docs` and `conversation` it returns the previous view itself. +Concurrent `loadEarlier()` calls, and concurrent `fillGap()` calls for one gap, +share one load, also when one passes a gap descriptor that has shrunk since; a +load that rejects, or whose loader throws, is not kept, so the next call loads +again. A page without a next cursor, or without items, ends the range: the gap +closes, or the transcript's start is reached. An empty window adds nothing. +`loadEarlier()` before the first merge rejects; after the start is reached, and +`fillGap()` for a gap that is gone, return the current view. +Keeping the scroll position when items are inserted above is the renderer's +job. -One commit produces one batch, in this order: `tool_execution_start`, -`message_start` of a first partial, `message_update`, `tool_execution_update`, -and retry/deferred events; then entries in append order with their -`message_start`/`message_end` or `entry_appended`, where a tool result's -`tool_execution_end` directly precedes its `message_start`, as in the coding -agent, and the `tool_execution_end` of nested calls, children first, precede -the entries; then the `tool_execution_end` of tools ending without an entry; -then `compaction_end`, `task_failed`, `turn_end`, `run_end`; then `submission` -events in ID order; then `inbox_update`, `agent_changed`, and `usage_changed`; -and last `compaction_start`, `run_start`, and `turn_start`. A stream buffers at most 100 undelivered -batches; adding another replaces every undelivered batch with one `snapshot` of -the newest committed state. The stream therefore converges but does not promise -every transition; `Submission.wait()` reports exact outcomes. A reconnecting -consumer attaches again and starts from its `snapshot`; nothing is replayed. -Transport backpressure and disconnect policy belong to the consumer. - -Print mode awaits its own input `Submission` and prints its answer. A TUI -renders `ConversationView`; events may drive transient animation. - -### 9.5 Task graph view +### 9.4 Task graph view The task graph view shows every live task of the Session as one structural value, for UIs and debugging: what runs, what waits for what, and which @@ -4994,7 +5818,7 @@ type TaskGraphNode = { /** Conversations this task owns, in ID order. */ readonly conversations: readonly ConversationId[]; /** Why it cannot run here, when it cannot (section 5.4); follows registry changes. */ - readonly blocked?: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly blocked?: "missing_task" | "task_too_old" | "migration_failed"; }; type TaskGraph = { @@ -5006,6 +5830,7 @@ type TaskGraphWatch = WatchHandle; interface Harness { watchTaskGraph(context: Context): Promise; + observeTaskGraph(observer: Observer, context: Context): Promise>; } ``` @@ -5032,20 +5857,27 @@ A UI that places it under its parent reads the conversation's owner edge from The Harness keeps at most one graph mount. The first `watchTaskGraph()` builds its revision on the Session line from the committed live tasks and the conversations they own, one owner scan per live task, and holds the line while it does; the mount is dropped when its last observer -detaches. It advances from `subscribeCommits()` publications like the -conversation view: one publication that changes a node yields one Chord batch +detaches. It advances from the Session's stored commits as memory adopts them, +at their publication (section 4), like the conversation view: a commit that +touches a conversation view and the graph changes both in that one step. A +`blocked` reason that changes without a commit (a registry change, a failed +migration) is derived again in a job on the Session line and published there, +outside any commit and its publication, with the background context. One +publication that changes a node yields one Chord batch that sets or deletes `tasks[id]`, or sets a node's `conversations`. A publication that changes no node creates no revision. Task and conversation records are immutable values; a node shares nothing with them. `watchTaskGraph()` returns the mount through the serialized watch of section -9.2. It reads only: it never enables scheduling. Close ends it at its +9.2, and `observeTaskGraph()` attaches a synchronous observer of it (section +9.1). Both read only: they never enable scheduling. Close ends them at its seal like every other watch (section 2.2). ```ts const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { - for (const node of Object.values(value.tasks)) { +graph.start(async (call) => { + if (call.kind !== "value") return; + for (const node of Object.values(call.value.tasks)) { const where = node.owner === undefined ? `conversation ${node.conversationId}` : `task ${node.owner}`; render(`${node.id} ${node.kind} ${node.state.status} under ${where}`); } @@ -5152,11 +5984,27 @@ type StorageWrite = */ interface Storage { /** - * Persist the batch atomically. Once it is durable, call `adopt(seq)` exactly once, synchronously and before this - * storage's next commit starts, then resolve with `seq`; a commit that fails before the batch is durable never calls - * it. `adopt` must not throw. + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Every batch has at least one write: a commit without + * writes never reaches Storage, so the Session never sends an empty batch. Stops at the first batch Durable's own + * checks refuse (global IDs, document actions, copy resolution): the batches before it are durable, it and the ones + * after are not. Calls `receive(index, seq)` once per durable batch, in order, synchronously, after all of them are + * durable and before the next commit starts, and resolves with the number of durable batches and the refusal, if + * any. Rejects for an unknown outcome (connection lost, statement or commit failed); `receive` calls made before the + * rejection still stand. A receipt is not publication: the Session publishes later; it calls no Storage method and + * never throws. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise<{ readonly stored: number; readonly refused?: unknown }>; + /** + * Optional. Run `publish`, the Session's publication of one commit (cache write-through, adoption, observers), + * inside this storage's own scope, such as a store host's batch. Call it exactly once, now or soon after; the + * Session publishes the next commit only once it ran. + */ + publishing?(publish: () => void, context: Context): void; /** * Allocate from the one global numeric namespace, above every ID minted since this storage was opened and every ID * stored, also across reopen: scans order by ID as creation order, and the Harness relies on it. The generic brand @@ -5187,6 +6035,19 @@ interface Storage { } ``` +Every backend implements `commit()` natively. It checks each batch (global +IDs, document actions, copy resolution) before that batch writes anything, so +a refusal leaves the earlier batches stored and nothing of its own; no +savepoints are needed, which Durable Objects' SQL does not allow. SQLite runs +one transaction per call. IndexedDB runs one `readwrite` transaction per call: +on a refusal it stops issuing writes and the transaction commits the prefix; +an error of IndexedDB itself aborts it, and the call rejects with nothing +stored. The store backend stores a call as one series of the store's commit +queue (`ChordStore.commitSeries`): one backend transaction, one seq and one +outbox commit per batch, the store's metadata written once. JSONL appends every batch's sidecar records and then +all their markers in one write (a torn append recovers a shorter complete +prefix), and memory applies them one by one. + Every error a method throws is final for the Session (section 10.1), so a backend retries its own transient failures. A read throws `StorageRequestError` for an invalid request, with no durable effect; from `commit()` it is fatal like any @@ -5257,42 +6118,6 @@ a stored version boundary; a version transition must be a base. The semantic conformance suite covers memory, SQLite, and JSONL. -Every built-in backend is also a `ReplayableStorage`: - -```ts -type LogWrite = Exclude; -type LogRecord = { readonly seq: Seq; readonly writes: readonly LogWrite[] }; - -interface ReplayableStorage extends Storage { - head(context: Context): Promise; - replay(record: LogRecord, context: Context): Promise; - log(after: Seq | undefined, context: Context): AsyncIterable; -} -``` - -`replay()` is `commit()` at an explicit sequence above the head; later commits -continue after it. A log never contains a copy: a copy is stored, and logged, as -the creation it became, so replay never reads a source. - -`log(after)` rebuilds the changes after `after` from stored state, in sequence -order: replayed into a backend holding this backend's state at `after`, they -produce its current state, and every read answers the same, historical document -reads included. Without `after` it rebuilds everything for an empty backend. -Every record carries the stamp of the commit that last wrote it: a conversation -its creation, a task or submission its last write, an entry its commit, a -document its creation and retirement, a revision its commit. The log groups the -records stamped after `after` by stamp. A document created after `after` is -created with the base stored at its creation, or, when a newer base reclaimed -that one, with the placeholder base `{ version: 1, value: {} }`, which no read -reaches; its retained revisions follow. So superseded states are absent: each -task and submission appears once, and a current-only document only with what -followed its newest base. Storage checks no references between records, so a -task-scoped document may precede the task's record. The last record is at the -head, empty when nothing else is there; nothing follows a head at `after`, and -an `after` past the head rejects. Records stored before stamps existed carry -stamp 0 and join the first record of a complete log. A commit while a log is -consumed makes it reject. - ### 10.1 Failures One rule per kind of error, so a host knows what happens without reading code. @@ -5310,15 +6135,14 @@ may not have committed, so a host that resubmits after reopening reuses its and every wait that starts later rejects with `SessionFailed`, whose `cause` is the first error; so does every Storage call still underway when it returns, whatever it returned. -- Watches and event streams end with `{ reason: "failed", error }` and keep - their last value. +- Watches end with `{ kind: "failed", error }` and keep their last value. - Running task invocations get their abort signal. `HarnessOptions.onReport` gets the error once, after the Session is sealed, so a report handler that calls back finds it failed. -- The Session closes itself. `closed` settles `{ reason: "failed", error }` once +- The Session closes itself. `closed` settles `{ kind: "failed", error }` once that close has run, also when the backend could not close cleanly; after a plain `close()` it settles - `{ reason: "closed" }`. Closing the backend waits for every Storage call + `{ kind: "closed" }`. Closing the backend waits for every Storage call underway, refusing new ones, and for running task code to return: code that ignores its abort signal keeps the backend open and `closed` pending. - A failing `Storage.close()` fails the Session too, unless it already failed; @@ -5335,8 +6159,35 @@ exempt: once a batch is admitted, whether it committed is unknown. A commit callback that catches a failed read cannot commit afterwards: the Session checks its health after the callback. Adopting a committed batch in memory that fails also fails the Session, since memory would be behind storage; so does a -`Storage.commit()` that resolves without adopting, or rejects after it adopted -(the adopted changes are not rolled back). +storage call that resolves without a receipt for a batch it reports stored. + +Each commit of one storage call (group commit, section 4) gets its own outcome, +in line order: + +- A commit Storage stored (a receipt) is published, then resolves. +- A call that stops at a refusal of commit `k`: the commits before `k` are + published and resolve; `k` rejects with the refusal; the commits after `k`, + and every commit admitted later, reject with `SessionFailed`, unpublished; + the Session fails with the refusal once the commits before `k` are + published. +- A call that rejects: the commits it stored are durable, so they are not + rolled back: they are published, `observeCommits()` observers get them and + the waits they settle resolve, and then they reject with the storage error, + as do the commits it did not store; the Session fails with that error. An own + observer that threw on such a commit is reported as a `listener_error`. +- A commit whose adoption or own observer throws: it rejects with the adoption + error, or stands and resolves for an observer's throw, and the Session fails; + the commits after it in the call are durable but not published, and reject + with `SessionFailed`. Reopening recovers them. +- A call that settles after another error already failed the Session, or a + commit of a call whose publication comes after another error failed it: what + it stored is durable but not published, and rejects with `SessionFailed` + whose cause is the first error, whatever the call returned. Reopening + recovers it; an `observeCommits()` consumer, such as a replica, misses it + until then. + +A job whose outcome comes after the one a failure is gets `SessionFailed`, even +when its callback ran before the failure; a job before it keeps its own outcome. **The Session's own bookkeeping fails it too.** A throw in a commit the scheduler makes for itself (reconciliation, reservation, a fault or terminal @@ -5346,7 +6197,9 @@ storage failure, a host callback such as a `RegistryReader` whose `snapshot()` throws, or a bug, and running the commit again fixes none of them. A throw in a commit listener of the Session's own components (scheduler, submissions, views, task graph, document observers) fails it as well, since their memory would fall -behind storage; the commit itself stands. +behind storage; the commit itself stands. Those listeners run at the commit's +publication (section 4), so the Session fails after the other observers and the +`observeCommits()` observers got the commit. **User code fails only its unit.** A task phase that throws ends that task `faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that @@ -5355,32 +6208,41 @@ throw are reported and skipped, except `beforeTool`, whose throw blocks the call a commit callback, `conversationCreated`, `init`, or a document's `initial`, `migrate`, or `checkpointWhen` that throws rolls its commit back and rejects it. Callbacks that must not fail their caller are contained: a host -`subscribeCommits()` or `subscribeClose()` listener, a watch listener, and a -view observer that throws, or returns a +`observeCommits()` observer or `subscribeClose()` listener, a watch listener, and a +view or document observer that throws, or returns a promise that rejects, are reported, and the others still run; a watch whose -listener throws ends with `listener_error`. A throwing or rejecting `onReport` is -dropped, since nothing is left to report it to. A throwing `now()` is reported -once, and `Date.now` serves instead. +listener throws, and an observer that throws, end `listener_error`. A throwing or rejecting `onReport` is +dropped, since nothing is left to report it to, and so is a report's own +metric or event that throws. A throwing `now()` is reported once, and +`Date.now` serves instead. A telemetry adapter that throws on a commit's events +or on the active-task metric is reported as `host_error` with callback +`telemetry` (a plain Session drops it); the commit resolves, and memory and the +views keep it. -**Every report says what happened.** `onReport(error, info)` gets a -`ReportInfo` whose `type` is one of: +**Every report says what happened.** `onReport(report)` gets one `Report`: the +`error`, and the fields of a `ReportInfo` whose `kind` is one of: -- `session_failed`: the error that failed the Session (a Storage error, often a +- `storage_failed`: the error that failed the Session (a Storage error, often a platform reset, or the Session's own bookkeeping); - `task_faulted`: a task ended `faulted`, because its phase threw (reason - `threw`) or its abort handler was cut off `restartLimit` times (reason - `restart_limit`); -- `task_parked`: the restart limit parked a task (section 5.4); + `threw`), or it or its abort handler was cut off `restartLimit` times (reason + `restart_limit`, section 5.4); +- `task_interrupted`: crashes ended a task: a built-in task ended itself at the + restart limit (reason `restart_limit`), or a generation's run ended because + the host kept stopping while the model answered (reason `interrupted`), with + the task's `restarts` (section 5.4); - `migration_failed`: a task's migration threw, and the task is blocked; - `task_outdated`: a running task keeps its old definition, which is missing - (`missing_task`) or cannot take the task (`incompatible_task`); + (`missing_task`), older than the task (`task_too_old`), or newer without a + `migrate` (`migration_failed`); - `task_error`: a task's own `runtime.report()`, or a failure running it; - `extension_error`: hooks, wrappers, system prompt sections, and a tool's invalid structured output, with the task that ran them, or for a wrapper outside a task the conversation whose agent was resolved; - `host_error`: a host callback, by its `HarnessOptions` name: `now`, `settings`, `registry`, `env`, `models` for provider calls such as cancelling - a deferred response, `setTimer`, or `onWake`; with the task that called it, + a deferred response, `setTimer`, `onWake`, or `telemetry` for a telemetry + adapter that threw while a commit was published; with the task that called it, if one did; - `listener_error`: a commit, close, watch, view, or document listener. @@ -5404,7 +6266,8 @@ after a committed base or retirement. ### 11.2 SQLite -One SQL transaction is one Session commit. SQLite stores: +One SQL transaction is one storage call: the batches of a `commit()` call, +each checked before it writes. SQLite stores: - conversation, entry, task, and submission records; - document records; @@ -5442,7 +6305,9 @@ Publication protocol: 2. Append one complete main marker listing those records. 3. Publish in memory only after the marker write succeeds. -Every commit uses this protocol; there is no standalone-sidecar fast path. +Every commit uses this protocol; there is no standalone-sidecar fast path. A +`commit()` call appends every batch's sidecar records first, then all its +markers in one write of `main.jsonl`; reads wait while it appends. JSONL creation accepts an `fsync` option that defaults to `false`. Without `fsync`, it guarantees ordinary process-crash consistency, not survival of power, host, kernel, or filesystem failure. With `fsync: true`, the backend @@ -5483,6 +6348,12 @@ These are contracts, not invitations to add defensive machinery: unsupported. - **Read after write:** read every required table row before the first table write. Document drafts remain usable afterward; table reads do not. +- **Admission inside a commit:** `submit(tx, draft)` throws `ReadAfterWrite` + after the commit's first table write: one admission per commit, before the + commit's own `appendEntry()`, `createTask()`, or `createSubmission()`. The + admission is provisional until the commit is stored: a later throw in the + callback, or a storage failure, discards it with the rest of the commit, + though its `pi.durable.submit` span already ended with a `result`. - **Writing to a busy conversation:** only the Harness appends to a conversation with an active run. Raw entries appended by `Harness.commit()` or a custom task while a generation prepares its request can misplace its system prompt entries; @@ -5510,8 +6381,8 @@ These are contracts, not invitations to add defensive machinery: messages, can misjudge; the Harness estimate uses entry order (section 8.3). - **Raw head writes into the past:** a head written directly with `tx.appendEntry()` that targets an entry before the conversation's active range changes model - context, but a mounted view keeps only the entries it already holds until the - mount is rebuilt. Use a write submission, whose stale check rejects it. + context in ways a queued write would not. Use a write submission, whose stale + check rejects it. - **Double-counted subagent spend:** a tool that runs an owned conversation must not report that conversation's usage in its result; the child's `pi.usage` already counts it, and subtree sums would count it twice. @@ -5532,6 +6403,29 @@ These are contracts, not invitations to add defensive machinery: background conversation-owned tasks awaited with `waitForTask()`; a non-background one would be aborted by any cascade that reaches its conversation. +- **Hooks in abort handlers:** a tool task's abort handler runs the `afterTool` + chain on its `aborted` result, and abort invocations cannot be aborted. A + slow hook delays the result, and with it `Conversation.abort()` and + `waitForIdle()`; a hanging one holds them until the host restarts, and the + handler's last attempt before the restart limit runs without hooks (section + 8.4). +- **Busy after the answer:** with `afterRun` hooks selected, `Submission.wait()` + resolves at the answer while the conversation stays busy until the after-run + tasks end: `whenBusy: "reject"` and a continue throw `ConversationBusy`, and a + steer, also with `whenIdle: "reject"`, queues and starts the next run. +- **After-run tasks waiting on their own run:** an after-run task that awaits + `waitForIdle()` of its own conversation, or the settlement of a follow-up it + submitted, waits for the run that waits for it, and never resolves. Submit and + let the run's end place it. +- **Missing after-run tasks:** an after-run task whose definition is uninstalled + stays blocked, and so does its run: the conversation is busy until Stop + orphans the task. +- **Fix loops:** a follow-up an after-run task submits starts a run with a fresh + turn budget, which asks `afterRun` again; `maxTurns` does not bound the loop, + the extension does, for example with a counter in its own document. +- **Submissions from after-run tasks:** admit them in the task's own commit with + `submit(tx, draft)`, before its table writes; one submitted outside a commit + should carry a stable `requestId`, as a rerun after a crash submits it again. - **Hook memo names:** hooks share the asking task's memo namespace with the task and other hooks. Prefix memo names. - **Long transactions:** an async commit callback holds the Session mutation @@ -5623,7 +6517,9 @@ These are contracts, not invitations to add defensive machinery: Durable has no: - whole-Session DOM; -- visible-undurable publication; +- visible-undurable publication beyond deferred commits (section 4.1): an + unmarked commit, and every commit that writes anything but existing latest, + initial-fork conversation documents, is stored before anything observes it; - Session-kernel semantic event journal or independently maintained event state; - session-scoped rewindable documents; - automatic checkpoint heuristic; diff --git a/packages/durable/package.json b/packages/durable/package.json index 040b9c287..ebe451a3d 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "description": "Durable conversation, task, and document runtime for Pi", "type": "module", "main": "./dist/index.js", @@ -90,6 +90,12 @@ "import": "./dist/storage/jsonl/node.js", "default": "./dist/storage/jsonl/node.js" }, + "./storage/store": { + "source": "./src/storage/store/index.ts", + "types": "./dist/storage/store/index.d.ts", + "import": "./dist/storage/store/index.js", + "default": "./dist/storage/store/index.js" + }, "./storage/sqlite": { "source": "./src/storage/sqlite/index.ts", "types": "./dist/storage/sqlite/index.d.ts", @@ -123,9 +129,10 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", - "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:workers", + "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:store && npm run test:workers", + "test:store": "vitest --run --config vitest.store.config.ts", "test:workers": "vitest --run --config test-workers/vitest.config.ts", "bench:storage": "vitest bench --config vitest.benchmark.config.ts", "bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts", @@ -137,6 +144,7 @@ "bench:document-cache": "node --conditions=source --experimental-strip-types test/session-document-cache-bench.ts", "bench:document-append": "node --conditions=source --experimental-strip-types test/document-append-bench.ts", "bench:codemode": "node --conditions=source --experimental-strip-types test/codemode-bench.ts", + "bench:conversation-view": "node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts", "prepublishOnly": "npm run clean && npm run build" }, "keywords": [ @@ -156,18 +164,17 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/durable/src/codemode/codemode.ts b/packages/durable/src/codemode/codemode.ts index dba1d1e57..a2e765779 100644 --- a/packages/durable/src/codemode/codemode.ts +++ b/packages/durable/src/codemode/codemode.ts @@ -37,16 +37,16 @@ import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; import { ownedSlice } from "@earendil-works/chord/delta"; import type { TextContent } from "@earendil-works/pi-ai"; -import { - type CodemodeHostCall, - type CodemodeOutcome, - type CodemodeResult, - type CodemodeRun, - type CodemodeSandboxBase, - toCodemodeIdentifier, +import { toCodemodeIdentifier } from "@earendil-works/pi-codemode/declarations"; +import type { + CodemodeHostCall, + CodemodeOutcome, + CodemodeResult, + CodemodeRun, + CodemodeSandboxBase, } from "@earendil-works/pi-codemode/portable"; import { CODEMODE_SOURCE_GRAMMAR, parseCodemodeSource } from "@earendil-works/pi-codemode/source"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { defineDoc } from "../documents.ts"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage } from "../errors.ts"; @@ -210,7 +210,7 @@ export type CodemodeGlobal = { execute(args: unknown, api: ToolExecutionApi, context: Context): Promise; }; -const codemodeSchema = Type.Object({ code: Type.String({ description: "Raw JavaScript source." }) }); +const codemodeSchema = obj({ code: str({ description: "Raw JavaScript source." }) }, { open: true }); export type CodemodeToolInput = Static; @@ -946,14 +946,10 @@ class CodemodeCall { const documents = tools.flatMap((tool) => { const own = namespaceOfTool(tool.name); if (typeof namespace === "string" && (own === undefined || !matchesNamespace(own, namespace))) return []; - const text = [ - tool.name, - tool.name.replaceAll("_", " "), - tool.description, - own?.name ?? "", - own?.description ?? "", - ]; - return [{ name: tool.name, text: text.join(" ") }]; + const namespaceText = + own === undefined ? "" : [own.name, own.description ?? "", own.instructions ?? ""].join(" "); + // `description` is the tool's sample: its description and declaration, with the parameters' names and descriptions. + return [{ name: tool.name, text: tool.description, namespaceText }]; }); return rankDocuments(query, documents, limit).map((name) => { const tool = tools.find((candidate) => candidate.name === name)!; diff --git a/packages/durable/src/codemode/models.ts b/packages/durable/src/codemode/models.ts index ca22e96ea..fcdf63b68 100644 --- a/packages/durable/src/codemode/models.ts +++ b/packages/durable/src/codemode/models.ts @@ -17,7 +17,7 @@ import type { ModelTypeMap, Usage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { obj, unknown } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -227,7 +227,7 @@ function resolveModel( ); } -const ModelCallParameters = Type.Object({ model: Type.Unknown(), context: Type.Unknown() }); +const ModelCallParameters = obj({ model: unknown(), context: unknown() }, { open: true }); /** The result of a model call as the tool returns it: the whole result for the script, usage counted by the Harness. */ function modelResult( @@ -255,7 +255,7 @@ export function createModelTools(): ToolRegistration[] { name: CLASSIFY_TOOL, description: "models.classify() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.classify", "classifier", args.model, api.models); @@ -271,7 +271,7 @@ export function createModelTools(): ToolRegistration[] { name: GENERATE_IMAGES_TOOL, description: "models.generateImages() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.generateImages", "image", args.model, api.models); diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 0d8da86f7..f2d97d12b 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -117,14 +117,17 @@ export interface WatchTarget { } /** - * What changed: + * What changed, tagged by `kind`: * - `paths`: something at or below each path may have changed (a directory path covers its whole subtree). Calls may be * spurious; a change is never missed while the watcher is healthy. * - `overflow`: coverage was uncertain for a while (lost events, reconnect); rescan everything that is watched. - * - `error`: the watcher stopped, for example because the watched tree grew past the environment's limit; no calls - * follow. + * - `failed`: the watcher stopped with `error`, for example because the watched tree grew past the environment's limit; + * no calls follow. */ -export type WatchChange = { paths: string[] } | { overflow: true } | { error: FileError }; +export type FileChange = + | { readonly kind: "paths"; readonly paths: string[] } + | { readonly kind: "overflow" } + | { readonly kind: "failed"; readonly error: FileError }; export interface FileWatcher { /** @@ -134,7 +137,7 @@ export interface FileWatcher { */ readonly mode: "native" | "polling"; /** Stop watching; no `onChange` call starts after this resolves. Idempotent. */ - close(context: Context): Promise; + stop(context: Context): Promise; } /** Where lines of a file are, as `BinaryReader.scanLines` found them. */ @@ -257,11 +260,11 @@ export interface FileSystem { /** * Report changes to files and directories, for hosts that load resources from the environment. When the returned * watcher exists, coverage is established: a host that watches before it loads cannot miss a change made during the - * load. See `WatchChange` for what is reported and `FileWatcher.mode` for how reliably. + * load. See `FileChange` for what is reported and `FileWatcher.mode` for how reliably. */ watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise>; canonicalPath(path: string, context: Context): Promise>; diff --git a/packages/durable/src/env/node-watch.ts b/packages/durable/src/env/node-watch.ts index 7f3b8ab55..e56645bb4 100644 --- a/packages/durable/src/env/node-watch.ts +++ b/packages/durable/src/env/node-watch.ts @@ -3,7 +3,7 @@ import { type FSWatcher, watch as fsWatch, type Stats } from "node:fs"; import { lstat, readdir, readFile, stat, statfs } from "node:fs/promises"; import { dirname, relative, sep } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "./index.ts"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "./index.ts"; /** How `NodeExecutionEnv` watches. */ export interface NodeWatchOptions { @@ -52,6 +52,8 @@ const DEBOUNCE_MS = 50; const FSEVENTS_SETTLE_MS = 500; const DEFAULT_POLL_MS = 2000; const DEFAULT_MAX_DIRECTORIES = 10_000; +/** The one `overflow` change; it carries nothing else. */ +const OVERFLOW: FileChange = { kind: "overflow" }; /** In polling mode, recently modified small files are also compared by content: a second write within the file system's * timestamp granularity can keep size and modification time. */ const HASH_MAX_BYTES = 256 * 1024; @@ -138,7 +140,7 @@ function excluded(target: ResolvedTarget, name: string): boolean { */ export class NodeFileWatcher implements FileWatcher { readonly #targets: readonly ResolvedTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #pollIntervalMs: number; readonly #maxDirectories: number; readonly #watchers = new Map(); @@ -149,11 +151,11 @@ export class NodeFileWatcher implements FileWatcher { #settleTimer: ReturnType | undefined; #running: Promise | undefined; #dirty = false; - #closed = false; + #stopped = false; private constructor( targets: readonly ResolvedTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, mode: "native" | "polling", options: NodeWatchOptions, ) { @@ -168,7 +170,7 @@ export class NodeFileWatcher implements FileWatcher { static async open( targets: readonly WatchTarget[], resolvePath: (path: string) => string, - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: NodeWatchOptions, ): Promise { const resolved = targets.map((target) => ({ @@ -200,14 +202,14 @@ export class NodeFileWatcher implements FileWatcher { return this.#mode; } - async close(_context: Context): Promise { - if (this.#closed) return; + async stop(_context: Context): Promise { + if (this.#stopped) return; this.#stop(); await this.#running; } #stop(): void { - this.#closed = true; + this.#stopped = true; clearTimeout(this.#timer); this.#timer = undefined; clearTimeout(this.#settleTimer); @@ -216,8 +218,8 @@ export class NodeFileWatcher implements FileWatcher { this.#watchers.clear(); } - #deliver(change: WatchChange): void { - if (this.#closed) return; + #deliver(change: FileChange): void { + if (this.#stopped) return; try { this.#onChange(change); } catch { @@ -226,7 +228,7 @@ export class NodeFileWatcher implements FileWatcher { } #schedulePoll(): void { - if (this.#closed || this.#mode !== "polling") return; + if (this.#stopped || this.#mode !== "polling") return; this.#timer = setTimeout(() => { this.#timer = undefined; void this.#flush().finally(() => this.#schedulePoll()); @@ -234,7 +236,7 @@ export class NodeFileWatcher implements FileWatcher { } #scheduleFlush(): void { - if (this.#closed || this.#timer !== undefined) return; + if (this.#stopped || this.#timer !== undefined) return; this.#timer = setTimeout(() => { this.#timer = undefined; void this.#flush(); @@ -242,7 +244,7 @@ export class NodeFileWatcher implements FileWatcher { } #scheduleSettle(): void { - if (this.#closed) return; + if (this.#stopped) return; clearTimeout(this.#settleTimer); this.#settleTimer = setTimeout(() => { this.#settleTimer = undefined; @@ -263,15 +265,15 @@ export class NodeFileWatcher implements FileWatcher { this.#events.clear(); const changed = await this.#sync(true); for (const path of events) changed.add(path); - if (changed.size > 0) this.#deliver({ paths: [...changed].sort() }); - } while (this.#dirty && !this.#closed); + if (changed.size > 0) this.#deliver({ kind: "paths", paths: [...changed].sort() }); + } while (this.#dirty && !this.#stopped); } catch (error) { const message = error instanceof Error ? error.message : String(error); const fileError = error instanceof FileError ? error : new FileError(isDenied(error) ? "permission_denied" : "invalid", message); - this.#deliver({ error: fileError }); + this.#deliver({ kind: "failed", error: fileError }); this.#stop(); } finally { this.#running = undefined; @@ -283,10 +285,10 @@ export class NodeFileWatcher implements FileWatcher { /** Rescan, report differences, and install watchers for new directories, rescanning until none are new. */ async #sync(report: boolean): Promise> { const changed = new Set(); - for (let round = 0; round < 10 && !this.#closed; round++) { + for (let round = 0; round < 10 && !this.#stopped; round++) { const scan = await this.#scan(); // Closed during the scan: installing watchers now would leak them. - if (this.#closed) break; + if (this.#stopped) break; const next = scan.snapshot; if (report) for (const path of this.#diff(this.#snapshot, next)) changed.add(path); this.#snapshot = next; @@ -471,14 +473,14 @@ export class NodeFileWatcher implements FileWatcher { this.#mode = "polling"; for (const { watcher } of this.#watchers.values()) watcher.close(); this.#watchers.clear(); - this.#deliver({ overflow: true }); + this.#deliver(OVERFLOW); clearTimeout(this.#timer); this.#timer = undefined; this.#schedulePoll(); } #onEvent(directory: string, filename: string | undefined): void { - if (this.#closed) return; + if (this.#stopped) return; const path = filename === undefined ? directory : `${directory.endsWith(sep) ? directory : directory + sep}${filename}`; // Events about unrelated siblings of an ancestor, or about excluded entries, are ignored. @@ -489,7 +491,7 @@ export class NodeFileWatcher implements FileWatcher { /** The file a target links to changed: report the target. */ #onLinkedFileEvent(target: string): void { - if (this.#closed) return; + if (this.#stopped) return; this.#events.add(target); this.#scheduleFlush(); } diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index fe93ce4ed..178255731 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -28,6 +28,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -41,7 +42,6 @@ import { type TextLine, type TextLineReader, toError, - type WatchChange, type WatchTarget, } from "./index.ts"; import { LineScanner } from "./line-scan.ts"; @@ -627,7 +627,7 @@ export class NodeExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); @@ -641,7 +641,7 @@ export class NodeExecutionEnv implements ExecutionEnv { ); const afterOpenAbort = abortResult(context.abortSignal); if (afterOpenAbort) { - await watcher.close(context); + await watcher.stop(context); return afterOpenAbort; } return ok(watcher); diff --git a/packages/durable/src/errors.ts b/packages/durable/src/errors.ts index b985a1c9a..622b8def6 100644 --- a/packages/durable/src/errors.ts +++ b/packages/durable/src/errors.ts @@ -13,10 +13,14 @@ export function errorMessage(error: unknown): string { } } -/** A transaction read a table after its first table write. Read every required row before writing. */ +/** + * A transaction read a table after its first table write, or admitted a submission (`submit(tx, draft)`) after it. Read + * every required row, and admit, before writing. + */ export class ReadAfterWrite extends Error { - constructor(method: string) { - super(`Tx.${method}() cannot read tables after the first table write`); + /** `operation` names the call, such as `Tx.task()`. */ + constructor(operation: string) { + super(`${operation} cannot read tables after the first table write`); this.name = "ReadAfterWrite"; } } @@ -33,14 +37,6 @@ export class StorageRequestError extends Error { } } -/** A commit landed while a storage's log was read, so the records read so far may not fit together. Read it again. */ -export class LogInterruptedError extends Error { - constructor() { - super("Storage changed while its log was read"); - this.name = "LogInterruptedError"; - } -} - /** * The Session failed, so nothing runs or commits any more. It fails when a Storage method throws, a commit cannot be * adopted, or the Harness's own bookkeeping throws (a scheduler commit, an internal commit listener, a registry whose @@ -74,3 +70,14 @@ export class ConversationBusy extends Error { this.conversationId = conversationId; } } + +/** An input with `whenIdle: "reject"` reached an idle conversation and was not admitted. */ +export class ConversationIdle extends Error { + readonly conversationId: ConversationId; + + constructor(conversationId: ConversationId) { + super(`Conversation ${conversationId} is idle`); + this.name = "ConversationIdle"; + this.conversationId = conversationId; + } +} diff --git a/packages/durable/src/harness/agent.ts b/packages/durable/src/harness/agent.ts index 64a15bb6a..df15a9f13 100644 --- a/packages/durable/src/harness/agent.ts +++ b/packages/durable/src/harness/agent.ts @@ -1,6 +1,6 @@ import { copyJson, type Draft } from "@earendil-works/chord"; import { defineDoc } from "../documents.ts"; -import { documentRetention } from "../session/session.ts"; +import { documentRetention, persistWithin } from "../session/session.ts"; import type { ConversationId, ConversationRecord, Tx } from "../types.ts"; import type { Agent, @@ -34,6 +34,7 @@ export const DEFAULT_COMPACTION_POLICY: CompactionPolicy = { export const DEFAULT_PROGRESS_POLICY: ProgressPolicy = { partialIntervalMs: 100, outputIntervalMs: 100, + persistWithinMs: 1_000, }; /** The reserved section key of the agent's `instructions`. */ @@ -56,7 +57,7 @@ const DEFAULT_CONTEXT_RETENTION_MS = 600_000; /** Default `settings.restartLimit`. */ export const DEFAULT_RESTART_LIMIT = 5; -/** A restart limit as given when it is a number of restarts, `1` or more, rounded down (`Infinity` never parks); else the default. */ +/** A restart limit as given when it is a number of restarts, `1` or more, rounded down (`Infinity`: no task ends for restarts); else the default. */ function restartLimit(limit: number | undefined): number { return typeof limit === "number" && limit >= 1 ? Math.floor(limit) : DEFAULT_RESTART_LIMIT; } @@ -73,6 +74,7 @@ export function resolveSettings(settings: HarnessSettings | undefined): Settings progress: { partialIntervalMs: settings?.progress?.partialIntervalMs ?? DEFAULT_PROGRESS_POLICY.partialIntervalMs, outputIntervalMs: settings?.progress?.outputIntervalMs ?? DEFAULT_PROGRESS_POLICY.outputIntervalMs, + persistWithinMs: persistWithin(settings?.progress?.persistWithinMs), }, toolExecution: settings?.toolExecution ?? "parallel", steeringMode: settings?.steeringMode ?? "one-at-a-time", @@ -156,6 +158,11 @@ function applyChange(state: Draft, change: AgentChange): void { ); set("instructions", change.instructions); set("cwd", change.cwd); + const maxTurns = change.maxTurns; + if (maxTurns !== undefined && maxTurns !== null && !(Number.isSafeInteger(maxTurns) && maxTurns >= 1)) { + throw new RangeError(`maxTurns must be an integer 1 or more, got ${String(maxTurns)}`); + } + set("maxTurns", maxTurns); } function isList(value: readonly T[] | object): value is readonly T[] { @@ -279,6 +286,7 @@ export function resolveAgent( sections: agentSections, ...(instructions === undefined ? {} : { instructions }), ...(state?.cwd === undefined ? {} : { cwd: state.cwd }), + ...(state?.maxTurns === undefined ? {} : { maxTurns: state.maxTurns }), }; return agent; } diff --git a/packages/durable/src/harness/compaction.ts b/packages/durable/src/harness/compaction.ts index b7376a882..e11830f6e 100644 --- a/packages/durable/src/harness/compaction.ts +++ b/packages/durable/src/harness/compaction.ts @@ -18,6 +18,8 @@ import { orderToolResults } from "./context.ts"; import { addCompactionStatus, compactionStatus, + cutOffCount, + cutOffTooOften, failCompactionStatus, LiveDoc, type LiveState, @@ -115,6 +117,7 @@ export const CompactionTask = defineTask ({ phase: "select" }), phases: { select: async (task, runtime, context) => { + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); const { conversationId } = runtime; const agent = await runtime.agent(context); const settings = runtime.settings; @@ -155,6 +158,7 @@ export const CompactionTask = defineTask ({ status: "running", checkpoint: { phase: "summarize", ...request } }), context); }, summarize: async (task, runtime, context) => { + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); const { phase: _, ...request } = task.state.checkpoint; const { model: ref, thinkingLevel, streamOptions, maxTokens, tail, firstKept, attempt } = request; const model = runtime.models.getModel(ref.provider, ref.modelId); @@ -219,6 +223,7 @@ export const CompactionTask = defineTask { const { phase: _, until, ...request } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); if (until > runtime.now()) return runtime.sleepUntil(until, task.state.checkpoint, context); const attempt = request.attempt + 1; await runtime.commit(async (tx) => { @@ -253,7 +258,7 @@ export async function createCompaction( owner === undefined ? ({ kind: "conversation" } as const) : { kind: "task" as const, taskId: owner }; const background = owner === undefined && input.reason !== "manual"; const taskId = await tx.createTask(CompactionTask, input, { ownership, conversationId, background }); - const status = { taskId, reason: input.reason, blocking: owner !== undefined, attempt: 1 }; + const status = { taskId, trigger: input.reason, blocking: owner !== undefined, attempt: 1 }; addCompactionStatus(await tx.doc(LiveDoc, conversationId), status); return taskId; } @@ -473,6 +478,18 @@ async function complete(runtime: Runtime, context: Context): Promise { }, context); } +/** Fail a compaction cut off `settings.restartLimit` times in a row (spec §5.4) instead of running it again. */ +async function failCutOff(runtime: Runtime, task: { readonly restarts?: number }, context: Context): Promise { + const message = `Compaction was interrupted ${cutOffCount(task)}`; + await runtime.commit(async (tx) => { + failCompactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId, message); + return { + status: "terminal", + outcome: { status: "failed", error: { message, detail: { reason: "restart_limit" } } }, + }; + }, context); +} + async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise { const message = ref === undefined ? "No model is configured" : `Model ${ref.provider}/${ref.modelId} is not available`; diff --git a/packages/durable/src/harness/context.ts b/packages/durable/src/harness/context.ts index c818b3cb5..d41e1a176 100644 --- a/packages/durable/src/harness/context.ts +++ b/packages/durable/src/harness/context.ts @@ -17,27 +17,31 @@ export type ContextBounds = { }; /** - * Capture the bounds of the current context, or of the context cut off at the visible entry `at`, with two O(1) - * reads. Run this on the Session line; entries at or below the tail are immutable, so `deriveContext()` can then scan - * them off the line. + * Capture the bounds of the current context, or of the context cut off at the visible entry `at`. The newest entry + * and head marker are known from the Session's commits, so the current bounds usually cost no Storage read. Run this + * on the Session line; entries at or below the tail are immutable, so `deriveContext()` can then scan them off the + * line. */ export async function captureContextBounds( + session: SessionImpl, storage: Storage, conversationId: ConversationId, context: Context, at?: EntryId, + /** Set when `at` is known to be visible, as an entry of a range of the conversation is. */ + visible = false, ): Promise { let tail: EntryId | undefined; if (at === undefined) { - tail = (await storage.scanEntries({ conversationId }, 1, undefined, context)).items[0]?.id; + tail = await session.latestEntryId(conversationId, context); if (tail === undefined) return undefined; } else { - if ((await storage.entry(conversationId, at, context)) === undefined) { + if (!visible && (await storage.entry(conversationId, at, context)) === undefined) { throw new Error(`Entry ${at} is not visible from conversation ${conversationId}`); } tail = at; } - return { head: await storage.findLatestHeadMarker(conversationId, tail, context), tail }; + return { head: await session.latestHeadMarker(conversationId, tail, context), tail }; } /** @@ -60,6 +64,11 @@ export type ContextRange = { readonly open: readonly Message[]; /** The baseline's system message; see `findBaseline()`. */ readonly baseline: Baseline; + /** + * Whether the range reached the conversation's newest entry when it was read, so the entries of later commits extend + * it (`advanceRange()`); a range cut off at an earlier entry is not. + */ + readonly current: boolean; }; /** A baseline's system message, `null` when the range cannot have one, or `undefined` while a later entry may be one. */ @@ -68,7 +77,9 @@ type Baseline = Message | null | undefined; /** * `readContext()` that reuses `previous`, an earlier range of the same conversation: with the same head marker, only * entries after its tail are scanned. Returns the view and the range to pass to the next read. A range outlives the - * read, so its entries are frozen like `MemoryStorage` records, and the returned view has its own arrays. + * read, so its entries are frozen like `MemoryStorage` records, and the returned view has its own arrays. A caller that + * keeps the range for `advanceRange()` checks `current` again, synchronously where it installs it: a commit can be + * published between this function's return and the caller's continuation. */ export async function readContextFrom( session: SessionImpl, @@ -78,23 +89,33 @@ export async function readContextFrom( at: EntryId | undefined, previous: ContextRange | undefined, ): Promise<{ readonly view: ContextView; readonly range: ContextRange | undefined }> { - const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at)); + const visible = at !== undefined && previous !== undefined && rangeEntry(previous, at) !== undefined; + const bounds = await session.readOnLine((line) => + captureContextBounds(session, line, conversationId, context, at, visible), + ); if (bounds === undefined) return { view: emptyView(), range: undefined }; freezeJson(bounds.head); + const current = at === undefined; let range: ContextRange; if (previous === undefined || previous.bounds.head?.id !== bounds.head?.id) { - range = deriveRange(bounds, freezeJson(await scanRange(storage, conversationId, rangeQuery(bounds), context))); + const entries = freezeJson(await scanRange(storage, conversationId, rangeQuery(bounds), context)); + range = deriveRange(bounds, entries, current); } else if (bounds.tail === previous.bounds.tail) { range = previous; } else if (bounds.tail < previous.bounds.tail) { range = deriveRange( bounds, previous.entries.filter((entry) => entry.id <= bounds.tail), + false, ); } else { const minEntryId = idFromNumber(previous.bounds.tail + 1); const added = await scanRange(storage, conversationId, { minEntryId, maxEntryId: bounds.tail }, context); - range = extendRange(previous, bounds, freezeJson(added)); + range = extendRange(previous, bounds, freezeJson(added), current); + } + // Entries committed while the scan ran off the line are not in the range: it is current only if none was. + if (range.current && session.knownLatestEntryId(conversationId) !== range.bounds.tail) { + range = { ...range, current: false }; } const { view } = range; return { @@ -116,7 +137,7 @@ export async function readContext( context: Context, at?: EntryId, ): Promise { - const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at)); + const bounds = await session.readOnLine((line) => captureContextBounds(session, line, conversationId, context, at)); return deriveContext(storage, conversationId, bounds, context); } @@ -133,15 +154,49 @@ export async function deriveContext( context: Context, ): Promise { if (bounds === undefined) return emptyView(); - return deriveRange(bounds, await scanRange(storage, conversationId, rangeQuery(bounds), context)).view; + return deriveRange(bounds, await scanRange(storage, conversationId, rangeQuery(bounds), context), false).view; } function emptyView(): ContextView { return { head: undefined, entries: [], contributions: [], messages: [] }; } +/** Entry `id` of `range`, by binary search: a range holds its entries in ID order, every one visible from its conversation. */ +export function rangeEntry(range: ContextRange, id: EntryId): EntryRecord | undefined { + const entries = range.entries; + let low = 0; + let high = entries.length - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const entry = entries[middle]!; + if (entry.id === id) return entry; + if (entry.id < id) low = middle + 1; + else high = middle - 1; + } + return undefined; +} + +/** + * `range` with the entries one commit appended to its conversation, in ID order, without reading Storage: the + * conversation's own new entries are its newest, and a new head marker's head never lies before the range's start. + * A range that is not `current` stays as it is; the next read extends it from Storage. `undefined` drops the range. + */ +export function advanceRange(range: ContextRange, appended: readonly EntryRecord[]): ContextRange | undefined { + if (!range.current || appended.length === 0) return range; + const added = freezeJson(appended); + let head = range.bounds.head; + for (const entry of added) if (entry.head !== undefined) head = entry as typeof head; + const bounds = { head, tail: added.at(-1)!.id }; + if (head?.id === range.bounds.head?.id) return extendRange(range, bounds, added, true); + const start = head!.head; + // Heads only move forward: compaction keeps a later entry, a reset starts at itself, an older head is stale. + if (range.bounds.head !== undefined && start < range.bounds.head.head) return undefined; + const entries = [...range.entries, ...added].filter((entry) => entry.id >= start); + return deriveRange(bounds, entries, true); +} + /** Derive the context view from the entries scanned within `bounds`. */ -function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[]): ContextRange { +function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[], current: boolean): ContextRange { const edits = new Map(); // Edits of every entry in the range count, including older head markers that `selectActive()` drops. for (const entry of entries) for (const edit of entry.edits ?? []) edits.set(edit.target, edit); @@ -151,17 +206,22 @@ function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[]): Co const baseline = findBaseline(bounds.head, active); const messages = leadWithSystem([...settled, ...orderToolResults(open)], baseline); const view = { head: bounds.head, entries: active, contributions, messages }; - return { bounds, entries, view, edited: new Set(edits.keys()), settled, open, baseline }; + return { bounds, entries, view, edited: new Set(edits.keys()), settled, open, baseline, current }; } /** * `previous` extended by the entries `added` after its tail under the same head marker: only their contributions and * the open messages are derived. An added edit can change an earlier entry, so it derives the whole range again. */ -function extendRange(previous: ContextRange, bounds: ContextBounds, added: readonly EntryRecord[]): ContextRange { +function extendRange( + previous: ContextRange, + bounds: ContextBounds, + added: readonly EntryRecord[], + current: boolean, +): ContextRange { const entries = [...previous.entries, ...added]; if (added.some((entry) => entry.edits !== undefined || entry.head !== undefined || previous.edited.has(entry.id))) { - return deriveRange(bounds, entries); + return deriveRange(bounds, entries, current); } const contributions = added.map((entry) => contribute(entry, undefined)); const { settled, open } = settle(previous.settled, [...previous.open, ...contributions.flat()]); @@ -172,7 +232,7 @@ function extendRange(previous: ContextRange, bounds: ContextBounds, added: reado contributions: [...previous.view.contributions, ...contributions], messages: leadWithSystem([...settled, ...orderToolResults(open)], baseline), }; - return { bounds, entries, view, edited: previous.edited, settled, open, baseline }; + return { bounds, entries, view, edited: previous.edited, settled, open, baseline, current }; } /** @@ -253,17 +313,6 @@ function settle( return { settled: [...settled, ...orderToolResults(open.slice(0, last))], open: open.slice(last) }; } -/** The raw active entries within captured bounds, without deriving model context. */ -export async function activeEntries( - storage: Storage, - conversationId: ConversationId, - bounds: ContextBounds | undefined, - context: Context, -): Promise { - if (bounds === undefined) return []; - return selectActive(bounds.head, await scanRange(storage, conversationId, rangeQuery(bounds), context)); -} - /** The visible range of `bounds`: from the head marker's head, or transcript start, through the tail. */ function rangeQuery(bounds: ContextBounds): Omit { const head = bounds.head; diff --git a/packages/durable/src/harness/define.ts b/packages/durable/src/harness/define.ts index b9bbd913f..f5e0df8be 100644 --- a/packages/durable/src/harness/define.ts +++ b/packages/durable/src/harness/define.ts @@ -1,6 +1,15 @@ import type { JsonValue } from "@earendil-works/chord"; -import type { TSchema } from "@earendil-works/pi-ai"; -import type { AnyTask, Extension, HookRegistration, HooksOf, PromptSection, ToolRegistration, Wrap } from "./types.ts"; +import type { Task } from "../types.ts"; +import type { + AnyTask, + Extension, + HookRegistration, + HooksOf, + PromptSection, + RunEndTask, + ToolRegistration, + Wrap, +} from "./types.ts"; /** Identity function that types an extension. */ export function defineExtension( @@ -10,7 +19,7 @@ export function defineExtension( +export function defineTool( tool: ToolRegistration, ): ToolRegistration { return tool; @@ -30,6 +39,14 @@ export function hook(task: K, handlers: Partial>): return { task: task.definition.name, handlers }; } +/** A task an `afterRun` hook asks for, its input typed by the task (spec §8.3). */ +export function runEndTask( + task: Task, + input: I, +): RunEndTask { + return { task, input }; +} + /** Wrap the tool named like `tool` wherever the wrapping extension is selected. */ export function wrapTool(tool: Tool, wrapper: (tool: Tool) => Tool): Wrap { return { tool: tool.name, wrap: wrapper }; diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts deleted file mode 100644 index 5d5d638ce..000000000 --- a/packages/durable/src/harness/events.ts +++ /dev/null @@ -1,505 +0,0 @@ -import { type Context, type JsonValue, NativeWatch, type WatchEnd } from "@earendil-works/chord"; -import type { Op, Path } from "@earendil-works/chord/delta"; -import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai"; -import type { - CommitChange, - CommitPublication, - ConversationId, - EntryRecord, - JsonObject, - SubmissionId, - SubmissionRecord, - TaskId, -} from "../types.ts"; -import { AgentDoc } from "./agent.ts"; -import type { InboxItem, InboxState } from "./inbox.ts"; -import type { CompactionFailure, CompactionStatus, LiveState, NestedToolSlot, ToolSlot } from "./live.ts"; -import { NestedCallDoc, type NestedCallState } from "./tool.ts"; -import type { AgentState, CompactionReason, Harness, NestedToolExecutionResult, ToolDiagnostic } from "./types.ts"; -import { UsageDoc, type UsageState } from "./usage.ts"; -import { scanAll } from "./util.ts"; -import { type ConversationView, conversationViews } from "./view.ts"; - -type Block = AssistantMessage["content"][number]; -type QueuedItem = { id: SubmissionId; mode: InboxItem["mode"] }; - -/** One change to the in-flight assistant message, relative to that message. */ -export type MessageChange = - | { type: "text_start" | "thinking_start" | "toolcall_start"; contentIndex: number; block: Block } - | { type: "text_delta" | "thinking_delta"; contentIndex: number; delta: string } - | { type: "toolcall_delta"; contentIndex: number; path: readonly (string | number)[]; delta: string } - | { type: "block"; contentIndex: number; block: Block } - | { type: "message"; message: AssistantMessage }; - -export type SnapshotEvent = { - type: "snapshot"; - entries: readonly EntryRecord[]; - run?: { inputs: readonly SubmissionId[] }; - /** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */ - generation?: { - attempt: number; - message?: AssistantMessage; - retry?: { at: number; error: string }; - deferred?: { pollAt: number }; - }; - tools: readonly ToolSlot[]; - /** `pi.live.nestedTools`: nested calls of running tool calls. */ - nestedTools: readonly NestedToolSlot[]; - /** `pi.live.compactions`: live compactions with their attempt and retry backoff. */ - compactions: readonly CompactionStatus[]; - /** `pi.live.compactionFailure`: the newest compaction failure since a compaction last produced a summary. */ - compactionFailure?: CompactionFailure; - inbox: readonly QueuedItem[]; - /** `pi.agent`; `{}` when absent. */ - agent: AgentState; - usage: UsageState; -}; - -/** Experimental agent event, shaped like the coding agent's session events (spec §9.4). */ -export type AgentEvent = - | SnapshotEvent - | { type: "run_start"; inputs: readonly SubmissionId[] } - | { type: "run_end"; inputs: readonly SubmissionId[] } - | { type: "turn_start" } - | { type: "turn_end" } - | { type: "message_start"; message: Message } - /** `usage` is the partial's current usage, as in the coding agent's JSON mode. */ - | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } - | { type: "message_end"; entry: EntryRecord } - | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) - | ({ - type: "tool_execution_update"; - /** A front trim and then an append of the retained window, or its replacement. */ - output?: { trimStart?: number; append?: string } | { set: string }; - details?: JsonValue; - diagnostics?: readonly ToolDiagnostic[]; - } & ToolEventCall) - /** - * A model-issued call ends with its result `entry`, a nested call with its `result`. Both are absent when the tool - * task faulted or was orphaned, or when the call's slot left `pi.live` unfinished: its run ended, or its parent - * settled first. - */ - | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) - | { type: "inbox_update"; items: readonly QueuedItem[] } - | { type: "submission"; record: SubmissionRecord } - | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } - | { type: "auto_retry_end"; attempt: number } - | { type: "deferred_poll"; pollAt: number } - | { type: "entry_appended"; entry: EntryRecord } - | { type: "agent_changed"; agent: AgentState } - | { type: "usage_changed"; usage: UsageState } - | { type: "task_failed"; taskId: TaskId; kind: string; message: string } - | { type: "compaction_start"; taskId: TaskId; reason: CompactionReason; blocking: boolean } - /** - * The task's receipt tells whether it produced a summary; the summary entry has its own events. `error` is set when - * the compaction failed, faulted, or was orphaned, as in `pi.live.compactionFailure`. - */ - | { type: "compaction_end"; taskId: TaskId; reason: CompactionReason; error?: string }; - -/** - * Which call a tool event is about. `toolCallId` is the call's ID in the transcript and in tool events: the provider's - * ID for a model-issued call, `/` for a nested one. Match events by it; do not parse it. `taskId` - * is the call's tool task, absent for a call that never got one (not offered, or not started in a sequential round). - * A nested call also names the call that made it, by call ID and by tool task. - */ -export type ToolEventCall = { - toolCallId: string; - toolName: string; - taskId?: TaskId; - parentToolCallId?: string; - parentTaskId?: TaskId; -}; - -/** Serialized stream of one conversation's event batches, one per commit. */ -export interface AgentEventStream { - /** The `snapshot` event at attachment. */ - readonly snapshot: SnapshotEvent; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; - stop(): Promise; - readonly closed: Promise; -} - -/** The typed parts of a view the events read. */ -type Parts = { - live: LiveState; - inbox: InboxState | undefined; - agent: AgentState | undefined; - usage: UsageState | undefined; -}; - -function parts(view: ConversationView): Parts { - return { - live: (view.docs["pi.live"] ?? {}) as LiveState, - inbox: view.docs["pi.inbox"] as InboxState | undefined, - agent: view.docs["pi.agent"] as AgentState | undefined, - usage: view.docs["pi.usage"] as UsageState | undefined, - }; -} - -function snapshotOf(view: ConversationView): SnapshotEvent { - const { live, inbox, agent, usage } = parts(view); - return { - type: "snapshot", - entries: view.entries, - ...(live.run === undefined ? {} : { run: { inputs: live.run.inputs } }), - ...(live.generation === undefined ? {} : { generation: live.generation as SnapshotEvent["generation"] }), - tools: live.tools ?? [], - nestedTools: live.nestedTools ?? [], - compactions: live.compactions ?? [], - ...(live.compactionFailure === undefined ? {} : { compactionFailure: live.compactionFailure }), - inbox: queued(inbox), - agent: agent ?? AgentDoc.definition.initial(), - usage: usage ?? UsageDoc.definition.initial(), - }; -} - -function queued(inbox: InboxState | undefined): QueuedItem[] { - return (inbox?.items ?? []).map((item) => ({ id: item.id, mode: item.mode })); -} - -/** - * Experimental: attach to one conversation's agent events (spec §9.4). The snapshot and the registration for later - * commits are captured atomically on the Session line; overflow replaces undelivered batches with one snapshot. - */ -export async function watchEvents( - harness: Harness, - conversationId: ConversationId, - context: Context, -): Promise { - const views = conversationViews(harness); - let watch!: NativeWatch; - let snapshot!: SnapshotEvent; - await views.attach( - conversationId, - async (initial, release, storage) => { - // Generations whose held outcome already ended their turn, read on the line with the snapshot. - const query = { conversationId, kind: "pi.generation", status: "completing" } as const; - const completing = await scanAll((cursor) => storage.scanTasks(query, 100, cursor, context)); - const held = new Set(completing.map((record) => record.id)); - let current = initial; - snapshot = snapshotOf(initial); - // Batches are the watch's values, each delivered (folding would drop events); an overflow delivers a snapshot of - // the newest view instead. - watch = new NativeWatch( - [], - release, - (error) => views.report(error), - () => [snapshotOf(current)], - ); - return { - publication: (before, after, ops, publication, commitContext) => { - current = after; - const events = translate(conversationId, before, after, ops, publication, held); - if (events.length > 0) watch.advance(events, commitContext); - }, - end: (failure) => watch.end(failure), - }; - }, - context, - ); - // Like a watch, the acquisition context governs the stream's lifetime. - const signal = context.abortSignal; - if (signal?.aborted) { - watch.cancel(); - throw signal.reason; - } - if (signal !== undefined) watch.observeCancellation(signal); - return { - snapshot, - start: (listener) => watch.start(listener), - stop: () => watch.stop(), - closed: watch.closed, - }; -} - -type TaskChange = Extract; - -/** The tool result for `callId` among `entries`. */ -function resultOf(entries: readonly EntryRecord[], callId: string): EntryRecord | undefined { - return entries.find((entry) => { - const message = entry.model?.[0]; - return message?.role === "toolResult" && message.toolCallId === callId; - }); -} - -/** Every event one publication causes, in the order of spec §9.4. */ -function translate( - conversationId: ConversationId, - before: ConversationView, - after: ConversationView, - viewOps: readonly Op[], - publication: CommitPublication, - held: Set, -): AgentEvent[] { - const entries: EntryRecord[] = []; - const tasks = new Map(); - const submissions: SubmissionRecord[] = []; - // Nested results this commit stored, by nested task ID; read now, since the caller's documents may retire next. - const nestedResults = new Map(); - for (const change of publication.changes) { - if ( - change.type === "document" && - change.record.kind === NestedCallDoc.definition.kind && - change.value !== null && - change.conversationId === conversationId - ) { - const state = change.value as NestedCallState; - if (state.taskId !== undefined && state.result !== undefined) { - nestedResults.set(state.taskId, state.result as unknown as NestedToolExecutionResult); - } - } - if (change.type === "entry" && change.value.conversationId === conversationId) entries.push(change.value); - if (change.type === "task" && change.value.conversationId === conversationId) - tasks.set(change.value.id, change.value); - if (change.type === "submission" && change.value.conversationId === conversationId) { - submissions.push(change.value); - } - } - if (viewOps.length === 0 && entries.length === 0 && tasks.size === 0 && submissions.length === 0) return []; - // Entries are appended in ID order; submission records are published in the order the commit first touched them. - submissions.sort((a, b) => a.id - b.id); - const was = parts(before); - const now = parts(after); - const events: AgentEvent[] = []; - - // Progress: tool starts, the in-flight message, tool updates, retry and deferred state. - const slotsBefore = new Map((was.live.tools ?? []).map((slot) => [slot.callId, slot])); - const slots = now.live.tools ?? []; - const nestedBefore = new Map((was.live.nestedTools ?? []).map((slot) => [slot.taskId, slot])); - const nested = now.live.nestedTools ?? []; - for (const slot of [...slots, ...nested]) { - const previous = "parentCallId" in slot ? nestedBefore.get(slot.taskId) : slotsBefore.get(slot.callId); - if (slot.status !== "running" || previous?.status === "running") continue; - // A nested slot carries the arguments the call runs with; a model-issued call's are in its intent checkpoint. - const checkpoint = slot.taskId === undefined ? undefined : tasks.get(slot.taskId)?.state.checkpoint; - const args = - "parentCallId" in slot - ? slot.arguments - : ((checkpoint as { arguments?: JsonObject } | undefined)?.arguments ?? {}); - events.push({ type: "tool_execution_start", ...callOf(slot), args }); - } - const partialBefore = was.live.generation?.message as AssistantMessage | undefined; - const partial = now.live.generation?.message as AssistantMessage | undefined; - if (partial !== undefined && partialBefore === undefined) events.push({ type: "message_start", message: partial }); - else if (partial !== undefined && partial !== partialBefore) { - events.push({ type: "message_update", usage: partial.usage, changes: messageChanges(viewOps, partial) }); - } - for (const [index, slot] of slots.entries()) { - const previous = slotsBefore.get(slot.callId); - if (slot.status !== "running" || previous?.status !== "running") continue; - const update = toolUpdate(viewOps, ["tools", index], slot, previous); - if (update === undefined) continue; - events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); - } - for (const [index, slot] of nested.entries()) { - const previous = nestedBefore.get(slot.taskId); - if (slot.status !== "running" || previous?.status !== "running") continue; - const update = toolUpdate(viewOps, ["nestedTools", index], slot, previous); - if (update === undefined) continue; - events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); - } - const generation = now.live.generation; - const generationBefore = was.live.generation; - if (generation?.retry !== undefined && generationBefore?.retry === undefined) { - const { at, error } = generation.retry; - events.push({ type: "auto_retry_start", attempt: generation.attempt, at, errorMessage: error }); - } - if (generationBefore?.retry !== undefined && generation?.retry === undefined) { - events.push({ type: "auto_retry_end", attempt: generationBefore.attempt }); - } - if (generation?.deferred !== undefined && generation.deferred.pollAt !== generationBefore?.deferred?.pollAt) { - events.push({ type: "deferred_poll", pollAt: generation.deferred.pollAt }); - } - - // Tools that end in this commit: a slot that becomes done, one created done (a call not offered), or an unfinished - // one that vanishes because its run ended. A done slot that vanishes ended earlier. - const toolEnds: Extract[] = []; - const endTool = (slot: ToolSlot, entryId: number | undefined): void => { - const entry = entries.find((candidate) => candidate.id === entryId); - toolEnds.push({ type: "tool_execution_end", ...callOf(slot), ...(entry === undefined ? {} : { entry }) }); - }; - for (const previous of slotsBefore.values()) { - if (previous.status === "done") continue; - const slot = slots.find((candidate) => candidate.callId === previous.callId); - if (slot?.status === "done") endTool(slot, slot.entry); - // A slot whose run ended in this commit may have had its result appended with it, as for unstarted calls. - else if (slot === undefined) endTool(previous, resultOf(entries, previous.callId)?.id); - } - for (const slot of slots) { - if (slot.status === "done" && !slotsBefore.has(slot.callId)) endTool(slot, slot.entry); - } - // Nested calls that end in this commit, the same way, children before the calls that made them: the lists hold - // parents first, so walk them backwards. - const endNested = (slot: NestedToolSlot): void => { - const result = nestedResults.get(slot.taskId); - events.push({ type: "tool_execution_end", ...callOf(slot), ...(result === undefined ? {} : { result }) }); - }; - const nestedNow = new Map(nested.map((slot) => [slot.taskId, slot])); - for (const previous of [...nestedBefore.values()].reverse()) { - if (previous.status === "done") continue; - const slot = nestedNow.get(previous.taskId); - if (slot?.status === "done" || slot === undefined) endNested(slot ?? previous); - } - for (const slot of [...nested].reverse()) { - if (slot.status === "done" && !nestedBefore.has(slot.taskId)) endNested(slot); - } - - // Entries in append order; a tool's end directly precedes its result's message, as in the coding agent. - let assistantAppended = false; - for (const entry of entries) { - events.push(...toolEnds.filter((end) => end.entry === entry)); - const message = entry.model?.[0]; - if (message === undefined) { - events.push({ type: "entry_appended", entry }); - continue; - } - // A streamed answer already started with its first partial. - const streamed = message.role === "assistant" && partialBefore !== undefined && !assistantAppended; - if (message.role === "assistant") assistantAppended = true; - if (!streamed) events.push({ type: "message_start", message }); - events.push({ type: "message_end", entry }); - } - // Ends without a result entry: a faulted or orphaned tool, or one whose run ended. - events.push(...toolEnds.filter((end) => end.entry === undefined)); - - // Compaction ends, task failures, then turn and run ends. - const compactionsBefore = was.live.compactions ?? []; - const compactions = now.live.compactions ?? []; - const failure = now.live.compactionFailure; - for (const { taskId, reason } of compactionsBefore) { - if (!compactions.some((status) => status.taskId === taskId)) { - const error = failure?.taskId === taskId ? failure.error : undefined; - events.push({ type: "compaction_end", taskId, reason, ...(error === undefined ? {} : { error }) }); - } - } - // A generation's turn ends when its outcome is committed: at a `completing` hold or at terminal, whichever comes - // first, so a successor created at the hold starts after it. - let turnEnded = false; - for (const task of tasks.values()) { - const status = task.state.status; - if (task.kind === "pi.generation" && status === "completing" && !held.has(task.id)) { - held.add(task.id); - turnEnded = true; - } - if (status !== "terminal") continue; - if (task.kind === "pi.generation" && !held.delete(task.id)) turnEnded = true; - const outcome = task.state.outcome; - if (outcome.status === "faulted" || outcome.status === "orphaned") { - const message = outcome.status === "faulted" ? outcome.error.message : outcome.reason; - events.push({ type: "task_failed", taskId: task.id, kind: task.kind, message }); - } - } - if (turnEnded) events.push({ type: "turn_end" }); - const run = now.live.run; - const runBefore = was.live.run; - const runChanged = run?.inputs[0] !== runBefore?.inputs[0]; - if (runBefore !== undefined && runChanged) events.push({ type: "run_end", inputs: runBefore.inputs }); - - // Submissions, document state, then what began. - for (const record of submissions) events.push({ type: "submission", record }); - if (now.inbox !== was.inbox) events.push({ type: "inbox_update", items: queued(now.inbox) }); - // A retired document reads as its initial value, as in a snapshot. - if (now.agent !== was.agent) { - events.push({ type: "agent_changed", agent: now.agent ?? AgentDoc.definition.initial() }); - } - if (now.usage !== was.usage) - events.push({ type: "usage_changed", usage: now.usage ?? UsageDoc.definition.initial() }); - for (const { taskId, reason, blocking } of compactions) { - if (!compactionsBefore.some((status) => status.taskId === taskId)) { - events.push({ type: "compaction_start", taskId, reason, blocking }); - } - } - if (run !== undefined && runChanged) events.push({ type: "run_start", inputs: run.inputs }); - if (run !== undefined && run.taskId !== runBefore?.taskId && tasks.get(run.taskId)?.kind === "pi.generation") { - events.push({ type: "turn_start" }); - } - return events; -} - -const PARTIAL_PATH = ["docs", "pi.live", "generation", "message"]; - -/** Translate the view operations on the in-flight message into message changes (spec §9.4). */ -function messageChanges(viewOps: readonly Op[], message: AssistantMessage): MessageChange[] { - const changes: MessageChange[] = []; - // A block sent whole already holds every later change to it in this batch. - const whole = new Set(); - for (const op of viewOps) { - // View operations never replace the root. - const path = op[1] as Path; - if (!startsWith(path, PARTIAL_PATH)) { - // The whole message or generation was replaced. - if (startsWith(PARTIAL_PATH, path)) return [{ type: "message", message }]; - continue; - } - const rest = path.slice(PARTIAL_PATH.length); - if (rest[0] === "usage") continue; - if (rest[0] !== "content") return [{ type: "message", message }]; - if (rest.length === 1) { - if (op[0] !== "p" || op[3] !== 0) return [{ type: "message", message }]; - for (const [offset, block] of (op[4] as unknown as Block[]).entries()) { - const type = - block.type === "text" ? "text_start" : block.type === "thinking" ? "thinking_start" : "toolcall_start"; - changes.push({ type, contentIndex: op[2] + offset, block }); - } - continue; - } - const contentIndex = rest[1] as number; - const field = rest[2]; - if (whole.has(contentIndex)) continue; - if (op[0] === "a" && rest.length === 3 && (field === "text" || field === "thinking")) { - changes.push({ type: field === "text" ? "text_delta" : "thinking_delta", contentIndex, delta: op[2] }); - } else if (op[0] === "a" && field === "arguments") { - changes.push({ type: "toolcall_delta", contentIndex, path: rest.slice(3), delta: op[2] }); - } else { - whole.add(contentIndex); - changes.push({ type: "block", contentIndex, block: message.content[contentIndex]! }); - } - } - return changes; -} - -/** The identifying fields of a slot's tool events. */ -function callOf(slot: ToolSlot | NestedToolSlot): ToolEventCall { - return { - toolCallId: slot.callId, - toolName: slot.name, - ...(slot.taskId === undefined ? {} : { taskId: slot.taskId }), - ...("parentCallId" in slot ? { parentToolCallId: slot.parentCallId, parentTaskId: slot.parentTaskId } : {}), - }; -} - -/** Output, details, and diagnostics changes of a running slot at `at` in `pi.live`, from the view operations on it. */ -function toolUpdate( - viewOps: readonly Op[], - at: readonly ["tools" | "nestedTools", number], - slot: ToolSlot | NestedToolSlot, - previous: ToolSlot | NestedToolSlot, -): Omit, "type" | keyof ToolEventCall> | undefined { - const outputPath = ["docs", "pi.live", ...at, "output"]; - let trimStart = 0; - let append = ""; - let set = false; - for (const op of viewOps) { - if (!startsWith(op[1] as Path, outputPath)) continue; - if (op[0] === "t") trimStart += op[2]; - else if (op[0] === "a") append += op[2]; - else set = true; - } - let output: { trimStart?: number; append?: string } | { set: string } | undefined; - if (set || (slot.output !== previous.output && trimStart === 0 && append === "")) - output = { set: slot.output ?? "" }; - else if (trimStart > 0 || append !== "") { - output = { ...(trimStart > 0 ? { trimStart } : {}), ...(append === "" ? {} : { append }) }; - } - // A safe replay clears a running slot's progress: removed details send `null`, removed diagnostics `[]`. - const update = { - ...(output === undefined ? {} : { output }), - ...(slot.details === previous.details ? {} : { details: slot.details ?? null }), - ...(slot.diagnostics === previous.diagnostics ? {} : { diagnostics: slot.diagnostics ?? [] }), - }; - return Object.keys(update).length === 0 ? undefined : update; -} - -function startsWith(path: Path, prefix: readonly (string | number)[]): boolean { - return prefix.length <= path.length && prefix.every((segment, index) => path[index] === segment); -} diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index d617713bd..17c28db47 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -21,20 +21,31 @@ import type { EntryId, NextTaskState, SubmissionId, - SubmissionSettlement, + Task, TaskId, TaskOutcome, TaskRuntime, Tx, TypedEntry, } from "../types.ts"; -import { addTools, sectionExtension } from "./agent.ts"; +import { addTools, agentHooks, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; -import { applyBoundary, type Boundary, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; -import { endRun, LiveDoc, type LiveState, type ToolSlot } from "./live.ts"; -import { PartialChanges } from "./partial-changes.ts"; +import { applyBoundary, type Boundary, hasQueuedReset, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; +import { + cutOffCount, + cutOffTooOften, + endRun, + LiveDoc, + type LiveState, + PROGRESS_COMMIT, + type RunEnd, + salvageable, + type ToolSlot, +} from "./live.ts"; +import { PartialChanges, type PartialMessage } from "./partial-changes.ts"; import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; import { ensureProviderSessionId } from "./provider.ts"; +import { endedByCrashes } from "./scheduler.ts"; import { appendToolResult, harnessError, ToolTask, type ToolTaskResult } from "./tool.ts"; import type { AfterFailedRun, @@ -45,6 +56,8 @@ import type { GenerationHooks, ModelRef, PromptInput, + RunEndTask, + RunSummary, ToolControl, UserInput, } from "./types.ts"; @@ -75,12 +88,24 @@ export type GenerationCheckpoint = /** Newest entry included in the request. */ cutoff: EntryId; } - | { phase: "retry"; attempt: number; compacted?: TaskId; until: number } + | { + phase: "retry"; + attempt: number; + compacted?: TaskId; + until: number; + /** + * The deferred response a poll was fetching when the host died: the next attempt polls it again instead of + * sending another request, so the provider's job is neither lost nor paid twice. Absent otherwise. + */ + poll?: { model: ModelRef; thinkingLevel?: ModelThinkingLevel; cutoff: EntryId; handle: DeferredHandle }; + } | { phase: "poll"; attempt: number; compacted?: TaskId; model: ModelRef; + /** Absent in checkpoints of earlier versions. */ + thinkingLevel?: ModelThinkingLevel; cutoff: EntryId; handle: DeferredHandle; pollAt: number; @@ -94,6 +119,20 @@ export type GenerationCheckpoint = tools: TaskId[]; /** Calls of a sequential round not started yet, in call order. */ pending: string[]; + } + | { + /** + * The run answered: its answer is committed and its inputs settled, and it keeps `pi.live.run` while `afterRun` + * hooks are asked for tasks (spec §8.3). A crash here asks them again: nothing was created yet. + */ + phase: "finish"; + run: RunSummary; + } + | { + /** Waiting on the tasks `afterRun` hooks asked for, which the generation owns; then the run ends. */ + phase: "drain"; + answer: EntryId; + tasks: TaskId[]; }; export type GenerationResult = { entryId: EntryId }; @@ -106,6 +145,7 @@ type Request = { readonly attempt: number; readonly compacted: TaskId | undefined; readonly model: ModelRef; + readonly thinkingLevel: ModelThinkingLevel | undefined; readonly cutoff: EntryId; /** Committed model context through `cutoff`, when the phase already derived it. */ readonly messages?: readonly Message[]; @@ -121,8 +161,15 @@ const DEFAULT_POLL_AFTER_MS = 5000; */ export const GenerationTask = defineTask({ name: "pi.generation", - version: 1, + version: 2, initial: () => ({ phase: "prepare", attempt: 1 }), + // Version 2 leaves calls to tools not offered in a sequential round's `pending`, flagged `unoffered` in their slots. + // A version 1 Harness would start their tool tasks unflagged, resolving the name, so it leaves version 2 blocked + // instead. A version 1 checkpoint has no such calls and is kept as it is. + migrate: (input, checkpoint) => ({ + input: input as GenerationInput, + checkpoint: checkpoint as GenerationCheckpoint, + }), phases: { /** * Render the system prompt and tool loadout and append the positional `pi.system` entries they need, then move to @@ -131,6 +178,7 @@ export const GenerationTask = defineTask { + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); const { conversationId } = runtime; const agent = await runtime.agent(context); const settings = runtime.settings; @@ -153,13 +201,18 @@ export const GenerationTask = defineTask message.role !== "system"); + if (last?.role !== "user" && last?.role !== "toolResult") return failNothingToAnswer(runtime, context); const shown = replaySections(view.messages); let env: ExecutionEnv | undefined; try { env = await runtime.env(context); } catch (error) { if (context.abortSignal?.aborted) throw error; - runtime.report(error, { type: "host", callback: "env" }); + runtime.report(error, { kind: "host", callback: "env" }); } const input: PromptInput = { conversationId, agent, env, shown: Object.fromEntries(shown), read: runtime }; const desired = await renderSections( @@ -168,7 +221,7 @@ export const GenerationTask = defineTask runtime.report(error, { - type: "extension", + kind: "extension", ...optionalExtension(sectionExtension(agent, section.key)), }), context, @@ -188,9 +241,9 @@ export const GenerationTask = defineTask { - let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]?.id; + // The context ends in a message the model answers, so the conversation has an entry. + let cutoff = (await tx.latestEntryId(conversationId))!; for (const entry of entries) cutoff = (await tx.appendEntry(SystemEntry, conversationId, entry)).id; - if (cutoff === undefined) throw new Error(`Conversation ${conversationId} has no entries to send`); // Checked in this commit, so a compaction admitted during preparation counts. if (threshold === "background" && (await tx.doc(LiveDoc, conversationId)).compactions === undefined) { await createCompaction(tx, conversationId, { reason: "threshold" }); @@ -207,14 +260,24 @@ export const GenerationTask = defineTask { + // Charging the attempt may itself be what keeps failing, such as converting a huge partial: the limit ends it. + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); + // The host stopped while this request was in flight: a failed attempt, not a resend (spec §8.3). + if (task.restarts !== undefined) return interrupted(runtime, task.state.checkpoint, context); const { attempt, compacted, model: ref, thinkingLevel, streamOptions, cutoff } = task.state.checkpoint; const conversationId = runtime.conversationId; - await runtime.commit(async (tx) => { - const live = await tx.doc(LiveDoc, conversationId); - await convertPartial(tx, live, conversationId); - live.generation = { attempt }; - return undefined; - }, context); + // The attempt shown is presentation, stored like progress (`PROGRESS_COMMIT`); converting a partial left by an + // interrupted attempt appends an entry, which makes the commit immediate. + await runtime.commit( + async (tx) => { + const live = await tx.doc(LiveDoc, conversationId); + await convertPartial(tx, live, conversationId); + live.generation = { attempt }; + return undefined; + }, + context, + PROGRESS_COMMIT, + ); const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); const view = await runtime.context(conversationId, context, { at: cutoff }); @@ -229,28 +292,40 @@ export const GenerationTask = defineTask providerPayload(runtime, model, payload, context), }; const message = await streamResponse(runtime, model, messages, options, attempt, context); - const request = { attempt, compacted, model: ref, cutoff, messages: view.messages }; + const request = { attempt, compacted, model: ref, thinkingLevel, cutoff, messages: view.messages }; await classify(runtime, request, message, context); }, /** After the backoff's sleep; a run before `until` (an early wake, or a record of an earlier version) sleeps again. */ retry: async (task, runtime, context) => { - const { attempt, compacted, until } = task.state.checkpoint; + const { attempt, compacted, until, poll } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); if (until > runtime.now()) return runtime.sleepUntil(until, task.state.checkpoint, context); await runtime.commit(async (tx) => { - (await tx.doc(LiveDoc, runtime.conversationId)).generation = { attempt: attempt + 1 }; - const checkpoint: GenerationCheckpoint = { - phase: "prepare", - attempt: attempt + 1, - ...(compacted === undefined ? {} : { compacted }), - }; - return { status: "running", checkpoint }; + const live = await tx.doc(LiveDoc, runtime.conversationId); + const kept = compacted === undefined ? {} : { compacted }; + // A poll the host died during polls its response again, as the next attempt. + if (poll !== undefined) { + const pollAt = runtime.now(); + live.generation = { attempt: attempt + 1, deferred: { pollAt } }; + return { + status: "running", + checkpoint: { phase: "poll", attempt: attempt + 1, ...kept, ...poll, pollAt }, + }; + } + live.generation = { attempt: attempt + 1 }; + return { status: "running", checkpoint: { phase: "prepare", attempt: attempt + 1, ...kept } }; }, context); }, /** After the sleep until `pollAt`; a run before it (an early wake, or a record of an earlier version) sleeps again. */ poll: async (task, runtime, context) => { - const { attempt, compacted, model: ref, cutoff, handle, pollAt } = task.state.checkpoint; + const { attempt, compacted, model: ref, thinkingLevel, cutoff, handle, pollAt } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); + // The host stopped while the poll's fetch was in flight: a failed attempt, as for a request. Checked before the + // time, which may have moved backwards on a replacement host; a sleeping poll is never cut off. + if (task.restarts !== undefined) return interrupted(runtime, task.state.checkpoint, context); if (pollAt > runtime.now()) return runtime.sleepUntil(pollAt, task.state.checkpoint, context); const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); @@ -258,18 +333,19 @@ export const GenerationTask = defineTask { + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); const { assistant, tools, pending } = task.state.checkpoint; const [next, ...rest] = pending; if (next === undefined) return finishToolRound(runtime, assistant, tools, context); // Sequential round: start the next call and wait for it. await runtime.commit(async (tx): Promise => { const live = await tx.doc(LiveDoc, runtime.conversationId); - const taskId = await createToolTask(tx, runtime, assistant, next); const slot = live.tools?.find((slot) => slot.callId === next && slot.taskId === undefined); + const taskId = await createToolTask(tx, runtime, assistant, next, slot?.unoffered); if (slot !== undefined) slot.taskId = taskId; const checkpoint: GenerationCheckpoint = { phase: "tools", @@ -280,24 +356,44 @@ export const GenerationTask = defineTask { + const { run } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endAnsweredCutOff(runtime, task, run.answer, context); + const prepared: PreparedTask[] = []; + await runtime.hooks.each("afterRun", async (hook) => { + const asked = await hook(run, runtime, context); + if (asked !== undefined) prepareRunEndTasks(runtime, asked.tasks, prepared); + }); + if (prepared.length === 0) return endAnsweredRun(runtime, run.answer, context); + await runtime.commit(async (tx): Promise => { + const tasks: TaskId[] = []; + for (const { task, input } of prepared) { + tasks.push(await tx.createTask(task, input, { ownership: { kind: "task", taskId: runtime.taskId } })); + } + const live = await tx.doc(LiveDoc, runtime.conversationId); + if (live.run?.taskId === runtime.taskId && live.run.afterRun !== undefined) live.run.afterRun.tasks = tasks; + const checkpoint = { phase: "drain", answer: run.answer, tasks } as const; + return { status: "waiting", checkpoint, on: tasks, policy: "allSettled" }; + }, context); + }, + /** The after-run tasks are terminal, whatever their outcomes: end the run. */ + drain: async (task, runtime, context) => { + const { answer } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endAnsweredCutOff(runtime, task, answer, context); + return endAnsweredRun(runtime, answer, context); + }, }, abort: async (task, runtime, context) => { const checkpoint = task.state.checkpoint; - if (checkpoint.phase === "poll") { - const model = runtime.models.getModel(checkpoint.model.provider, checkpoint.model.modelId); - if (model !== undefined) { - try { - await runtime.models.cancelDeferred(model, checkpoint.handle, { - signal: runtime.signal, - telemetry: context.telemetry, - }); - } catch (error) { - runtime.report(error, { type: "host", callback: "models" }); - } - } - } + const polled = deferredOf(checkpoint); + if (polled !== undefined) await cancelDeferred(runtime, polled, context); const conversationId = runtime.conversationId; - // Runs after the round's tool tasks are terminal; calls never started get `aborted` results (spec §8.5). + // Runs after the round's tool tasks are terminal; calls never started get `aborted` results (spec §8.5), unoffered + // ones `tool_unavailable`. const unstarted = checkpoint.phase === "tools" ? await readCalls(runtime, checkpoint.assistant, checkpoint.pending, context) @@ -306,9 +402,14 @@ export const GenerationTask = defineTask slot.callId === call.id)?.unoffered === true; + const result = unoffered + ? harnessError("tool_unavailable", `Tool ${call.name} is not available`) + : harnessError("aborted", `Tool ${call.name} was aborted`); await appendToolResult(tx, conversationId, call, result, { timestamp: runtime.now() }); } + // In `finish` and `drain` the run answered already: it ends without a failure, its inputs stay `done`. endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "aborted" }); return { status: "terminal", outcome: { status: "aborted" } }; }, context); @@ -327,11 +428,17 @@ async function readCalls( return callIds.flatMap((id) => calls.filter((call) => call.id === id).slice(0, 1)); } -/** A tool task for call `callId`, owned by the generation. */ -function createToolTask(tx: Tx, runtime: Runtime, assistant: EntryId, callId: string): Promise> { +/** A tool task for call `callId`, owned by the generation; `unoffered` for a call to a tool its request did not offer. */ +function createToolTask( + tx: Tx, + runtime: Runtime, + assistant: EntryId, + callId: string, + unoffered: true | undefined, +): Promise> { return tx.createTask( ToolTask, - { kind: "model", assistant, callId }, + { kind: "model", assistant, callId, ...(unoffered === undefined ? {} : { unoffered }) }, { ownership: { kind: "task", taskId: runtime.taskId } }, ); } @@ -386,6 +493,109 @@ async function failModelError(runtime: Runtime, text: string, context: Context): }, context); } +/** + * The host stopped while this attempt's request or poll was in flight (`TaskRecord.restarts`): a failed attempt under + * `settings.retry` (spec §8.3). The committed partial becomes an aborted entry; the next attempt follows the backoff, a + * poll's by polling its deferred response again, or, with no retry left, the run ends `interrupted`. + */ +async function interrupted( + runtime: Runtime, + checkpoint: Extract, + context: Context, +): Promise { + const { attempt, compacted } = checkpoint; + const policy = runtime.settings.retry; + const retry = policy.enabled && attempt <= policy.maxRetries; + const until = retry ? runtime.now() + retryDelayMs(policy, attempt) : 0; + const polled = checkpoint.phase === "poll" ? checkpoint : undefined; + const text = + polled === undefined + ? `The host stopped while the model was answering, on attempt ${attempt}` + : `The host stopped while the deferred answer was fetched, on attempt ${attempt}`; + // A run that ends does not leave the provider's job running. + if (!retry && polled !== undefined) await cancelDeferred(runtime, polled, context); + await runtime.commit(async (tx): Promise => { + // Read before the first table write; only a failure uses it. + const failure = retry ? undefined : await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + await convertPartial(tx, live, runtime.conversationId); + if (failure === undefined) { + live.generation = { attempt, retry: { at: until, error: text } }; + const poll = + polled === undefined + ? {} + : { + poll: { + model: polled.model, + ...(polled.thinkingLevel === undefined ? {} : { thinkingLevel: polled.thinkingLevel }), + cutoff: polled.cutoff, + handle: polled.handle, + }, + }; + const next = { phase: "retry", attempt, ...(compacted === undefined ? {} : { compacted }), until, ...poll }; + return { status: "sleeping", checkpoint: next as GenerationCheckpoint, until }; + } + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "interrupted", detail: text }); + return { + status: "terminal", + outcome: { status: "failed", error: { message: text, detail: { reason: "interrupted" } } }, + }; + }, context); +} + +/** + * End a run whose generation was cut off `settings.restartLimit` times in a row (spec §5.4). At the limit it still + * keeps a committed partial as an aborted entry and cancels a deferred response it polled; past the limit that cleanup + * may be what cut it off, so it ends without it. + */ +async function endCutOff( + runtime: Runtime, + task: { readonly restarts?: number; readonly state: { readonly checkpoint: GenerationCheckpoint } }, + context: Context, +): Promise { + const checkpoint = task.state.checkpoint; + const text = `The generation was interrupted ${cutOffCount(task)} in its ${checkpoint.phase} phase`; + const salvage = salvageable(task, runtime); + const polled = deferredOf(checkpoint); + if (salvage && polled !== undefined) await cancelDeferred(runtime, polled, context); + await runtime.commit(async (tx): Promise => { + const failure = await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + if (salvage) await convertPartial(tx, live, runtime.conversationId); + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "restart_limit", detail: text }); + return { + status: "terminal", + outcome: { status: "failed", error: { message: text, detail: { reason: "restart_limit" } } }, + }; + }, context); +} + +/** The deferred response a checkpoint polls, or will poll again after a retry's backoff. */ +function deferredOf( + checkpoint: GenerationCheckpoint, +): { readonly model: ModelRef; readonly handle: DeferredHandle } | undefined { + if (checkpoint.phase === "poll") return checkpoint; + return checkpoint.phase === "retry" ? checkpoint.poll : undefined; +} + +/** Cancel a deferred response no attempt will poll again; a failure is reported, never thrown. */ +async function cancelDeferred( + runtime: Runtime, + checkpoint: { readonly model: ModelRef; readonly handle: DeferredHandle }, + context: Context, +): Promise { + const model = runtime.models.getModel(checkpoint.model.provider, checkpoint.model.modelId); + if (model === undefined) return; + try { + await runtime.models.cancelDeferred(model, checkpoint.handle, { + signal: runtime.signal, + telemetry: context.telemetry, + }); + } catch (error) { + runtime.report(error, { kind: "host", callback: "models" }); + } +} + /** Settle the run's inputs `unanswered` with `no_model` and fail. */ async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise { const message = @@ -393,11 +603,28 @@ async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: await runtime.commit(async (tx) => { const failure = await prepareFailure(tx, runtime); const live = await tx.doc(LiveDoc, runtime.conversationId); - await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "no_model" }); + await endFailedRun(tx, runtime, live, failure, { + status: "unanswered", + reason: "no_model", + detail: message, + settlement: { status: "unanswered", reason: "no_model" }, + }); return { status: "terminal", outcome: { status: "failed", error: { message, detail: { reason: "no_model" } } } }; }, context); } +/** End the run without a request: its context has nothing the model would answer. */ +async function failNothingToAnswer(runtime: Runtime, context: Context): Promise { + const message = "The conversation has nothing to answer"; + await runtime.commit(async (tx) => { + const failure = await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "nothing_to_answer" }); + const error = { message, detail: { reason: "nothing_to_answer" } }; + return { status: "terminal", outcome: { status: "failed", error } }; + }, context); +} + /** What a failed run's end needs from the table, read before the commit's first table write. */ type Failure = { readonly policy: AfterFailedRun; readonly boundary?: Boundary }; @@ -417,14 +644,14 @@ async function endFailedRun( runtime: Runtime, live: Draft, failure: Failure, - settlement: SubmissionSettlement, + end: Extract, ): Promise { const conversationId = runtime.conversationId; - endRun(tx, live, runtime.taskId, settlement); + endRun(tx, live, runtime.taskId, end); if (failure.policy === "withdraw") await withdrawQueuedInputs(tx, conversationId, "run_failed"); if (failure.boundary === undefined) return; - const { users } = await applyBoundary(tx, failure.boundary, "final", runtime.now()); - if (users.length > 0) await startRun(tx, conversationId, live, users); + const { users, first } = await applyBoundary(tx, failure.boundary, "final", runtime.now()); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }, { keepFailure: true }); } /** @@ -438,11 +665,27 @@ export async function convertPartial(tx: Tx, live: Draft, conversatio await appendAssistant(tx, conversationId, { ...message, stopReason: "aborted" }); } +/** The `beforeProviderRequest` replacement chain over one provider request body. */ +async function providerPayload( + runtime: Runtime, + model: Model, + payload: unknown, + context: Context, +): Promise { + let current = payload; + await runtime.hooks.each("beforeProviderRequest", async (hook) => { + const replaced = await hook({ model, payload: current }, runtime, context); + if (replaced !== undefined) current = replaced.payload; + }); + return current; +} + /** * Stream one request and return the terminal message. Partials commit as trailing writes at most every - * `progress.partialIntervalMs` (default 100 ms) with one commit in flight; `finally` stops the throttle and awaits that - * commit, so no stale partial lands after the outcome. A commit appends what text and thinking grew by and applies - * the argument changes since the last one (`PartialChanges`), so its cost follows the new text, not the strings' length. + * `progress.partialIntervalMs` (default 100 ms) with one commit in flight, deferred (`PROGRESS_COMMIT`); `finally` stops + * the throttle and awaits that commit, so no stale partial lands after the outcome. A commit appends what text and + * thinking grew by and applies the argument changes since the last one (`PartialChanges`), so its cost follows the new + * text, not the strings' length. */ async function streamResponse( runtime: Runtime, @@ -465,15 +708,20 @@ async function streamResponse( if (partial === undefined || stopped) return; inFlight = (async () => { // Copy synchronously: the provider keeps mutating its partial. The changes since the last commit belong to it. - const message = copyJson(partial, { omitUndefinedProperties: true }) as { content: JsonValue[] }; + // `copyJson()` returns untyped JSON; a copy of a message is that message's JSON representation. + const message = copyJson(partial, { omitUndefinedProperties: true }) as PartialMessage; const taken = changes.take(message); try { - await runtime.commit(async (tx) => { - const live = await tx.doc(LiveDoc, runtime.conversationId); - live.generation ??= { attempt }; - changes.commit(live.generation as Draft>, message, taken); - return undefined; - }, context); + await runtime.commit( + async (tx) => { + const live = await tx.doc(LiveDoc, runtime.conversationId); + live.generation ??= { attempt }; + changes.commit(live.generation, message, taken); + return undefined; + }, + context, + PROGRESS_COMMIT, + ); } catch (error) { changes.restore(taken); throw error; @@ -516,7 +764,9 @@ async function classify( // An abort mark or close: the abort invocation or the reopened run handles the committed state. runtime.signal.throwIfAborted(); const conversationId = runtime.conversationId; - const { attempt, compacted, model: ref, cutoff } = request; + const { attempt, compacted, model: ref, thinkingLevel, cutoff } = request; + // Record the requested level, as pi-agent-core's agent loop does. + if (thinkingLevel !== undefined) message.thinkingLevel = thinkingLevel; if (message.stopReason === "deferred" && message.deferred !== undefined) { const handle = message.deferred; const pollAt = Math.max( @@ -530,6 +780,7 @@ async function classify( attempt, ...(compacted === undefined ? {} : { compacted }), model: ref, + ...(thinkingLevel === undefined ? {} : { thinkingLevel }), cutoff, handle, pollAt, @@ -600,8 +851,9 @@ async function classify( /** * A final answer; the final boundary places queued items (spec §6). The first `onYield` continuation appends a user - * message and hands the run to a successor generation, but only when the boundary selected no user item and no reset. - * Otherwise the run's inputs settle `done`, and selected user items start the next run. + * message and hands the run to a successor generation, but only when the inbox holds no user item and no reset. + * Otherwise the run's inputs settle `done`; with `afterRun` hooks selected the run waits for the tasks they ask for, + * in `finish` and `drain`, before the final boundary; without, selected user items start the next run now. */ async function answer(runtime: Runtime, message: AssistantMessage, context: Context): Promise { let continuation: UserInput | undefined; @@ -609,6 +861,7 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont if (continuation !== undefined) return; continuation = (await hook(message, runtime, context))?.continue; }); + const afterRun = await asksAfterRun(runtime, context); const conversationId = runtime.conversationId; await runtime.commit(async (tx): Promise => { // Queue modes are read on the Session line, when the boundary is decided. @@ -616,24 +869,142 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont const live = await tx.doc(LiveDoc, conversationId); const entry = await appendAssistant(tx, conversationId, message); const result: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; - const { users, reset } = await applyBoundary(tx, boundary, "final", runtime.now()); - if (continuation !== undefined && users.length === 0 && !reset) { - const user = { role: "user", content: continuation, timestamp: runtime.now() } as const; + const continues = takesContinuation(continuation, boundary) ? continuation : undefined; + if (continues === undefined && afterRun) { + const held = holdRun(tx, live, runtime, { reason: "answer", answer: entry.id, last: entry.id }); + if (held !== undefined) return held; + } + const { users, first } = await applyBoundary(tx, boundary, "final", runtime.now()); + if (continues !== undefined) { + const user = { role: "user", content: continues, timestamp: runtime.now() } as const; await tx.appendEntry(UserEntry, conversationId, { model: [user] }); + // The continuation starts the run's turn count over: bounding `onYield` loops is the extension's job. + if (live.run?.taskId === runtime.taskId) live.run.turns = 0; handOver(live, runtime.taskId, await createGeneration(tx, conversationId)); delete live.generation; return result; } endRun(tx, live, runtime.taskId, { status: "done", answer: entry.id }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); return result; }, context); } +/** Whether the phase's agent selects an `afterRun` hook, so a run that answers waits for its tasks (spec §8.3). */ +async function asksAfterRun(runtime: Runtime, context: Context): Promise { + const hooks = agentHooks(await runtime.agent(context), GenerationTask.definition.name); + return hooks.some(({ handlers }) => typeof (handlers as Partial).afterRun === "function"); +} + /** - * Append the tool-calling answer and start its tool round in one commit (spec §8.3). A call to a tool the request did - * not offer gets its `tool_unavailable` result here; every other call gets a tool task owned by the generation, only the - * first one now when the round is sequential. The generation then waits for them in its `tools` phase, keeping the run. + * Commit A of a run that answered with `afterRun` hooks selected (spec §8.3): settle the run's inputs `done` with the + * answer and keep `pi.live.run` for the after-run tasks, then ask the hooks in `finish`. The final boundary waits for + * the run's end, so queued items stay queued. + */ +function holdRun(tx: Tx, live: Draft, runtime: Runtime, end: Omit): Next | undefined { + const run = live.run; + if (run?.taskId !== runtime.taskId) return undefined; + for (const id of run.inputs) tx.settleSubmission(id, { status: "done", answer: end.answer }); + run.inputs = []; + run.afterRun = {}; + delete live.generation; + delete live.tools; + delete live.nestedTools; + // A run started before `first` was recorded has its answer as its first entry. + const summary: RunSummary = { ...end, first: run.first ?? end.answer }; + return { status: "running", checkpoint: { phase: "finish", run: summary } }; +} + +/** A task an `afterRun` hook asked for, checked and ready to create. */ +type PreparedTask = { readonly task: Task; readonly input: JsonValue }; + +/** + * Check the tasks one `afterRun` hook asked for, off the line, and add those that pass to `prepared`, so a broken request + * is skipped instead of failing the creating commit: its kind must be an installed task that is not built-in, its input + * JSON, and its definition's `initial()` must return a JSON checkpoint of one of its phases. The task is created from + * the installed definition, as every task runs under it, with that checkpoint, so `initial()` runs once. Then throws + * the rejections, which `hooks.each` reports against the hook's extension. + */ +function prepareRunEndTasks(runtime: Runtime, requests: readonly RunEndTask[], prepared: PreparedTask[]): void { + const rejected: unknown[] = []; + for (const request of requests) { + try { + const name = request.task.definition.name; + if (name.startsWith("pi.")) throw new Error(`An afterRun hook cannot start the built-in task ${name}`); + const installed = runtime.registry.task(name); + if (installed === undefined) throw new Error(`Task ${name} is not installed`); + const input = copyJson(request.input, { omitUndefinedProperties: true }); + // The registry erases definitions; an installed one is a task definition of its kind. + const definition = installed.definition as PreparedTask["task"]["definition"]; + const checkpoint = copyJson(definition.initial(input), { omitUndefinedProperties: true }) as { phase: string }; + if (!Object.hasOwn(definition.phases, checkpoint.phase)) { + throw new Error(`Task ${name} starts at the unknown phase ${String(checkpoint.phase)}`); + } + prepared.push({ task: { definition: { ...definition, initial: () => checkpoint } }, input }); + } catch (error) { + rejected.push(error); + } + } + if (rejected.length === 1) throw rejected[0]; + if (rejected.length > 1) + throw new AggregateError(rejected, `${rejected.length} afterRun task requests were rejected`); +} + +/** + * Commit B of a run that answered (spec §8.3): its after-run tasks are terminal, or there were none. Place what the + * final boundary selects, end the run, and start the next one for selected user items. + */ +async function endAnsweredRun(runtime: Runtime, answer: EntryId, context: Context): Promise { + const conversationId = runtime.conversationId; + await runtime.commit(async (tx): Promise => { + const boundary = await prepareBoundary(tx, conversationId, runtime.settings); + const live = await tx.doc(LiveDoc, conversationId); + const { users, first } = await applyBoundary(tx, boundary, "final", runtime.now()); + endRun(tx, live, runtime.taskId, { status: "done", answer }); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); + return { status: "terminal", outcome: { status: "completed", result: { entryId: answer } } }; + }, context); +} + +/** + * End a run that answered once its `finish` or `drain` phase was cut off `settings.restartLimit` times (spec §5.4): + * the run answered, so it ends `completed`, without a failure and without the tasks it would still have waited for, + * marked so the scheduler reports it `task_interrupted`. Queued items stay queued, as after a failed run. + */ +async function endAnsweredCutOff( + runtime: Runtime, + task: { readonly restarts?: number; readonly state: { readonly checkpoint: GenerationCheckpoint } }, + answer: EntryId, + context: Context, +): Promise { + const text = `The generation was interrupted ${cutOffCount(task)} in its ${task.state.checkpoint.phase} phase`; + await runtime.commit(async (tx): Promise => { + const live = await tx.doc(LiveDoc, runtime.conversationId); + endRun(tx, live, runtime.taskId, { status: "done", answer }); + const completed: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: answer } } }; + return endedByCrashes(completed, text); + }, context); +} + +/** + * Whether an answer's `onYield` continuation is taken (spec §8.3), decided from the inbox before the final boundary + * places anything: only when it holds no user item, of which the boundary would select one, and no reset, which would + * otherwise run first. A continuation starts the turn count over. + */ +function takesContinuation(continuation: UserInput | undefined, boundary: Boundary): continuation is UserInput { + const items = boundary.inbox.items; + return ( + continuation !== undefined && + !items.some((item) => item.mode === "steer" || item.mode === "followUp") && + !hasQueuedReset(items) + ); +} + +/** + * Append the tool-calling answer and start its tool round in one commit (spec §8.3). Every call gets a tool task owned by + * the generation, in call order, only the first one now when the round is sequential; a call to a tool the request did + * not offer is flagged, and its task settles `tool_unavailable`. The generation then waits for them in its `tools` + * phase, keeping the run. */ async function startToolRound( runtime: Runtime, @@ -661,31 +1032,34 @@ async function startToolRound( const tools: TaskId[] = []; const pending: string[] = []; for (const call of calls) { - if (!offered.has(call.name)) { - const unavailable = harnessError("tool_unavailable", `Tool ${call.name} is not available`); - const result = await appendToolResult(tx, conversationId, call, unavailable, { timestamp: runtime.now() }); - slots.push({ callId: call.id, name: call.name, status: "done", entry: result.id }); - continue; - } + const unoffered = offered.has(call.name) ? {} : ({ unoffered: true } as const); if (sequential && tools.length > 0) { pending.push(call.id); - slots.push({ callId: call.id, name: call.name, status: "pending" }); + slots.push({ callId: call.id, name: call.name, status: "pending", ...unoffered }); continue; } - const taskId = await createToolTask(tx, runtime, entry.id, call.id); + const taskId = await createToolTask(tx, runtime, entry.id, call.id, unoffered.unoffered); tools.push(taskId); - slots.push({ callId: call.id, name: call.name, taskId, status: "pending" }); + slots.push({ callId: call.id, name: call.name, taskId, status: "pending", ...unoffered }); } delete live.generation; live.tools = slots; + if (live.run?.taskId === runtime.taskId) { + // A tool round is one turn of the run (`AgentState.maxTurns`). + live.run.turns += 1; + // A run its boundary started without placing anything, as a continue after an answer, begins here. + live.run.first ??= entry.id; + } const checkpoint = { phase: "tools", assistant: entry.id, tools, pending } as const; return { status: "waiting", checkpoint, on: tools, policy: "allSettled" }; }, context); } /** - * The round's tools are terminal: apply their controls and either end the run at the final boundary (`terminate`, - * `handoff`, or a queued reset) or hand it to the next generation at the `postTools` boundary (spec §8.5). + * The round's tools are terminal: apply their controls, append a `handoff`'s reset, and then end the run at the final + * boundary (`terminate`), end it like a failure at the conversation's turn limit (`turn_limit`), end it at a queued + * reset, or hand it to the next generation at the `postTools` boundary, in the handoff's new context when there is one + * (spec §8.5). */ async function finishToolRound( runtime: Runtime, @@ -707,33 +1081,55 @@ async function finishToolRound( const slots = (await runtime.snapshot(LiveDoc, conversationId, context))?.tools ?? []; const results = slots.flatMap((slot) => (slot.entry === undefined ? [] : [slot.entry])); await runtime.hooks.each("afterTools", (hook) => hook(assistant, results, runtime, context)); - // Every call of the round, including those answered without a task, must ask to terminate. - const terminate = - slots.length > 0 && - slots.every((slot) => slot.taskId !== undefined && controls.get(slot.taskId)?.terminate === true); - const added = [...controls.values()].flatMap((control) => control?.addTools ?? []); + // In call order, the slots' order; a slot without a task (faulted or orphaned before it got one) has no control. + const ordered = slots.map((slot) => (slot.taskId === undefined ? undefined : controls.get(slot.taskId))); + // Every call of the round must ask to terminate. + const terminate = slots.length > 0 && ordered.every((control) => control?.terminate === true); + const added = ordered.flatMap((control) => control?.addTools ?? []); // The last handoff in call order wins. - const handoff = [...controls.values()].findLast((control) => control?.handoff !== undefined)?.handoff; + const handoff = ordered.findLast((control) => control?.handoff !== undefined)?.handoff; + // Read from the phase's agent, as every agent field; a `configure()` during the round applies at its end. + const maxTurns = (await runtime.agent(context)).maxTurns; + const afterRun = terminate && (await asksAfterRun(runtime, context)); await runtime.commit(async (tx): Promise => { const boundary = await prepareBoundary(tx, conversationId, runtime.settings); if (added.length > 0) await addTools(tx, conversationId, added); const live = await tx.doc(LiveDoc, conversationId); const now = runtime.now(); - if (terminate || handoff !== undefined) { - if (handoff !== undefined) { - const message = { role: "user", content: handoff, timestamp: now } as const; - const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); - boundary.head = entry.id; - } - const { users } = await applyBoundary(tx, boundary, "final", now); + // The run's last entry so far: the newest result, or the assistant when no result has an entry. + let last = results.reduce((newest, id) => (id > newest ? id : newest), assistant); + if (handoff !== undefined) { + // The handoff's reset precedes every queued item, so queued writes and selected steers land in its context. + const message = { role: "user", content: handoff, timestamp: now } as const; + const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); + boundary.head = entry.id; + last = entry.id; + } + const turns = live.run?.taskId === runtime.taskId ? live.run.turns : 0; + const held = afterRun ? holdRun(tx, live, runtime, { reason: "terminate", answer: assistant, last }) : undefined; + if (held !== undefined) return held; + if (terminate) { + const { users, first } = await applyBoundary(tx, boundary, "final", now); endRun(tx, live, runtime.taskId, { status: "done", answer: assistant }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); + } else if (maxTurns !== undefined && turns >= maxTurns) { + // The turn limit ends the run like a failure: `afterFailedRun` decides the queued items, and a continue resumes + // it on the round's results. The table was read for the boundary already, before any write. + const policy = runtime.settings.afterFailedRun; + const failure = { policy, ...(policy === "continue" ? { boundary } : {}) }; + await endFailedRun(tx, runtime, live, failure, { + status: "unanswered", + reason: "turn_limit", + detail: turns === 1 ? "1 turn" : `${turns} turns`, + }); } else { - const { users, reset } = await applyBoundary(tx, boundary, "postTools", now); + const { users, first, reset } = await applyBoundary(tx, boundary, "postTools", now); if (reset) { // The queued reset cut the run's context before an answer. endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "reset" }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) { + await startRun(tx, conversationId, live, { inputs: users, first }, { keepFailure: true }); + } } else { delete live.tools; delete live.nestedTools; @@ -758,14 +1154,22 @@ async function appendAssistant( return tx.appendEntry(AssistantEntry, conversationId, { model: [message] }); } -/** Start a run for `inputs`, placed input submissions: a new generation takes `pi.live.run`. */ +/** + * Start a run for `inputs`, placed input submissions, whose first entry is `first` (`LiveRun.first`): a new generation + * takes `pi.live.run`. Removes `runFailure`, unless `keepFailure`: a run started in the commit that ended the failed one + * keeps its failure. + */ export async function startRun( tx: Tx, conversationId: ConversationId, live: Draft, - inputs: SubmissionId[], + run: { readonly inputs: SubmissionId[]; readonly first: EntryId | undefined }, + options?: { readonly keepFailure?: boolean }, ): Promise { - live.run = { taskId: await createGeneration(tx, conversationId, { submissionIds: inputs }), inputs }; + const { inputs, first } = run; + const taskId = await createGeneration(tx, conversationId, { submissionIds: inputs }); + live.run = { taskId, inputs, turns: 0, ...(first === undefined ? {} : { first }) }; + if (options?.keepFailure !== true) delete live.runFailure; } /** A generation owned by its conversation. */ diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index d22f368cc..8fb91bcc0 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -20,6 +20,7 @@ import { contained, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import { hostTimers } from "../timers.ts"; import type { + CommitOptions, ConversationId, ConversationOwnership, ConversationRecord, @@ -27,7 +28,10 @@ import type { EntryId, EntryQuery, EntryRecord, + Observation, + Observer, Page, + Report, ReportInfo, Storage, SubmissionId, @@ -40,12 +44,12 @@ import { AgentDoc, configure, createAgent, resolveAgent, resolveSettings } from import { createCompaction } from "./compaction.ts"; import { readContext } from "./context.ts"; import { InboxDoc, withdrawQueuedInputs } from "./inbox.ts"; -import { LiveDoc, settleSchedulerOutcome } from "./live.ts"; +import { LIMITS_RESTARTS, LiveDoc, settleSchedulerOutcome } from "./live.ts"; import { ProviderDoc } from "./provider.ts"; import { BUILTIN_TASKS } from "./registry.ts"; import { type InvocationBinding, TaskScheduler } from "./scheduler.ts"; import { Submissions } from "./submissions.ts"; -import { TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; +import { type TaskGraph, TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.ts"; import type { Agent, @@ -70,8 +74,8 @@ import type { Wake, } from "./types.ts"; import { addUsageState, UsageDoc, type UsageState } from "./usage.ts"; -import { scanAll } from "./util.ts"; -import { ConversationViews } from "./view.ts"; +import { conversationEntries, scanAll } from "./util.ts"; +import { type ConversationView, ConversationViews, type ConversationWindow } from "./view.ts"; const SCAN_PAGE_SIZE = 256; @@ -102,6 +106,9 @@ type ConversationHost = { /** Key of the Harness's internal index-size getter, for tests. */ export const SCHEDULER_INDEX_SIZES = Symbol.for("pi-durable.schedulerIndexSizes"); +/** Key of the Harness's internal idle-state getter (kept contexts against a full recompute), for tests. */ +export const SCHEDULER_IDLE_STATE = Symbol.for("pi-durable.schedulerIdleState"); + class ConversationImpl implements Conversation { readonly id: ConversationId; readonly #host: ConversationHost; @@ -119,8 +126,12 @@ class ConversationImpl implements Conversation { return this.#host.harness.commitWith((tx) => configure(tx, this.id, change), context); } - submit(submission: SubmissionDraft, context: Context): Promise { - return this.#host.submissions.submit(this.id, submission, context); + submit(submission: SubmissionDraft, context: Context): Promise; + submit(tx: Tx, submission: SubmissionDraft): Promise; + submit(first: SubmissionDraft | Tx, second: Context | SubmissionDraft): Promise { + // The overloads pair a draft with a context, and anything else (a transaction) with a draft. + if ("type" in first) return this.#host.submissions.submit(this.id, first, second as Context); + return this.#host.submissions.admit(first, this.id, second as SubmissionDraft); } async compact(instructions: string | undefined, context: Context): Promise> { @@ -138,8 +149,8 @@ class ConversationImpl implements Conversation { await this.#host.submissions.submit(this.id, { type: "write", entry }, context); } - commit(change: (tx: Tx) => T | Promise, context: Context): Promise { - return this.#host.harness.commitWith(change, context, { conversationId: this.id }); + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise { + return this.#host.harness.commitWith(change, context, { conversationId: this.id }, options); } context(context: Context, options?: { readonly at?: EntryId }): Promise { @@ -152,13 +163,8 @@ class ConversationImpl implements Conversation { cursor: Cursor | undefined, context: Context, ): Promise> { - const bounded: EntryQuery = { - conversationId: this.id, - ...(query.minEntryId === undefined ? {} : { minEntryId: query.minEntryId }), - ...(query.maxEntryId === undefined ? {} : { maxEntryId: query.maxEntryId }), - ...(query.order === undefined ? {} : { order: query.order }), - }; - return this.#host.harness.readOnLine(() => this.#host.storage.scanEntries(bounded, limit, cursor, context)); + const scan = conversationEntries(this.id, query); + return this.#host.harness.readOnLine((storage) => storage.scanEntries(scan, limit, cursor, context)); } fork(at: EntryId, options: ConversationCreateOptions, context: Context): Promise { @@ -175,8 +181,8 @@ class ConversationImpl implements Conversation { return this.#host.tasks.waitForIdle(this.id, context); } - watch(context: Context): Promise { - return this.#host.views.watch(this.id, context); + watch(context: Context, window?: ConversationWindow): Promise { + return this.#host.views.watch(this.id, context, window); } } @@ -213,8 +219,15 @@ class HarnessImpl extends SessionImpl implements const report = safeReport(options.onReport, options.telemetry ?? NOOP_TELEMETRY_CONTEXT); const now = safeNow(options.now, report); const hostSettings = currentSettings(options.settings); - // Read at each use, as all settings are, but only this field: it is read on every commit. - super(rawStorage, now, () => hostSettings()?.documentRetentionMs, options.setTimer, options.readOnly === true); + // Read at each use, as all settings are, but only these fields: they are read on every commit. + super( + rawStorage, + now, + () => hostSettings()?.documentRetentionMs, + options.setTimer, + options.readOnly === true, + () => hostSettings()?.progress?.persistWithinMs, + ); // Every component reads and writes through the Session's failure guard: `rawStorage` serves only `super()`. const storage = this.storage; this.#options = options; @@ -232,20 +245,21 @@ class HarnessImpl extends SessionImpl implements env: (id, callContext) => this.buildEnv(id, callContext), now, onWake: options.onWake, - timers: hostTimers(options.setTimer, (error) => report(error, { type: "host_error", callback: "setTimer" })), + timers: hostTimers(options.setTimer, (error) => report(error, { kind: "host_error", callback: "setTimer" })), writeAtClose: (writes) => this.writeAtClose(writes), report: this.#report, + limitsRestarts: (kind) => LIMITS_RESTARTS.has(kind), settleOutcome: settleSchedulerOutcome, withdrawInputs: withdrawQueuedInputs, conversation: async (id, binding, callContext) => { - const record = await this.readOnLine(() => storage.conversation(id, callContext)); + const record = await this.readOnLine((line) => line.conversation(id, callContext)); return record === undefined ? undefined : boundConversation(id, binding, this.#submissions, this.#tasks); }, context: withoutAbortSignal(context), readOnly: options.readOnly === true, }); - this.#submissions = new Submissions(this, storage, now, settings, () => this.#tasks.resume(), options.telemetry); - this.#taskGraph = new TaskGraphView(this, storage, { + this.#submissions = new Submissions(this, now, settings, () => this.#tasks.resume(), options.telemetry); + this.#taskGraph = new TaskGraphView(this, { blocked: (record) => this.#tasks.blockedReason(record), subscribeBlocked: (listener) => this.#tasks.subscribeBlocked(listener), }); @@ -254,7 +268,7 @@ class HarnessImpl extends SessionImpl implements storage, tasks: this.#tasks, submissions: this.#submissions, - views: new ConversationViews(this, storage), + views: new ConversationViews(this), now, create: (target, createOptions, context) => this.#create(target, createOptions, context), }; @@ -269,7 +283,7 @@ class HarnessImpl extends SessionImpl implements const registry = snapshot ?? this.#options.registry.snapshot(); const state = await this.snapshot(AgentDoc, id, context); return resolveAgent(state, registry, resolveSettings(this.#hostSettings()), (error, extension) => - this.#report(error, { type: "extension_error", conversationId: id, extension }, context), + this.#report(error, { kind: "extension_error", conversationId: id, extension }, context), ); } @@ -297,7 +311,7 @@ class HarnessImpl extends SessionImpl implements } getTask(id: TaskId, context: Context): Promise | undefined> { - return this.readOnLine(() => this.storage.task(id, context)) as Promise< + return this.readOnLine((storage) => storage.task(id, context)) as Promise< TaskRecord | undefined >; } @@ -307,11 +321,16 @@ class HarnessImpl extends SessionImpl implements return this.#tasks.indexSizes; } + /** Internal, for tests: the kept contexts' idle state and its disagreements with a full recompute. */ + get [SCHEDULER_IDLE_STATE](): TaskScheduler["idleState"] { + return this.#tasks.idleState; + } + inspect(context: Context): Promise { - return this.readOnLine(async () => { + return this.readOnLine(async (storage) => { const { scheduling, tasks } = await this.#tasks.inspect(this.#options.registry.snapshot()); const scan = (status: "queued" | "placed") => - scanAll((cursor) => this.storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); + scanAll((cursor) => storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); const submissions = [...(await scan("queued")), ...(await scan("placed"))].sort((a, b) => a.id - b.id); return { scheduling, tasks, submissions }; }); @@ -333,11 +352,6 @@ class HarnessImpl extends SessionImpl implements return this.#tasks.abort(id, context); } - retryTask(id: TaskId, context: Context): Promise { - this.assertUsable(); - return this.#tasks.retry(id, context); - } - async waitForTask(id: TaskId, context: Context): Promise> { this.#tasks.resume(); return this.#tasks.waitForTask(id, context) as Promise>; @@ -350,8 +364,8 @@ class HarnessImpl extends SessionImpl implements /** Sum every conversation's committed `pi.usage`. Each document is read at its own point; totals only grow. */ async usage(context: Context): Promise { - const conversations = await this.readOnLine(() => - scanAll((cursor) => this.storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), + const conversations = await this.readOnLine((storage) => + scanAll((cursor) => storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), ); const total = UsageDoc.definition.initial(); for (const { id } of conversations) { @@ -365,6 +379,21 @@ class HarnessImpl extends SessionImpl implements return this.#taskGraph.watch(context); } + async observeTaskGraph(observer: Observer, context: Context): Promise> { + this.assertUsable(); + return this.#taskGraph.observe(observer, context); + } + + async observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + window?: ConversationWindow, + ): Promise> { + this.assertUsable(); + return this.#host.views.observe(id, observer, context, window); + } + root( context: Context, options?: { readonly agent?: AgentChange; readonly init?: ConversationInit }, @@ -374,7 +403,7 @@ class HarnessImpl extends SessionImpl implements async conversation(id: ConversationId, context: Context): Promise { this.assertUsable(); - const record = await this.readOnLine(() => this.storage.conversation(id, context)); + const record = await this.readOnLine((storage) => storage.conversation(id, context)); return record === undefined ? undefined : new ConversationImpl(record.id, this.#host); } @@ -382,10 +411,15 @@ class HarnessImpl extends SessionImpl implements return this.#create({ kind: "independent", ownership: options.ownership }, options, context); } - override report(error: unknown, info: ReportInfo = { type: "listener_error" }, context?: Context): void { + override report(error: unknown, info: ReportInfo = { kind: "listener_error" }, context?: Context): void { this.#report(error, info, context); } + /** The ownership memberships a commit's task changes write, from the scheduler's index. */ + protected override membershipKeys(tx: Transaction): readonly string[] | Promise { + return this.#tasks.membershipKeys(tx); + } + /** Signal and join task invocations after the close listeners and before Storage closes; writes no task outcome. */ protected override beforeClose(): Promise { return this.#tasks.join(); @@ -425,30 +459,35 @@ class HarnessImpl extends SessionImpl implements } } -/** Preserve the host callback's error containment; telemetry adapters are responsible for their own failures. */ +/** + * Report to `onReport`, then count it and record its event. Never throws, since reports also come from inside Storage's + * `adopt`: a throwing `onReport` is dropped, and so is a throwing metric or event, as reporting it would report again. + */ function safeReport( - onReport: ((error: unknown, info: ReportInfo) => void) | undefined, + onReport: ((report: Report) => void) | undefined, telemetry: TelemetryContext, ): (error: unknown, info: ReportInfo, context?: Context) => void { const reports = createTypedMetrics(telemetry.metrics ?? NOOP_TELEMETRY_METRICS, [DURABLE_TELEMETRY_SCHEMA]).counter( "pi.durable.reports", ); return (error, info, context) => { - contained( - () => onReport?.(error, info), - () => {}, - ); + if (onReport !== undefined) { + contained( + () => onReport({ ...info, error }), + () => {}, + ); + } const attributes: SpanAttributes = { - type: info.type, + type: info.kind, }; - switch (info.type) { + switch (info.kind) { case "task_faulted": + case "task_interrupted": case "task_outdated": attributes.reason = info.reason; attributes.extension = info.extension; attributes["task.id"] = info.task.id; break; - case "task_parked": case "migration_failed": case "task_error": case "extension_error": @@ -459,8 +498,12 @@ function safeReport( attributes["task.id"] = info.task?.id; break; } - reports.add(1, { type: info.type }); - context?.telemetry.addEvent("pi.durable.report", attributes); + try { + reports.add(1, { type: info.kind }); + context?.telemetry.addEvent("pi.durable.report", attributes); + } catch { + // Dropped: see above. + } }; } @@ -481,7 +524,7 @@ export function safeNow( // Set first: a report handler that reads the clock again must not report again. if (!reported) { reported = true; - report(error, { type: "host_error", callback: "now" }); + report(error, { kind: "host_error", callback: "now" }); } return Date.now(); } @@ -505,18 +548,30 @@ function boundConversation( return operation(bind(context)); }; }; + function submit(draft: SubmissionDraft, context: Context): Promise; + function submit(tx: Tx, draft: SubmissionDraft): Promise; + async function submit( + first: SubmissionDraft | Tx, + second: Context | SubmissionDraft, + ): Promise { + binding.check(); + // The overloads pair a draft with a context, and anything else (a transaction) with a draft. + if (!("type" in first)) { + // The caller's commit carries its own context, so the invocation's abort is checked here. + binding.signal.throwIfAborted(); + return submissions.admit(first, id, second as SubmissionDraft); + } + const submission = await submissions.submit(id, first, bind(second as Context)); + return { + id: submission.id, + status: bound((callContext) => submission.status(callContext)), + wait: bound((callContext) => submission.wait(callContext)), + abort: bound((callContext) => submission.abort(callContext)), + }; + } return { id, - submit: async (draft, context) => { - binding.check(); - const submission = await submissions.submit(id, draft, bind(context)); - return { - id: submission.id, - status: bound((callContext) => submission.status(callContext)), - wait: bound((callContext) => submission.wait(callContext)), - abort: bound((callContext) => submission.abort(callContext)), - }; - }, + submit, abort: async (context, options) => { binding.check(); return tasks.abortConversation(id, options?.background === true, bind(context)); diff --git a/packages/durable/src/harness/inbox.ts b/packages/durable/src/harness/inbox.ts index ced697767..3469e8e79 100644 --- a/packages/durable/src/harness/inbox.ts +++ b/packages/durable/src/harness/inbox.ts @@ -41,8 +41,15 @@ export type Boundary = { head: EntryId | undefined; }; -/** Selected user items, in ID order, and whether a `head: "self"` write (a reset) was placed. */ -export type BoundaryResult = { readonly users: SubmissionId[]; readonly reset: boolean }; +/** + * Selected user items, in ID order, the first entry the boundary placed, a write's or a user item's (a run it starts + * begins there, `LiveRun.first`), and whether a `head: "self"` write (a reset) was placed. + */ +export type BoundaryResult = { + readonly users: SubmissionId[]; + readonly first: EntryId | undefined; + readonly reset: boolean; +}; /** * Read what a boundary needs. Table reads must precede the commit's first table write, so callers prepare the @@ -86,6 +93,7 @@ export async function applyBoundary( users.sort((a, b) => a - b); // `appendEntry()` copies the drafts' values; the items are removed only afterwards. + let first: EntryId | undefined; for (const index of writes) { const item = items[index] as Draft>; const draft = item.entry as unknown as EntryDraft; @@ -96,6 +104,7 @@ export async function applyBoundary( const entry = await tx.appendEntry(conversationId, draft); if (draft.head !== undefined) boundary.head = draft.head === "self" ? entry.id : draft.head; tx.placeSubmission(item.id, entry.id); + first ??= entry.id; } const placed: SubmissionId[] = []; for (const index of users) { @@ -103,10 +112,11 @@ export async function applyBoundary( const entry = await appendInput(tx, conversationId, item.content as UserInput, item.entry as InputEntry, now); tx.placeSubmission(item.id, entry.id); placed.push(item.id); + first ??= entry.id; } const removed = [...writes, ...users].sort((a, b) => b - a); for (const index of removed) items.splice(index, 1); - return { users: placed, reset }; + return { users: placed, first, reset }; } /** Append the entry an input is placed as: `entry`'s kind and data, or `pi.user`, with the user message of `content`. */ diff --git a/packages/durable/src/harness/json.ts b/packages/durable/src/harness/json.ts index c96119fe8..5b218fb43 100644 --- a/packages/durable/src/harness/json.ts +++ b/packages/durable/src/harness/json.ts @@ -1,8 +1,6 @@ import type { JsonValue } from "@earendil-works/chord"; import { appendText, type Draft } from "@earendil-works/chord/delta"; -type JsonContainer = Record | JsonValue[]; - /** How `assignJson()` writes one value instead of assigning it leaf by leaf. */ export type AssignPlan = /** The string grew by `text` since it was last assigned: append it, when the lengths agree. */ @@ -19,10 +17,18 @@ export type AssignPlan = * string member grew by, and the committed string plus that text has the new string's length, the text is appended * with `appendText()` instead. */ -export function assignJson(target: JsonContainer, key: string | number, value: JsonValue, plan?: AssignPlan): void { +export function assignJson( + target: T, + key: K, + value: T[K] & JsonValue, + plan?: AssignPlan, +): void { if (plan?.kind === "keep") return; + // Every target is a document container: JSON members or items, whatever its declared type names them. const slots = target as Record; - const current = slots[key]; + // A `__proto__` member the target lacks reads through to Object.prototype; merging into that would pollute every + // object in the process. Other inherited members are functions, which are not merged into. + const current: JsonValue | undefined = key === "__proto__" && !Object.hasOwn(slots, key) ? undefined : slots[key]; const fields = plan?.kind === "fields" ? plan.fields : undefined; if (isRecord(current) && isRecord(value)) { for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name]; diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 4d5397f71..9888a402e 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -2,7 +2,16 @@ import type { Draft, JsonRepresentation, JsonValue } from "@earendil-works/chord import type { AssistantMessage, Usage } from "@earendil-works/pi-ai"; import { defineDoc } from "../documents.ts"; import type { Transaction } from "../session/transaction.ts"; -import type { EntryId, JsonObject, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; +import type { + CommitOptions, + EntryId, + JsonObject, + SubmissionId, + SubmissionSettlement, + TaskId, + TaskRecord, + Tx, +} from "../types.ts"; import { convertPartial } from "./generation.ts"; import type { SchedulerOutcome } from "./scheduler.ts"; import type { CompactionReason, CompactionResult, ToolDiagnostic } from "./types.ts"; @@ -24,18 +33,17 @@ type SlotProgress = { /** Presentation of one tool call of the current round. */ export type ToolSlot = SlotProgress & { - /** - * Absent for a call not started yet (sequential round) and for a call its request did not offer, which starts `done` - * with the `entry` generation wrote. - */ + /** Absent for a call not started yet (sequential round). */ taskId?: TaskId; + /** A call to a tool its request did not offer: its task settles `tool_unavailable` without running anything. */ + unoffered?: true; /** Result entry once done; absent when the tool task faulted or was orphaned. */ entry?: EntryId; }; /** * Presentation of one nested call: a call a running tool made through `executeTool()`. `callId` is - * `/`, the call's ID in tool events and unique because keys contain no `/`. The Harness relates + * `/`, the call's ID, unique because keys contain no `/`. The Harness relates * slots by task ID: `taskId` is the call's tool task, `parentTaskId` the calling one, model-issued or nested. */ export type NestedToolSlot = SlotProgress & { @@ -62,7 +70,8 @@ export type NestedToolSummary = { /** Presentation of one live compaction task (spec §8.7). */ export type CompactionStatus = { taskId: TaskId; - reason: CompactionReason; + /** What started it: `compact()`, a threshold, or a context overflow. */ + trigger: CompactionReason; /** Whether a generation waits for it: a compaction the generation owns. */ blocking: boolean; attempt: number; @@ -73,15 +82,83 @@ export type CompactionStatus = { /** The newest compaction that failed since a compaction last produced a summary (spec §8.7). */ export type CompactionFailure = { taskId: TaskId; - reason: CompactionReason; + /** What started the compaction that failed. */ + trigger: CompactionReason; /** The failed or faulted outcome's error message, or why the scheduler orphaned the task. */ error: string; }; +/** + * Why a run ended without an answer: + * - `model_error`: the model's response failed, after any retries, or the compaction a request needed failed; + * - `no_model`: no model is configured, or the configured one is not available; + * - `aborted`: the run was aborted (`Conversation.abort()`, `abortTask()`); `detail` says when its abort handler was + * cut off `settings.restartLimit` times and did not finish; + * - `reset`: a queued reset started a new context at the post-tools boundary, before an answer (a tool's handoff + * continues the run); + * - `faulted`: the run's task threw where it cannot recover, or storage rejected its commit; + * - `orphaned`: the run's task was aborted while no registered definition could take it; + * - `interrupted`: the host stopped while the model was answering, more often than `settings.retry` retries; + * - `restart_limit`: the run's generation was cut off `settings.restartLimit` times in a row; a request or poll it was + * cut off during counts as a failed attempt first, so this ends one whose charge itself keeps failing; + * - `nothing_to_answer`: the context did not end in a user message or tool results (empty, or ending in an answer), so + * no request was sent, as for a continue after an answer; + * - `turn_limit`: a tool round reached the conversation's `maxTurns`; a continue resumes on its results. + */ +export type RunFailureReason = + | "model_error" + | "no_model" + | "aborted" + | "reset" + | "faulted" + | "orphaned" + | "interrupted" + | "restart_limit" + | "nothing_to_answer" + | "turn_limit"; + +/** The newest run that ended without an answer (spec §8.2). */ +export type RunFailure = { + /** The task that ended the run: the run's task at its end. */ + taskId: TaskId; + /** The run's inputs, settled `unanswered`. */ + inputs: SubmissionId[]; + reason: RunFailureReason; + /** + * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`); `interrupted` and `restart_limit`: what + * was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. + */ + detail?: string; + /** The failed run's `run.first`, so a continue's `afterRun` hooks see its history too. */ + first?: EntryId; +}; + +/** Run control (spec §8.2). */ +export type LiveRun = { + /** The task that settles the run's inputs: its generation, handed from one to the next. */ + taskId: TaskId; + /** The run's placed inputs, settled when it ends; `[]` once it answered and runs only its `afterRun` tasks. */ + inputs: SubmissionId[]; + /** Tool rounds so far, which `AgentState.maxTurns` bounds; an `onYield` continuation starts the count over. */ + turns: number; + /** + * The run's first entry: the first one the boundary that started it placed, a continue's the failed run's it + * continues; otherwise its first tool-calling answer. A run started before it was recorded may lack it; `afterRun` + * hooks then get the answer as the first entry. + */ + first?: EntryId; + /** + * Present once the run answered and its generation waits for the tasks `afterRun` hooks asked for, listed once + * created: queued items stay queued until they are terminal, and the run then ends without a failure. + */ + afterRun?: { tasks?: TaskId[] }; +}; + /** Built-in live conversation state: run control and presentation of the current generation and tool round. */ export type LiveState = { - /** Run control: the task that settles the run's inputs, and those inputs; present exactly while busy. */ - run?: { taskId: TaskId; inputs: SubmissionId[] }; + /** Run control; present exactly while busy, also while a run that answered runs its `afterRun` tasks. */ + run?: LiveRun; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -106,15 +183,34 @@ export type LiveState = { * removes it; one that is aborted or finds nothing to compact leaves it. */ compactionFailure?: CompactionFailure; + /** + * Why the newest run ended without an answer, so a client can show it after the run ended. Set in the commit that + * ends the run; removed when a later commit starts a run. A run started in the same commit, by + * `afterFailedRun: "continue"` or after a reset, keeps it. + */ + runFailure?: RunFailure; }; export const LiveDoc = defineDoc({ kind: "pi.live", - version: 1, + version: 3, scope: "conversation", history: "latest", fork: "initial", initial: () => ({}), + // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`. Versions 1 and 2 (as + // builds before `run.turns` stored it) may lack `run.turns`: a run in flight across the upgrade counts its turns + // from 0. Version 2 already names the trigger `trigger`, so renaming leaves it alone. + migrate: (value) => { + const live = value as { run?: JsonObject; compactions?: JsonObject[]; compactionFailure?: JsonObject }; + const { run, compactions, compactionFailure } = live; + return { + ...value, + ...(run === undefined ? {} : { run: { ...run, turns: typeof run.turns === "number" ? run.turns : 0 } }), + ...(compactions === undefined ? {} : { compactions: compactions.map(renameReason) }), + ...(compactionFailure === undefined ? {} : { compactionFailure: renameReason(compactionFailure) }), + } as LiveState; + }, // REMINDER: a complete base whenever nothing runs (spec §8.2): no generation and no running tool slot. That holds // while idle, in the commit handing a generation over to its tool round, and between tools, so the delta chain // spans at most one generation or the overlapping execution of one round's tools. A slot holds output only while @@ -126,18 +222,93 @@ export const LiveDoc = defineDoc({ !(value.nestedTools ?? []).some((slot) => slot.status === "running"), }); +/** A copy of a version 1 compaction status or failure with `reason` renamed `trigger`, its keys in their order. */ +function renameReason(value: JsonObject): JsonObject { + return Object.fromEntries(Object.entries(value).map(([key, field]) => [key === "reason" ? "trigger" : key, field])); +} + +/** + * How the progress throttles commit a partial answer and running tool output into `pi.live`: deferred, so storage gets + * them together, at the next commit that reads or writes `pi.live` or after `settings.progress.persistWithinMs`. Every + * commit that consumes or clears streamed fields reads `pi.live`, so it stores them with its own changes (spec §8.2). + */ +export const PROGRESS_COMMIT: CommitOptions = { persistence: { kind: "deferred" } }; + /** Built-in task kinds that can own `pi.live.run`. */ const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation"]); const TOOL_TASK_KIND = "pi.tool"; const COMPACTION_TASK_KIND = "pi.compaction"; /** - * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation`, `tools`, and - * `nestedTools`, whose presentation belongs to the ending run. + * Built-in task kinds that end themselves once cut off `settings.restartLimit` times (`TaskRecord.restarts`); the + * scheduler faults a task of any other kind instead (spec §5.4). */ -export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlement: SubmissionSettlement): void { +export const LIMITS_RESTARTS: ReadonlySet = new Set([...RUN_TASK_KINDS, TOOL_TASK_KIND, COMPACTION_TASK_KIND]); + +/** Whether `task` was cut off `settings.restartLimit` times in a row, so its phase ends it instead of running. */ +export function cutOffTooOften( + task: { readonly restarts?: number }, + runtime: { readonly settings: { readonly restartLimit: number } }, +): boolean { + // The settings are read only for a task that was cut off at all, so the normal path resolves nothing more. + return task.restarts !== undefined && task.restarts >= runtime.settings.restartLimit; +} + +/** + * Whether a task at the limit may still do the cleanup it does when it ends, such as converting a partial: only at the + * limit. Past it, that cleanup may be what cut the task off, so the task ends without it. + */ +export function salvageable( + task: { readonly restarts?: number }, + runtime: { readonly settings: { readonly restartLimit: number } }, +): boolean { + return (task.restarts ?? 0) <= runtime.settings.restartLimit; +} + +/** How often a task was cut off, for a message: "once", or "3 times in a row". */ +export function cutOffCount(task: { readonly restarts?: number }): string { + const restarts = task.restarts ?? 0; + return restarts === 1 ? "once" : `${restarts} times in a row`; +} + +/** How a run ends: with an answer, or without one, settling its inputs `unanswered` and recording why. */ +export type RunEnd = + | { readonly status: "done"; readonly answer: EntryId } + | { + readonly status: "unanswered"; + readonly reason: RunFailureReason; + readonly detail?: string; + /** The inputs' settlement; default `unanswered` with `reason` and `detail`. */ + readonly settlement?: SubmissionSettlement; + }; + +/** + * End the run owned by `taskId`: settle each of its inputs, remove `run`, and, without an answer, record + * `runFailure`. A run that answered already, and runs only its `afterRun` tasks, records no failure, however it ends. + * Always removes `generation`, `tools`, and `nestedTools`, whose presentation belongs to the ending run. + */ +export function endRun(tx: Tx, live: Draft, taskId: TaskId, end: RunEnd): void { if (live.run?.taskId === taskId) { - for (const id of live.run.inputs) tx.settleSubmission(id, settlement); + const inputs = [...live.run.inputs]; + const { first } = live.run; + if (end.status === "done") { + for (const id of inputs) tx.settleSubmission(id, { status: "done", answer: end.answer }); + } else if (live.run.afterRun === undefined) { + const { reason, detail } = end; + const settlement = end.settlement ?? { + status: "unanswered", + reason, + ...(detail === undefined ? {} : { detail }), + }; + for (const id of inputs) tx.settleSubmission(id, settlement); + live.runFailure = { + taskId, + inputs, + reason, + ...(detail === undefined ? {} : { detail }), + ...(first === undefined ? {} : { first }), + }; + } delete live.run; } delete live.generation; @@ -168,7 +339,7 @@ export function removeCompactionStatus(live: Draft, taskId: TaskId): /** Remove the status of compaction task `taskId` and record that it failed with `error`. */ export function failCompactionStatus(live: Draft, taskId: TaskId, error: string): void { const status = compactionStatus(live, taskId); - if (status !== undefined) live.compactionFailure = { taskId: status.taskId, reason: status.reason, error }; + if (status !== undefined) live.compactionFailure = { taskId: status.taskId, trigger: status.trigger, error }; removeCompactionStatus(live, taskId); } @@ -248,21 +419,33 @@ export async function settleSchedulerOutcome( removeNestedSlots(live, record.id); return; } + // Only an abort handler is faulted for the limit here (the built-in phases end themselves): the task was aborted, + // and keeps that meaning. Its run ends `aborted`, with the limit as detail, and an aborted compaction is no failure. + const detail = outcome.status === "faulted" ? outcome.error.detail : undefined; + const cutOff = + typeof detail === "object" && detail !== null && "reason" in detail && detail.reason === "restart_limit"; if (record.kind === COMPACTION_TASK_KIND) { - const error = outcome.status === "faulted" ? outcome.error.message : outcome.reason; - failCompactionStatus(await tx.doc(LiveDoc, record.conversationId), record.id, error); + const live = await tx.doc(LiveDoc, record.conversationId); + if (cutOff) removeCompactionStatus(live, record.id); + else failCompactionStatus(live, record.id, outcome.status === "faulted" ? outcome.error.message : outcome.reason); return; } if (!RUN_TASK_KINDS.has(record.kind)) return; const live = await tx.doc(LiveDoc, record.conversationId); if (live.run?.taskId !== record.id) return; - await convertPartial(tx, live, record.conversationId); + // Cut off, the abort handler may have died converting the partial: the run ends without it. + if (!cutOff) await convertPartial(tx, live, record.conversationId); endRun( tx, live, record.id, outcome.status === "faulted" - ? { status: "unanswered", reason: "faulted", detail: outcome.error.message } - : { status: "unanswered", reason: outcome.reason }, + ? { status: "unanswered", reason: cutOff ? "aborted" : "faulted", detail: outcome.error.message } + : { + status: "unanswered", + reason: "orphaned", + detail: outcome.reason, + settlement: { status: "unanswered", reason: outcome.reason }, + }, ); } diff --git a/packages/durable/src/harness/partial-changes.ts b/packages/durable/src/harness/partial-changes.ts index 355a9c93c..bf1a8ab04 100644 --- a/packages/durable/src/harness/partial-changes.ts +++ b/packages/durable/src/harness/partial-changes.ts @@ -1,18 +1,25 @@ // What a streamed answer changed in its partial message between two partial commits, so a commit costs the new text. -import type { JsonValue } from "@earendil-works/chord"; +import type { JsonRepresentation, JsonValue } from "@earendil-works/chord"; import { appendText, type Draft } from "@earendil-works/chord/delta"; -import type { AssistantMessageEvent } from "@earendil-works/pi-ai"; +import type { AssistantMessage, AssistantMessageEvent, ToolCall } from "@earendil-works/pi-ai"; import { type JsonChange, StreamingJsonParser } from "@earendil-works/pi-ai/utils/streaming-json"; import { type AssignPlan, assignJson } from "./json.ts"; +import type { LiveState } from "./live.ts"; type JsonContainer = Record | JsonValue[]; +/** A partial message as `pi.live.generation.message` holds it. */ +export type PartialMessage = JsonRepresentation; + +/** Committed arguments of a streamed tool call. */ +type Arguments = JsonRepresentation["arguments"]; + /** What one partial commit writes besides the partial message itself; from `PartialChanges.take()`. */ export type PartialCommit = { /** The plan `assignJson()` writes the message with: text appends, and the arguments it leaves to `changes`. */ readonly plan: AssignPlan; /** Each tracked tool call by content index: its argument changes since the last commit, in order, and its snapshot. */ - readonly toolCalls: ReadonlyMap; + readonly toolCalls: ReadonlyMap; }; /** What a tracked tool call changed since the last commit, and whether any commit took its changes yet. */ @@ -80,16 +87,18 @@ export class PartialChanges { * Take what changed since the last call, for a commit of `message`, a copy of the partial. Clears the arguments of * tracked tool calls in `message`: the commit writes them from the changes. Call `restore()` if the commit fails. */ - take(message: { readonly content: readonly JsonValue[] }): PartialCommit { + take(message: PartialMessage): PartialCommit { const blocks = new Map(); for (const [index, { field, text }] of this.#text) { blocks.set(index, { kind: "fields", fields: new Map([[field, { kind: "append", text }]]) }); } this.#text = new Map(); - const toolCalls = new Map(); + const toolCalls = new Map(); for (const [index, call] of this.#toolCalls) { - const block = message.content[index] as Record; - toolCalls.set(index, { changes: call.changes, snapshot: block.arguments! }); + const block = message.content[index]; + // A tracked call is the tool call its `toolcall_start` named; the narrowing only tells TypeScript. + if (block?.type !== "toolCall") continue; + toolCalls.set(index, { changes: call.changes, snapshot: block.arguments }); block.arguments = {}; blocks.set(index, { kind: "fields", fields: new Map([["arguments", { kind: "keep" }]]) }); call.changes = []; @@ -106,25 +115,32 @@ export class PartialChanges { * its first piece; when the committed message lost a call whose changes were already committed (another writer reset * it), the call falls back to its snapshot from now on. */ - commit(generation: Draft>, message: JsonValue, commit: PartialCommit): void { - const committedBlocks = (generation.message as { content?: unknown[] } | undefined)?.content?.length ?? 0; + commit( + generation: Draft>, + message: PartialMessage, + commit: PartialCommit, + ): void { + const committedBlocks = generation.message?.content.length ?? 0; const apply: [number, readonly JsonChange[]][] = []; for (const [index, { changes, snapshot }] of commit.toolCalls) { const call = this.#toolCalls.get(index); if (call === undefined) continue; - if (index >= committedBlocks && call.committed) { + const block = message.content[index]; + if (index >= committedBlocks && call.committed && block?.type === "toolCall") { // The earlier changes are not in the committed message: commit the snapshot and stop tracking the call. this.#toolCalls.delete(index); - (message as { content: Record[] }).content[index]!.arguments = snapshot; + block.arguments = snapshot; continue; } call.committed = true; apply.push([index, changes]); } assignJson(generation, "message", message, commit.plan); - const content = (generation.message as unknown as { content: Draft>[] }).content; + const content = generation.message?.content ?? []; for (const [index, changes] of apply) { - for (const change of changes) applyChange(content[index]!, change); + const block = content[index]; + if (block?.type !== "toolCall") continue; + for (const change of changes) applyChange(block, change); } } @@ -138,13 +154,15 @@ export class PartialChanges { } /** Apply one argument change to the committed arguments of `block`. */ -function applyChange(block: Draft>, change: JsonChange): void { +function applyChange(block: Draft>, change: JsonChange): void { const key = change.path.at(-1); if (key === undefined) { - // Only an object root is tracked, and only `set` starts it. - if (change.kind === "set") block.arguments = value(change); + // Only an object root is tracked (`observe()` drops any other), and only `set` starts it. + const root = change.kind === "set" ? value(change) : undefined; + if (isRecord(root)) block.arguments = root; return; } + // The path follows the parsed argument text, not a type: each segment names a container `observe()` saw set. let parent = block.arguments as Draft; for (const segment of change.path.slice(0, -1)) { parent = (parent as Record>)[segment]!; @@ -161,6 +179,6 @@ function value(change: Extract): JsonValue { return change.value as JsonValue; } -function isRecord(value: JsonValue | undefined): boolean { +function isRecord(value: JsonValue | undefined): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/packages/durable/src/harness/registry-reader.ts b/packages/durable/src/harness/registry-reader.ts new file mode 100644 index 000000000..97f8a44bd --- /dev/null +++ b/packages/durable/src/harness/registry-reader.ts @@ -0,0 +1,34 @@ +// A Harness's registry over a Chord registry (Chord v2 slice 8 §5): the composed value of p3's extensions registry, +// read as Durable's `RegistryReader`. + +import type { Context } from "@earendil-works/chord/context"; +import type { RegistryValue } from "@earendil-works/chord/next"; +import type { RegistryReader, RegistrySnapshot, ToolRegistration } from "./types.ts"; + +/** + * A `RegistryReader` over a Chord registry value whose composition is a registry snapshot: `snapshot()` reads the + * current composition (the last good one while `compose` fails), `subscribe` observes its compositions, its listener + * called synchronously inside the Chord batch that composed (so it only wakes the scheduler; a Chord call there + * throws `invalid/reentrant`). `context` is the observation's: aborting it detaches every listener. + */ +export function registryReader( + value: RegistryValue>, + context: Context, +): RegistryReader { + return { + snapshot: () => value.value, + subscribe: (listener) => { + const observation = value.observe( + { + advance: (call) => { + // A failed composition keeps the last snapshot: nothing to reconsider. + if (call.kind === "value") listener(); + }, + end: () => {}, + }, + context, + ); + return () => observation.detach(); + }, + }; +} diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 8fde30b63..63cbe89ac 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -11,12 +11,14 @@ import { } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage, ReadOnlySession } from "../errors.ts"; -import type { SessionImpl } from "../session/session.ts"; +import { ALL_KEYS, conversationKey, INTENT_KEY, ownedKey, ROOTS_KEY, taskKey } from "../session/keys.ts"; +import type { ObservedCommit, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { Timers } from "../timers.ts"; import type { - CommitPublication, + CommitOptions, ConversationId, + ConversationRecord, DocumentWatch, EntryId, EntryRecord, @@ -31,12 +33,13 @@ import type { TaskDefinition, TaskId, TaskOutcome, + TaskOutcomeError, TaskRecord, TaskRuntime, TaskState, } from "../types.ts"; import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.ts"; -import { type ContextRange, readContextFrom } from "./context.ts"; +import { advanceRange, type ContextRange, rangeEntry, readContextFrom } from "./context.ts"; import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.ts"; import type { Agent, @@ -51,7 +54,7 @@ import type { TaskInspection, Wake, } from "./types.ts"; -import { closedError, optionalExtension, scanAll, Waiters } from "./util.ts"; +import { closedError, conversationEntries, optionalExtension, scanAll, Waiters } from "./util.ts"; type AnyTaskRecord = TaskRecord; /** A record that can still run code: pending, running, waiting, or sleeping. */ @@ -65,13 +68,11 @@ type ErasedRuntime = TaskRuntime; type ErasedRunningTask = RunningTask; const SCAN_PAGE_SIZE = 256; +const NO_KEYS: readonly string[] = []; const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"] as const; /** What `blockedReason()` reads of a task: the task graph keeps only this of each record. */ -export type BlockedSubject = Pick< - AnyTaskRecord, - "id" | "kind" | "version" | "abortRequested" | "abortReason" | "restarts" -> & { +export type BlockedSubject = Pick & { readonly status: TaskState["status"]; }; @@ -83,16 +84,12 @@ export function blockedSubject(record: AnyTaskRecord): BlockedSubject { version: record.version, abortRequested: record.abortRequested, ...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }), - ...(record.restarts === undefined ? {} : { restarts: record.restarts }), status: record.state.status, }; } -/** - * Why a task cannot be reserved: no registered definition can take it, or it was cut off `restartLimit` times in a row. - * Derived from the registry, the settings, and the record's `restarts`. - */ -export type BlockedReason = "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; +/** Why a task cannot be reserved: no registered definition can take it. Derived from the registry. */ +export type BlockedReason = "missing_task" | "task_too_old" | "migration_failed"; type Resolution = | { readonly kind: "ready"; readonly task: AnyTask; readonly record: RunnableTaskRecord } @@ -146,8 +143,22 @@ type ReportedTask = { readonly task: AnyTask | undefined } | undefined; /** Outcome of the phase that just returned, judged by the next step. */ type PhaseResult = { readonly checkpoint: Checkpoint; readonly failure?: { readonly error: unknown } }; -/** Step decision: continue with the next phase, end the invocation, or end it by writing `faulted`. */ -type Decision = boolean | { readonly fault: unknown }; +/** + * Step decision, made in a decision region: continue with the next phase, end the invocation, or end it by writing + * `faulted`. + */ +type StepPlan = + | { readonly kind: "continue" | "progress" | "end" | "cascade" } + | { readonly kind: "fault"; readonly error: unknown }; + +/** Continue with the next phase, the first one. */ +const CONTINUE: StepPlan = { kind: "continue" }; +/** Rule 5: the phase made durable progress; `#progress()` decides once the decision stands. */ +const PROGRESS: StepPlan = { kind: "progress" }; +/** End the invocation without a write. */ +const END: StepPlan = { kind: "end" }; +/** End it: an owner's cancellation intent reaches it before its cascade did; schedule the cascade. */ +const CASCADE: StepPlan = { kind: "cascade" }; /** Terminal outcomes the scheduler writes without running task code. */ export type SchedulerOutcome = Extract, { readonly status: "faulted" | "orphaned" }>; @@ -175,6 +186,9 @@ type Overlay = { links?: ReadonlyMap>; }; +/** Tasks a finalize pass has made terminal in its commit so far. */ +type Finalized = { has(id: TaskId): boolean }; + /** A node of the downward index: a task, or a conversation as `c`. */ type Node = TaskId | `c${number}`; @@ -215,6 +229,11 @@ export type TaskSchedulerOptions = { /** While closing, on the line after admitted commits settle, write what the callback returns straight to Storage. */ readonly writeAtClose: (writes: () => readonly StorageWrite[]) => Promise; readonly report: (error: unknown, info: ReportInfo, context?: Context) => void; + /** + * Task kinds whose run phases end the task themselves once it was cut off `settings.restartLimit` times (the + * built-in tasks, from `TaskRecord.restarts`); the scheduler ends a task of any other kind `faulted` instead. + */ + readonly limitsRestarts: (kind: string) => boolean; /** Harness cleanup staged in the commit that makes an outcome the scheduler wrote itself terminal. */ readonly settleOutcome: (tx: Transaction, record: AnyTaskRecord, outcome: SchedulerOutcome) => Promise; /** Withdraw a conversation's queued inputs, for conversation abort and abort cascades. */ @@ -252,9 +271,16 @@ export type TaskSchedulerOptions = { * later below a dropped node finds it unknown and loads its chain from storage again. * * Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers - * reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the - * precedence rules and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the - * step that decides its end, so a runtime commit it queued either lands before that decision or is rejected. + * reservation, marks, runtime commits, finalization, and the step before each phase, which applies the precedence rules + * and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the step that decides + * its end, so a runtime commit it queued either lands before that decision or is rejected. + * + * Decisions stay serialized; their storage overlaps (group commit). The Session stores the commits of several jobs in + * one call while the line runs on, and this memory changes only as each commit is published. So every decision reads + * it in a decision region (`#region()`): synchronous, recording the keys it reads (a task's record, the live work below + * a node, the cancellation intent set), staging nothing; when one of them has an unpublished writer, the region waits + * for its publication and runs again, and only a region that read nothing unpublished is applied. Every commit writes + * the memberships its task changes alter (`membershipKeys()`), so a walk that a pending commit would change waits. */ export class TaskScheduler { readonly #telemetry: NonNullable; @@ -274,6 +300,7 @@ export class TaskScheduler { readonly #writeAtClose: (writes: () => readonly StorageWrite[]) => Promise; readonly #report: (error: unknown, info: ReportInfo, context?: Context) => void; readonly #settleOutcome: TaskSchedulerOptions["settleOutcome"]; + readonly #limitsRestarts: TaskSchedulerOptions["limitsRestarts"]; readonly #withdrawInputs: TaskSchedulerOptions["withdrawInputs"]; readonly #conversation: TaskSchedulerOptions["conversation"]; readonly #context: Context; @@ -283,7 +310,24 @@ export class TaskScheduler { * newer entries. `idleSince` is the Harness time it was first seen idle. Derived and never persisted; dropped at the * first idle check after `settings.contextRetentionMs` of idleness, and at close. */ - readonly #contexts = new Map(); + readonly #contexts = new Map< + ConversationId, + { range: ContextRange; idleSince: number | undefined; expiresAt?: number } + >(); + /** + * The idle kept contexts by `idleSince`, oldest first: the expiry order, so neither an expiry nor finding the next one + * scans `#contexts`. Kept sorted also when the Harness clock goes back (see `#markIdle`). + */ + readonly #idleOrder = new Map(); + /** At least the newest `idleSince` in `#idleOrder` (drops do not lower it): an older one added means the clock went back. */ + #idleNewest = Number.NEGATIVE_INFINITY; + /** + * A chain load resolved what `#idle()` treats as unknown for a live task whose chain is not loaded, which can make a + * scope idle without a task change below it; the next idle check then covers every scope. + */ + #unknownsChanged = false; + /** Idle checks that covered every scope (`idleState`, a test diagnostic). */ + #fullIdleChecks = 0; /** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */ #expiry: { readonly at: number; readonly disarm: () => void } | undefined; readonly #invocations = new Map(); @@ -337,12 +381,17 @@ export class TaskScheduler { readonly #blockedListeners = new Set<() => void>(); /** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */ #snapshotReported = false; - /** Tasks a pass left out as parked; each pass checks them again, as `settings.restartLimit` may have risen. */ - readonly #parkedTasks = new Set(); #reconcileScheduled = false; /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */ #reconcilingWork = 0; #sweepScheduled = false; + /** + * Nodes chain loads installed for jobs whose commits are not published yet, with how many such jobs: the sweep keeps + * them, as work a pending commit creates below them needs them. + */ + readonly #pins = new Map(); + /** Keys the decision region underway reads (`#region()`); set only while one runs. */ + #reading: Set | undefined; #cascadePending = false; #unsubscribeRegistry: () => void = () => {}; #enabled = false; @@ -369,6 +418,7 @@ export class TaskScheduler { this.#writeAtClose = options.writeAtClose; this.#report = options.report; this.#settleOutcome = options.settleOutcome; + this.#limitsRestarts = options.limitsRestarts; this.#withdrawInputs = options.withdrawInputs; this.#conversation = options.conversation; this.#context = options.context; @@ -376,20 +426,19 @@ export class TaskScheduler { } /** - * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing. A - * task that reaches the restart limit this way is parked and reported. Read-only, it only loads them: a `running` - * task stays as committed, running in another process. + * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing; + * a task cut off `settings.restartLimit` times ends when it is next reserved. Read-only, it only loads them: a + * `running` task stays as committed, running in another process. */ async open(context: Context): Promise { - this.#session.observeCommits((publication) => this.#observe(publication)); + this.#session.followCommits((publication) => this.#observe(publication)); this.#session.subscribeClose(() => this.#seal()); this.#unsubscribeRegistry = this.#registry.subscribe(() => { this.#notifyBlocked(); this.#kick(); }); - const limit = this.#restartLimit(); - const parked: AnyTaskRecord[] = []; await this.#session.commitWith(async (tx) => { + await this.#published(); // Every table read before the first write. const scans = []; for (const status of LIVE_STATUSES) { @@ -405,7 +454,6 @@ export class TaskScheduler { const restarts = (record.restarts ?? 0) + 1; const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint }); tx.setTask({ ...pending, restarts } as AnyTaskRecord); - if (restarts >= limit && !record.abortRequested) parked.push(record); } if (record.state.status === "waiting" && record.state.policy === "failFast") { this.#failFastChecks.add(record.id); @@ -413,13 +461,6 @@ export class TaskScheduler { } } }, context); - for (const record of parked) { - const restarts = (record.restarts ?? 0) + 1; - const error = new Error( - `Task ${record.id} (${record.kind}) was cut off ${restarts} times without progress; it is parked until retryTask()`, - ); - this.#report(error, { type: "task_parked", ...this.#aboutTask(recordTask(record)) }); - } if (this.#readOnly) return; // Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes. this.#cascadePending = true; @@ -439,30 +480,14 @@ export class TaskScheduler { } /** - * Run a task parked by the restart limit again: clears its restart count and enables scheduling. Does nothing for a - * task that is not parked; rejects for an unknown one. - */ - async retry(id: TaskId, context: Context): Promise { - await this.#session.commitWith(async (tx) => { - const current = await tx.task(id); - if (current === undefined) throw new Error(`Task ${id} does not exist`); - const status = current.state.status; - if (status === "terminal" || status === "completing" || !this.#parked(current)) return; - tx.setTask(progressed(current)); - }, context); - this.resume(); - } - - /** - * Why a live task cannot run here, if it cannot: its definition (missing, too old, failed migration) or the restart - * limit. A running task, an abort-marked one, and one holding its outcome are never blocked. + * Why a live task cannot run here, if it cannot: its definition is missing, too old, or failed to migrate. A running + * task, an abort-marked one, and one holding its outcome are never blocked. */ blockedReason(record: BlockedSubject): BlockedReason | undefined { const status = record.status; if (status === "running" || status === "completing" || status === "terminal") return undefined; // An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked. if (record.abortRequested && record.abortReason === undefined) return undefined; - if (this.#parked(record)) return "restart_limit"; let snapshot: RegistrySnapshot; try { snapshot = this.#registry.snapshot(); @@ -492,8 +517,22 @@ export class TaskScheduler { const now = this.#now(); // A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as // recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not. - let active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork(); + const active = + this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork() || this.#runsNow(now); + // The earliest sleep not yet due: only `#sleeping` holds sleeping tasks, so no pass visits every live task. let at: number | undefined; + for (const id of this.#sleeping) { + const record = this.#live.get(id); + const state = record?.state; + if (state?.status === "sleeping" && !record!.abortRequested && state.until > now) { + at = at === undefined ? state.until : Math.min(at, state.until); + } + } + return at === undefined ? { active } : { active, at }; + } + + /** Whether a live task can run now: the scan `wake()` needs only while no invocation runs. */ + #runsNow(now: number): boolean { let snapshot: RegistrySnapshot | undefined; let snapshotFailed = false; for (const record of this.#live.values()) { @@ -501,12 +540,7 @@ export class TaskScheduler { if (state.status === "completing") continue; if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id))) continue; - if (state.status === "sleeping" && !record.abortRequested && state.until > now) { - at = at === undefined ? state.until : Math.min(at, state.until); - continue; - } - if (active) continue; - if (this.#parked(record)) continue; + if (state.status === "sleeping" && !record.abortRequested && state.until > now) continue; // An abort invocation waits for the live work below it, which counts itself. if (record.abortRequested && this.#waitingOn(record, true).length > 0) continue; // A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned. @@ -521,9 +555,9 @@ export class TaskScheduler { } if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind))) continue; } - active = true; + return true; } - return at === undefined ? { active } : { active, at }; + return false; } /** @@ -566,12 +600,14 @@ export class TaskScheduler { if (keepRestart && current.abortReason === "restart") return { result: "marked" as const }; if (invocation === undefined && current.state.status !== "completing") { await this.#loadScopes(false); - if (!this.#hasOwnedLive(id)) { - const resolution = this.#resolve(current as RunnableTaskRecord, this.#registry.snapshot()); - if (resolution.kind === "blocked") { - await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason }); - return { result: "marked" as const }; - } + const owns = await this.#region(() => this.#hasOwnedLive(id), this.#reloadScopes); + // Resolved once that decision stands, as it may run `migrate`. + const resolution = owns + ? undefined + : this.#resolve(current as RunnableTaskRecord, this.#registry.snapshot()); + if (resolution?.kind === "blocked") { + await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason }); + return { result: "marked" as const }; } } if (!current.abortRequested || current.abortReason !== undefined) tx.setTask(withAbortMark(current)); @@ -587,10 +623,12 @@ export class TaskScheduler { async waitForTask(id: TaskId, context: Context): Promise> { // Check and register on the line so no terminal publication falls between them. - const found = await this.#session.readOnLine(async () => { + const found = await this.#session.readOnLine(async (storage) => { if (this.#closing) throw closedError(this.#session); - if (this.#live.has(id)) return { promise: this.#taskWaiters.add(id, context) }; - const record = await this.#storage.task(id, context); + if (await this.#region(() => this.#liveRecord(id) !== undefined)) { + return { promise: this.#taskWaiters.add(id, context) }; + } + const record = await storage.task(id, context); if (record === undefined) throw new Error(`Task ${id} does not exist`); return { promise: Promise.resolve(record as SettledTask) }; }); @@ -615,6 +653,7 @@ export class TaskScheduler { */ async abortConversation(conversationId: ConversationId, background: boolean, context: Context): Promise { const reached = await this.#session.commitWith(async (tx) => { + await this.#published(); const queued = await this.#loadScopes(true); const scope = { conversation: conversationId }; const reached: TaskId[] = []; @@ -633,12 +672,104 @@ export class TaskScheduler { await this.waitForIdle(conversationId, context); } + /** + * The ownership memberships a commit's task and conversation changes write (group commit §4.2), at its admission on + * the line: for each task created or made terminal, every node from its parent up to and including the nearest live + * task, or the root conversation, and then the roots too, conservatively, as a pending end may unlist the root a + * creation finds listed; for each conversation created under a task, that task; and the intent set when + * a change sets, upgrades, or clears a mark or a held or ended outcome other than `completed`. Loads the chains it + * walks; the nodes it installs stay until the commit is published. + */ + membershipKeys(tx: Transaction): readonly string[] | Promise { + const tasks = tx.stagedTasks(); + const conversations = tx.stagedConversations(); + if (tasks.length === 0 && conversations.every((record) => record.owner === undefined)) return NO_KEYS; + return this.#membershipKeys(tx, tasks, conversations); + } + + async #membershipKeys( + tx: Transaction, + tasks: readonly AnyTaskRecord[], + conversations: readonly ConversationRecord[], + ): Promise { + const keys = new Set(); + const overlay = overlayOf(tx); + const created = new Set(tx.createdTaskIds()); + const reached = new Map(); + for (const record of tasks) { + if (intentChanges(this.#live.get(record.id), record)) keys.add(INTENT_KEY); + if (!created.has(record.id) && record.state.status !== "terminal") continue; + await this.#loadChain(parentOf(record), overlay); + if ((await this.#chainKeys(parentNode(record), overlay, keys, reached)) !== undefined) keys.add(ROOTS_KEY); + } + for (const conversation of conversations) { + const owner = conversation.owner?.taskId; + if (owner === undefined) continue; + await this.#loadChain({ task: owner }, overlay); + await this.#chainKeys(owner, overlay, keys, reached); + } + return [...keys]; + } + + /** + * Add the memberships from `start` up to and including the nearest live task, or the root conversation, which it + * returns. `reached` keeps, per node walked, where its walk ended, so a chain several tasks of a commit share is + * walked once. Whether an ancestor is live can change under a pending commit, so it is read after a touch of the + * ancestor's record: a pending end must not stop the chain early. Ownership fields are immutable, so not keyed. + */ + async #chainKeys( + start: Node, + overlay: Overlay, + keys: Set, + reached: Map, + ): Promise { + const walked: string[] = []; + let root: ConversationId | undefined; + let node: Node | null | undefined = start; + while (node !== undefined && node !== null) { + const key = ownedKey(node); + // Walked up from here already, for another task of the commit. + if (reached.has(key)) { + root = reached.get(key); + break; + } + keys.add(key); + walked.push(key); + if (typeof node === "string") { + const conversation = conversationOf(node); + node = this.#edge(conversation, overlay); + if (node === null) root = conversation; + continue; + } + let record: AnyTaskRecord | undefined = overlay.tasks.get(node); + if (record === undefined) { + const gate = this.#session.touch(taskKey(node)); + if (gate !== undefined) { + await gate; + // It may have ended meanwhile, and its node been swept: load the chain above it again. + await this.#loadChain({ task: node }, overlay); + } + record = this.#live.get(node); + } + if (record !== undefined && record.state.status !== "terminal") break; + const above: TaskNode | undefined = record ?? this.#settled.get(node); + node = above === undefined ? undefined : parentNode(above); + } + for (const key of walked) reached.set(key, root); + return root; + } + // ─── Scheduling ──────────────────────────────────────────────────────── - #observe(publication: CommitPublication): void { + #observe(publication: ObservedCommit): void { + // A commit held in memory changes documents only. + if (publication.kind === "memory") return; + this.#advanceContexts(publication); const updated: AnyTaskRecord[] = []; const failed: TaskId[] = []; let changed = false; + // The conversations whose idleness the commit's task changes can change; undefined when a chain is not known. + let dirty: Set | undefined = new Set(); // Edges first, so the tasks a commit creates in a new conversation hang from it. for (const change of publication.changes) { if (change.type === "conversation" && !this.#edges.has(change.value.id)) { @@ -652,6 +783,10 @@ export class TaskScheduler { const previous = this.#live.get(record.id); if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous))) failed.push(record.id); this.#track(previous, record); + // Idleness counts live tasks, so only a task that became live or ended can change it. Collected before `#sweep()`, + // which may forget the chain of a task that ended. + const liveness = previous === undefined || record.state.status === "terminal"; + if (liveness && dirty !== undefined && !this.#addScopesAbove(record, dirty)) dirty = undefined; if (record.state.status === "terminal") { this.#failedMigrations.delete(record.id); this.#failFastChecks.delete(record.id); @@ -662,9 +797,10 @@ export class TaskScheduler { } if (record.abortRequested && previous?.abortRequested !== true) { this.#cascadePending = true; - // Signal a run invocation of the newly marked task; its next step ends it. + // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which + // never runs inside a publication: the signal follows in a microtask, after host commit listeners. const invocation = this.#invocations.get(record.id); - if (invocation?.mode === "run") invocation.controller.abort(); + if (invocation?.mode === "run") queueMicrotask(() => invocation.controller.abort()); } // A request replacing a `restart` mark upgrades the marks of the work below. if (previous?.abortReason !== undefined && record.abortReason === undefined) this.#cascadePending = true; @@ -711,10 +847,53 @@ export class TaskScheduler { if (this.#cascadePending) this.#scheduleReconcile(); this.#sweep(); if (!changed) return; - this.#settleIdle(); + this.#settleIdle(dirty); this.#kick(); } + /** + * Add the conversations above `record` to `scopes`: those whose idleness a change of it can change. False when a node + * on the way up is not known. A conversation already in `scopes` has everything above it there too. + */ + #addScopesAbove(record: AnyTaskRecord, scopes: Set): boolean { + for (const step of this.#above(parentOf(record))) { + if ("unknown" in step) return false; + if (!("conversation" in step)) continue; + if (scopes.has(step.conversation)) return true; + scopes.add(step.conversation); + } + return true; + } + + /** + * Extend each kept context range with the entries a stored commit appended to its conversation, so a later read + * needs no Storage scan. Entries are published in ID order. + */ + #advanceContexts(publication: Extract): void { + if (this.#contexts.size === 0) return; + let appended: Map | undefined; + for (const change of publication.changes) { + if (change.type !== "entry" || !this.#contexts.has(change.value.conversationId)) continue; + appended ??= new Map(); + const entries = appended.get(change.value.conversationId); + if (entries === undefined) appended.set(change.value.conversationId, [change.value]); + else entries.push(change.value); + } + if (appended === undefined) return; + for (const [conversationId, entries] of appended) { + const kept = this.#contexts.get(conversationId)!; + // An idle range past its retention is dropped here too: entry-only commits do not run `#settleIdle()`, and a + // host may decline the expiry timer. `expiresAt` was fixed when it went idle, so no settings are read here. + if (kept.idleSince !== undefined && kept.expiresAt !== undefined && kept.expiresAt <= this.#now()) { + this.#dropContext(conversationId); + continue; + } + const range = advanceRange(kept.range, entries); + if (range === undefined) this.#dropContext(conversationId); + else kept.range = range; + } + } + /** * `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them * is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer. @@ -728,47 +907,98 @@ export class TaskScheduler { } } - /** Resolve idle waiters, and drop each kept context whose conversation has been idle for the retention period. */ - #settleIdle(): void { - for (const conversationId of this.#idleWaiters.keys()) { - if (this.#idle(conversationId)) this.#idleWaiters.resolve(conversationId); - } + /** + * After task changes below the conversations in `scopes` (every conversation when undefined): resolve their idle + * waiters and the Harness-wide ones, drop the kept contexts whose idle period passed the retention, then start or end + * the idle periods of the kept contexts in `scopes`. A task change can change the idleness only of the conversations + * above it, so no check visits every conversation. + */ + #settleIdle(scopes: ReadonlySet | undefined): void { + const all = scopes === undefined || this.#unknownsChanged; + this.#unknownsChanged = false; + if (all) this.#fullIdleChecks++; + this.#resolveIdleWaiters(all ? undefined : scopes); const now = this.#now(); const retention = this.#contextRetentionMs(); - for (const [conversationId, kept] of this.#contexts) { - if (kept.idleSince !== undefined && now - kept.idleSince >= retention) { - this.#contexts.delete(conversationId); - } else if (!this.#idle(conversationId)) { + // Expired first: a context past its retention is dropped even when the same commit makes it busy again. + this.#expireContexts(now, retention); + for (const conversationId of all ? [...this.#contexts.keys()] : scopes) { + const kept = this.#contexts.get(conversationId); + if (kept === undefined) continue; + if (!this.#idle(conversationId)) { + if (kept.idleSince === undefined) continue; kept.idleSince = undefined; - } else if (retention > 0) { - kept.idleSince ??= now; - } else { - this.#contexts.delete(conversationId); + this.#idleOrder.delete(conversationId); + } else if (retention <= 0) { + this.#dropContext(conversationId); + } else if (kept.idleSince === undefined) { + kept.idleSince = now; + kept.expiresAt = now + retention; + this.#markIdle(conversationId, now); } } this.#scheduleExpiry(); } + /** Resolve the idle waiters of the conversations in `scopes` (all when undefined) and the Harness-wide ones. */ + #resolveIdleWaiters(scopes: ReadonlySet | undefined): void { + for (const conversationId of this.#idleWaiters.keys()) { + if (conversationId !== undefined && scopes !== undefined && !scopes.has(conversationId)) continue; + if (this.#idle(conversationId)) this.#idleWaiters.resolve(conversationId); + } + } + /** - * Run `#settleIdle()` when the earliest idle context expires, on a background timer, which never keeps the process - * alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer could - * keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry. + * Drop the kept contexts idle for at least `retention`, oldest first, with the retention setting of now; with no + * retention, every idle one, also those whose idle period a clock that went back makes start after now. + */ + #expireContexts(now: number, retention: number): void { + for (const [conversationId, idleSince] of this.#idleOrder) { + if (retention > 0 && now - idleSince < retention) return; + this.#dropContext(conversationId); + } + } + + /** + * Start the idle period of a kept context in the expiry order. An idle period older than the newest one kept, which + * a clock that went back starts, sorts the order again, and the newest is taken from what is kept then. + */ + #markIdle(conversationId: ConversationId, idleSince: number): void { + this.#idleOrder.delete(conversationId); + this.#idleOrder.set(conversationId, idleSince); + if (this.#idleOrder.size === 1 || idleSince >= this.#idleNewest) { + this.#idleNewest = idleSince; + return; + } + const sorted = [...this.#idleOrder].sort((a, b) => a[1] - b[1]); + this.#idleOrder.clear(); + for (const [id, since] of sorted) this.#idleOrder.set(id, since); + this.#idleNewest = sorted.at(-1)![1]; + } + + /** Forget a kept context, idle or not, and its place in the expiry order. */ + #dropContext(conversationId: ConversationId): void { + this.#contexts.delete(conversationId); + this.#idleOrder.delete(conversationId); + } + + /** + * Drop the expired contexts when the earliest idle context expires, on a background timer, which never keeps the + * process alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer + * could keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check + * expiry. */ #scheduleExpiry(): void { - const retention = this.#contextRetentionMs(); - let at: number | undefined; - for (const kept of this.#contexts.values()) { - if (kept.idleSince !== undefined && (at === undefined || kept.idleSince + retention < at)) { - at = kept.idleSince + retention; - } - } + const oldest = this.#idleOrder.values().next(); + const at = oldest.done === true ? undefined : oldest.value + this.#contextRetentionMs(); if (this.#expiry !== undefined && this.#expiry.at === at) return; this.#expiry?.disarm(); this.#expiry = undefined; if (at === undefined || this.#closing) return; const disarm = this.#timers.background(() => { this.#expiry = undefined; - this.#settleIdle(); + this.#expireContexts(this.#now(), this.#contextRetentionMs()); + this.#scheduleExpiry(); }, at - this.#now()); if (disarm !== undefined) this.#expiry = { at, disarm }; } @@ -792,45 +1022,21 @@ export class TaskScheduler { this.#reconcileScheduled = false; const work = this.#reconcileWork(); if (work) this.#reconcilingWork++; - const cascade = this.#cascadePending; - this.#cascadePending = false; - const checks = [...this.#failFastChecks]; - this.#failFastChecks.clear(); try { - await this.#session.commitWith(async (tx) => { + await this.#session.commitInternal(async (tx) => { if (this.#closing) return; + // A cascade scans the queued inputs, which waits for every unpublished commit: a request made before the scan + // is covered by it, and none is published while the job holds the line after it. + const cascade = this.#cascadePending; const queued = await this.#loadScopes(cascade); - // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a - // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. - const marks = new Map(); - const mark = (record: AnyTaskRecord, reason: "restart" | undefined): void => { - const staged = marks.get(record.id); - if (staged !== undefined && staged.reason === undefined) return; - if (record.abortRequested && (record.abortReason === undefined || reason !== undefined)) return; - marks.set(record.id, { record, reason }); - }; - // Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down - // from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on - // every pass, as loading edges can reveal a cancelled owner. - for (const id of this.#intent) { - const owner = this.#live.get(id)!; - // Only an owner abandoned after a restart, and nothing else, passes its reason on. - const restart = owner.abortReason === "restart" && !failedOutcome(owner); - for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) { - if (live !== undefined && !node.background) mark(live, restart ? "restart" : undefined); - } - } - for (const id of checks) { - const waiter = this.#live.get(id); - if (waiter?.state.status !== "waiting" || !(await this.#anyFailed(waiter.state.on))) continue; - // Every other live task: the failed one keeps its own outcome. - for (const member of waiter.state.on) { - const record = this.#live.get(member); - if (record !== undefined && !failedOutcome(record)) mark(record, undefined); - } - } + if (cascade) this.#cascadePending = false; + const marks = await this.#marks(); for (const { record, reason } of marks.values()) tx.setTask(withAbortMark(record, reason)); - for (const id of queued) if (this.#belowCancelled({ conversation: id })) await this.#withdrawInputs(tx, id); + const withdrawn = await this.#region( + () => queued.filter((id) => this.#belowCancelled({ conversation: id })), + this.#reloadScopes, + ); + for (const id of withdrawn) await this.#withdrawInputs(tx, id); await this.#finalize(tx); }, this.#context); } catch (error) { @@ -840,7 +1046,87 @@ export class TaskScheduler { } if (work) this.#reconcilingWork--; this.#scheduleWake(); - this.#settleIdle(); + // Its commit's task changes were settled at their publication; a waiter that asked for this pass checks again, and + // what its chain loads resolved can make any scope idle. + if (this.#unknownsChanged) { + this.#settleIdle(undefined); + return; + } + this.#resolveIdleWaiters(undefined); + // A pass checks expiry too, for hosts that decline the expiry timer. + this.#expireContexts(this.#now(), this.#contextRetentionMs()); + this.#scheduleExpiry(); + } + + /** + * The abort marks committed records imply (spec §5.4, §5.5): every live foreground task below a live owner with + * cancellation intent, which is its nearest one (a walk down from each owner with intent does not enter another, + * whose own walk covers what is below it), and every other live task in the `on` of a `failFast` waiter one of whose + * tasks failed. A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades + * a `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. Derived + * again on every pass, as loading edges can reveal a cancelled owner. Takes the `failFast` checks once its decision + * stands. + */ + async #marks(): Promise> { + // Terminal tasks in a checked `on`, read from Storage: each one's outcome is final. + const ended = new Map(); + for (;;) { + const plan = await this.#region(() => this.#planMarks(ended), this.#reloadScopes); + if ("missing" in plan) { + for (const id of plan.missing) + ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null); + continue; + } + for (const id of plan.checked) this.#failFastChecks.delete(id); + return plan.marks; + } + } + + /** `#marks()`' decision region; the `failFast` members it needs from Storage first, when `ended` lacks them. */ + #planMarks(ended: ReadonlyMap): + | { readonly missing: readonly TaskId[] } + | { + readonly marks: Map; + readonly checked: readonly TaskId[]; + } { + const marks = new Map(); + // A mark rewrites the whole record, so the record it copies is read as a task record, keyed: a commit of the task + // that is not published yet makes the region wait and run again, rather than stage a mark over a stale checkpoint. + const mark = (found: AnyTaskRecord, reason: "restart" | undefined): void => { + const record = this.#liveRecord(found.id)!; + const staged = marks.get(record.id); + if (staged !== undefined && staged.reason === undefined) return; + if (record.abortRequested && (record.abortReason === undefined || reason !== undefined)) return; + marks.set(record.id, { record, reason }); + }; + this.#record(INTENT_KEY); + for (const id of this.#intent) { + const owner = this.#liveRecord(id)!; + // Only an owner abandoned after a restart, and nothing else, passes its reason on. + const restart = owner.abortReason === "restart" && !failedOutcome(owner); + for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) { + if (live !== undefined && !node.background) mark(live, restart ? "restart" : undefined); + } + } + const checked = [...this.#failFastChecks]; + const missing: TaskId[] = []; + for (const id of checked) { + const waiter = this.#liveRecord(id); + if (waiter?.state.status !== "waiting") continue; + let failed = false; + for (const member of waiter.state.on) { + const record = this.#liveRecord(member) ?? ended.get(member); + if (record === undefined) missing.push(member); + else if (record !== null && failedOutcome(record)) failed = true; + } + if (!failed) continue; + // Every other live task: the failed one keeps its own outcome. + for (const member of waiter.state.on) { + const record = this.#liveRecord(member); + if (record !== undefined && !failedOutcome(record)) mark(record, undefined); + } + } + return missing.length > 0 ? { missing } : { marks, checked }; } /** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */ @@ -848,55 +1134,86 @@ export class TaskScheduler { return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0; } - /** Whether any of `ids` holds or ended with an outcome other than `completed`. */ - async #anyFailed(ids: readonly TaskId[]): Promise { - for (const id of ids) { - const record = this.#live.get(id) ?? (await this.#storage.task(id, this.#context)); - if (record !== undefined && failedOutcome(record)) return true; - } - return false; - } - /** * Write the terminal record of every `completing` task without live ordinary owned work. Finalizing one can free its * owner, so this repeats over the commit's candidates until nothing changes. A held scheduler outcome gets its - * Harness cleanup here. + * Harness cleanup here. In rounds, one decision region each: the holders the tasks of one round free are checked once, + * in the next, so a holder of many tasks that end together is walked once, not once per task. The first round takes + * the `completing` tasks to check once its decision stands. */ async #finalize(tx: Transaction): Promise { - // A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the - // nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks - // down skip them. - // In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that - // end together is walked once, not once per task. - let round = [...this.#finalizeChecks]; - this.#finalizeChecks.clear(); const done = new Set(); + // The commit's candidates, kept in step with the terminal records each round stages. const overlay = overlayOf(tx); - while (round.length > 0) { - const next = new Set(); - for (const id of round) { - const record = overlay.tasks.get(id) ?? this.#live.get(id); - if (record?.state.status !== "completing" || done.has(id)) continue; - if (this.#hasOwnedLive(id, overlay, done)) continue; - const outcome = record.state.outcome; - const terminal = withState(record, { status: "terminal", outcome }); + let round: readonly TaskId[] | undefined; + for (;;) { + const ids = round; + const plan = await this.#region( + () => this.#planRound(ids ?? [...this.#finalizeChecks], overlay, done), + this.#reloadScopes, + ); + if (ids === undefined) this.#finalizeChecks.clear(); + for (const { record, terminal } of plan.finalized) { tx.setTask(terminal); - (overlay.tasks as Map).set(id, terminal); - done.add(id); + (overlay.tasks as Map).set(record.id, terminal); + done.add(record.id); + const outcome = (record.state as { readonly outcome: TaskOutcome }).outcome; // REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit. if (outcome.status === "faulted" || outcome.status === "orphaned") { await this.#settleOutcome(tx, record, outcome); } - const above = this.#nearestLiveAbove(record, overlay); - if (above?.state.status === "completing") next.add(above.id); } - round = [...next]; + if (plan.next.length === 0) return; + round = plan.next; } } + /** + * One finalize round's decision: which of `ids` are `completing` without live ordinary owned work below them, in the + * commit's candidates with the round's own finalizations applied in order, and the holders above them to check next. + * Tasks finalized have no live work below them, so walks down skip them. It applies the round's terminal records to + * `overlay` while it decides and takes them back before it returns, so a run again starts from the same candidates. + */ + #planRound( + ids: readonly TaskId[], + overlay: Overlay, + finalized: ReadonlySet, + ): { + readonly finalized: readonly { readonly record: AnyTaskRecord; readonly terminal: AnyTaskRecord }[]; + readonly next: readonly TaskId[]; + } { + const tasks = overlay.tasks as Map; + // The tasks finalized so far, earlier rounds' and this one's, without copying the former. + const round = new Set(); + const done = { has: (id: TaskId) => round.has(id) || finalized.has(id) }; + const ended: { readonly record: AnyTaskRecord; readonly terminal: AnyTaskRecord; readonly staged: boolean }[] = + []; + const next = new Set(); + try { + for (const id of ids) { + const staged = tasks.get(id); + const record = staged ?? this.#liveRecord(id); + if (record?.state.status !== "completing" || done.has(id)) continue; + if (this.#hasOwnedLive(id, overlay, done)) continue; + const terminal = withState(record, { status: "terminal", outcome: record.state.outcome }); + tasks.set(id, terminal); + round.add(id); + ended.push({ record, terminal, staged: staged !== undefined }); + const above = this.#nearestLiveAbove(record, overlay); + if (above?.state.status === "completing") next.add(above.id); + } + } finally { + for (const { record, staged } of ended) { + if (staged) tasks.set(record.id, record); + else tasks.delete(record.id); + } + } + return { finalized: ended, next: [...next] }; + } + /** * Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs, - * on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback. + * on the Session line; returns the latter. Reads through the line's Storage, so it may run inside a commit callback. */ async #loadScopes(queued: boolean): Promise { for (const id of [...this.#unloaded]) { @@ -904,8 +1221,9 @@ export class TaskScheduler { if (this.#unloaded.has(id)) await this.#loadChain({ task: id }); } if (!queued) return []; + // A scan, which cannot be keyed: it waits for every unpublished commit. const submissions = await scanAll((cursor) => - this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context), + this.#session.lineStorage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context), ); const inputs = submissions.filter((submission) => submission.type === "input"); const conversations = [...new Set(inputs.map((submission) => submission.conversationId))]; @@ -916,43 +1234,70 @@ export class TaskScheduler { /** * Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the * walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal - * a cancelled owner, so it makes the next reconcile derive cascade marks. + * a cancelled owner, so it makes the next reconcile derive cascade marks. The nodes it installs stay until the job's + * commit is published (`#pin()`). */ async #loadChain(start: Up, overlay?: Overlay): Promise { const passed: TaskId[] = []; + let installed = false; let at: Up | undefined = start; - while (at !== undefined) { - if ("task" in at) { - const id: TaskId = at.task; - let node: TaskNode | undefined = this.#live.get(id) ?? this.#settled.get(id); - // Created in this commit: the load from its own record covers what is above it. - if (node === undefined && overlay?.tasks.has(id) === true) break; - if (node !== undefined && !this.#unloaded.has(id)) break; - if (node === undefined) { - const record = await this.#storage.task(id, this.#context); - if (record === undefined) return; - node = nodeOf(record); - if (record.state.status === "terminal") this.#settle(id, node); - // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. - this.#scheduleReconcile(); + try { + while (at !== undefined) { + if ("task" in at) { + const id: TaskId = at.task; + let node: TaskNode | undefined = this.#live.get(id) ?? this.#settled.get(id); + // Created in this commit: the load from its own record covers what is above it. + if (node === undefined && overlay?.tasks.has(id) === true) break; + if (node !== undefined && !this.#unloaded.has(id)) break; + if (node === undefined) { + // Through the line's Storage: a task whose creation or end is not published yet waits for it. + const record = await this.#session.lineStorage.task(id, this.#context); + if (record === undefined) return; + node = nodeOf(record); + if (record.state.status === "terminal") { + this.#settle(id, node); + this.#pin(id); + installed = true; + } + // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. + this.#scheduleReconcile(); + } + passed.push(id); + at = parentOf(node); + } else { + let edge = this.#edge(at.conversation, overlay); + if (edge === undefined) { + // Through the line's Storage: a conversation whose creation is not published yet waits for it, so a missing + // record is never mistaken for an ownerless one. + const record = await this.#session.lineStorage.conversation(at.conversation, this.#context); + edge = record?.owner?.taskId ?? null; + this.#setEdge(at.conversation, edge); + this.#pin(conversationNode(at.conversation)); + installed = true; + // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. + this.#scheduleReconcile(); + } + at = edge === null ? undefined : { task: edge }; } - passed.push(id); - at = parentOf(node); - } else { - let edge = this.#edge(at.conversation, overlay); - if (edge === undefined) { - const record = await this.#storage.conversation(at.conversation, this.#context); - edge = record?.owner?.taskId ?? null; - this.#setEdge(at.conversation, edge); - // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. - this.#scheduleReconcile(); - } - at = edge === null ? undefined : { task: edge }; + } + } finally { + // `#idle()` counts a live task whose chain is not loaded as inside every scope it may be in: what this load + // resolved, also when it stopped early, can make such a scope idle without a task change below it. + if (passed.some((id) => this.#live.has(id)) || (installed && this.#liveUnloaded())) { + this.#unknownsChanged = true; + // A waiter such a task held is checked again at the pass's end, also when this load was not a reconcile's. + this.#scheduleReconcile(); } } for (const id of passed) this.#unloaded.delete(id); } + /** Whether a live task's chain is not loaded yet: one `#idle()` counts as inside every scope it may be in. */ + #liveUnloaded(): boolean { + for (const id of this.#unloaded) if (this.#live.has(id)) return true; + return false; + } + #setEdge(conversationId: ConversationId, owner: TaskId | null): void { this.#edges.set(conversationId, owner); // The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep. @@ -1000,7 +1345,8 @@ export class TaskScheduler { */ #sweep(): void { for (const node of this.#dropQueue) { - if (this.#below.has(node)) continue; + // Queued again once its pins go. + if (this.#below.has(node) || this.#pins.has(node)) continue; if (typeof node === "string") { this.#edges.delete(conversationOf(node)); this.#roots.delete(conversationOf(node)); @@ -1012,6 +1358,70 @@ export class TaskScheduler { this.#dropQueue.clear(); } + /** + * Run `decide`, a decision region (group commit §6): synchronous, reading scheduler memory and recording the keys it + * reads, staging nothing and changing no scheduler state. When a key it read has an unpublished writer, its result, + * or its error, is dropped: once that writer is published, `refresh` reloads what the region needs and it runs + * again. A waiting job holds the line, so unpublished writes only shrink, and the runs converge. What a region read + * that had no unpublished writer gets none for the rest of the job, so its result stands while the job applies it. + */ + async #region(decide: () => T, refresh?: () => Promise): Promise { + for (;;) { + const keys = new Set(); + this.#reading = keys; + let result: { readonly value: T } | { readonly error: unknown }; + try { + result = { value: decide() }; + } catch (error) { + result = { error }; + } finally { + this.#reading = undefined; + } + const gates: Promise[] = []; + for (const key of keys) { + const gate = this.#session.touch(key); + if (gate !== undefined) gates.push(gate); + } + if (gates.length === 0) { + if ("error" in result) throw result.error; + return result.value; + } + await Promise.all(gates); + await refresh?.(); + } + } + + /** Load the chains a region walks, again after it waited: what was published meanwhile may list new work. */ + readonly #reloadScopes = (): Promise => this.#loadScopes(false); + + /** In a decision region, note that it reads `key`; anywhere else, nothing. */ + #record(key: string): void { + this.#reading?.add(key); + } + + /** A live task's committed record, read by a decision region. */ + #liveRecord(id: TaskId): AnyTaskRecord | undefined { + this.#record(taskKey(id)); + return this.#live.get(id); + } + + /** + * Keep `node`, which a chain load of the job on the line installed, until that job's commit is published or fails: + * work it creates below the node needs it, and the sweep would drop it as unneeded before then. + */ + #pin(node: Node): void { + this.#pins.set(node, (this.#pins.get(node) ?? 0) + 1); + this.#session.onJobEnd(() => { + const count = this.#pins.get(node)! - 1; + if (count > 0) { + this.#pins.set(node, count); + return; + } + this.#pins.delete(node); + this.#dropIfUnneeded(node); + }); + } + /** * Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task * above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run. @@ -1021,10 +1431,10 @@ export class TaskScheduler { const invocation = this.#invocations.get(id); if (invocation !== undefined) invocation.record = record as TaskRecord; if (record.state.status === "terminal") { - if (previous !== undefined && !this.#closing) this.#activeMetric.add(-1, { task: record.kind }); + if (previous !== undefined && !this.#closing) this.#countActive(-1, record.kind); this.#resumed.delete(id); } else if (previous === undefined && !this.#closing) { - this.#activeMetric.add(1, { task: record.kind }); + this.#countActive(1, record.kind); } if (previous?.state.status === "waiting") for (const on of previous.state.on) deleteFrom(this.#waiters, on, id); if (record.state.status === "terminal") { @@ -1113,10 +1523,11 @@ export class TaskScheduler { * at or below a background task holds nothing above it. */ #nearestLiveAbove(node: TaskNode, overlay?: Overlay): AnyTaskRecord | undefined { + this.#record(INTENT_KEY); for (let current = node; !current.background; ) { const owner = current.owner ?? this.#edge(current.conversationId, overlay); if (owner === undefined || owner === null) return undefined; - const record = overlay?.tasks.get(owner) ?? this.#live.get(owner); + const record = overlay?.tasks.get(owner) ?? this.#liveRecord(owner); if (record !== undefined && record.state.status !== "terminal") return record; const above = record ?? this.#settled.get(owner); if (above === undefined) return undefined; @@ -1136,7 +1547,9 @@ export class TaskScheduler { /** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */ #edge(id: ConversationId, overlay?: Overlay): TaskId | null | undefined { - return overlay?.edges.has(id) === true ? overlay.edges.get(id) : this.#edges.get(id); + if (overlay?.edges.has(id) === true) return overlay.edges.get(id); + this.#record(conversationKey(id)); + return this.#edges.get(id); } #node(id: TaskId, overlay?: Overlay): TaskNode | undefined { @@ -1184,6 +1597,7 @@ export class TaskScheduler { const links = overlay === undefined ? undefined : this.#links(overlay); const pending: Node[] = [start]; for (let at = pending.pop(); at !== undefined; at = pending.pop()) { + this.#record(ownedKey(at)); for (const children of [this.#below.get(at), links?.get(at)]) { for (const child of children ?? []) { if (typeof child === "string") { @@ -1238,7 +1652,7 @@ export class TaskScheduler { */ #ownedLive( owner: TaskId, - options: { readonly overlay?: Overlay; readonly first?: boolean; readonly done?: ReadonlySet } = {}, + options: { readonly overlay?: Overlay; readonly first?: boolean; readonly done?: Finalized } = {}, ): TaskId[] { const { overlay, first = false, done } = options; const found: TaskId[] = []; @@ -1254,7 +1668,7 @@ export class TaskScheduler { return found; } - #hasOwnedLive(owner: TaskId, overlay?: Overlay, done?: ReadonlySet): boolean { + #hasOwnedLive(owner: TaskId, overlay?: Overlay, done?: Finalized): boolean { const options = { first: true, ...(overlay === undefined ? {} : { overlay }), @@ -1269,6 +1683,7 @@ export class TaskScheduler { * edge is not loaded. */ #inScope(start: Up, scope: Scope, crossBackground = false): boolean | undefined { + this.#record(INTENT_KEY); for (const step of this.#above(start)) { if ("unknown" in step) return undefined; if ("conversation" in step) { @@ -1290,6 +1705,7 @@ export class TaskScheduler { /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */ #cancellingOwner(start: Up): AnyTaskRecord | undefined { + this.#record(INTENT_KEY); // Nothing cancels, the common case: no walk. if (this.#intent.size === 0) return undefined; for (const step of this.#above(start)) { @@ -1302,6 +1718,18 @@ export class TaskScheduler { return undefined; } + /** + * Count a task in or out of `pi.durable.tasks.active`. Commit observers count at a commit's publication, which must not + * throw, so a throwing adapter is reported. + */ + #countActive(delta: 1 | -1, kind: string): void { + try { + this.#activeMetric.add(delta, { task: kind }); + } catch (error) { + this.#report(error, TELEMETRY); + } + } + /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */ #failSession(error: unknown, context?: Context): void { if (!this.#closing) this.#session.fail(error, context); @@ -1309,7 +1737,7 @@ export class TaskScheduler { /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */ #seal(): void { - for (const record of this.#live.values()) this.#activeMetric.add(-1, { task: record.kind }); + for (const record of this.#live.values()) this.#countActive(-1, record.kind); this.#resumed.clear(); this.#closing = true; try { @@ -1321,13 +1749,17 @@ export class TaskScheduler { this.#taskWaiters.rejectAll(error); this.#idleWaiters.rejectAll(error); this.#contexts.clear(); + this.#idleOrder.clear(); this.#expiry?.disarm(); this.#expiry = undefined; this.#timer?.disarm(); this.#timer = undefined; } - /** Recompute the wake in a microtask, off the Session line: arm the built-in timer, or tell `onWake` of a change. */ + /** + * Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one + * already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change. + */ #scheduleWake(): void { if (this.#wakeScheduled || this.#closing || this.#readOnly) return; this.#wakeScheduled = true; @@ -1377,11 +1809,6 @@ export class TaskScheduler { } } - /** A task cut off `restartLimit` times in a row; an abort mark still runs its abort handler. */ - #parked(record: Pick): boolean { - return !record.abortRequested && record.restarts !== undefined && record.restarts >= this.#restartLimit(); - } - /** * The report info of a task's failure, with the extension that installed its definition: the code that failed is * that extension's. Built-in tasks have none. @@ -1407,20 +1834,6 @@ export class TaskScheduler { return { task: reportTask(task), ...optionalExtension(extension) }; } - /** Make parked tasks that a raised `settings.restartLimit` no longer parks candidates again, and tell the graph. */ - #unparkRaised(): void { - let unparked = false; - for (const id of this.#parkedTasks) { - const record = this.#live.get(id); - if (record !== undefined && this.#parked(record)) continue; - this.#parkedTasks.delete(id); - if (record === undefined) continue; - this.#runnable.add(id); - unparked = true; - } - if (unparked) queueMicrotask(() => this.#notifyBlocked()); - } - /** `settings.restartLimit`, or the default when the host's settings throw. */ #restartLimit(): number { try { @@ -1436,7 +1849,7 @@ export class TaskScheduler { try { listener(); } catch (error) { - this.#report(error, { type: "listener_error" }); + this.#report(error, { kind: "listener_error" }); } } } @@ -1483,59 +1896,45 @@ export class TaskScheduler { const faulted: { readonly record: AnyTaskRecord; readonly message: string }[] = []; let abandoned: TaskId[] = []; try { - await this.#session.commitWith(async (tx) => { + await this.#session.commitInternal(async (tx) => { if (!this.#enabled || this.#closing) return; - for (const id of this.#abandoned) { - const record = this.#live.get(id); - if (record !== undefined && !record.abortRequested) tx.setTask(withAbortMark(record, "restart")); - } if (this.#abandoned.size > 0) { - abandoned = [...this.#abandoned]; + const ids = [...this.#abandoned]; + const unmarked = await this.#region(() => + ids + .map((id) => this.#liveRecord(id)) + .filter((record) => record !== undefined && !record.abortRequested), + ); + for (const record of unmarked) tx.setTask(withAbortMark(record!, "restart")); + abandoned = ids; return; } await this.#loadScopes(false); // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after // a deadline, and the wake it schedules no longer keeps a timer for a due sleep. this.#wakeDue(); - this.#unparkRaised(); + const plan = await this.#region(() => this.#planReservation(), this.#reloadScopes); + for (const id of plan.dropped) this.#runnable.delete(id); + // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule + // the cascade, which may not have run yet. + if (plan.cascade) { + this.#cascadePending = true; + this.#scheduleReconcile(); + } // Taken once per pass, and only when some task is a candidate. let snapshot: RegistrySnapshot | undefined; - for (const id of [...this.#runnable]) { - const record = this.#live.get(id); - if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) { - this.#runnable.delete(id); - continue; - } - // It returns once what it waits for ends, its record changes, or the work below it ends. - if (this.#waitingOn(record, true).length > 0) { - this.#runnable.delete(id); - continue; - } - // It returns at the first pass after its time comes, which the timer, `resume()`, or any change starts. - if (this.#notDue(record)) { - this.#runnable.delete(id); - continue; - } - // It returns when `retry()` clears its restart count, or at a pass after `settings.restartLimit` rose. - if (this.#parked(record)) { - this.#runnable.delete(id); - // Newly parked without a commit, as after a lowered limit: the graph must hear it. - if (!this.#parkedTasks.has(id)) queueMicrotask(() => this.#notifyBlocked()); - this.#parkedTasks.add(id); - continue; - } - const runnable = record as RunnableTaskRecord; + for (const record of plan.ready) { + const id = record.id; const mode = record.abortRequested ? "abort" : "run"; - // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule - // the cascade, which may not have run yet. - if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) { - this.#cascadePending = true; - this.#scheduleReconcile(); - continue; - } - // An abort handler cut off `restartLimit` times in a row would run forever; nothing is left to park it for. - if (mode === "abort" && (record.restarts ?? 0) >= this.#restartLimit()) { - const message = `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`; + // Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that + // does not end itself at the limit, ends the task `faulted` instead. + // The settings are read only for a task that was cut off at all. + const cutOff = record.restarts !== undefined && record.restarts >= this.#restartLimit(); + if (cutOff && (mode === "abort" || !this.#limitsRestarts(record.kind))) { + const message = + mode === "abort" + ? `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times` + : `Task ${id} (${record.kind}) was cut off ${record.restarts} times without progress`; await this.#terminate(tx, record, { status: "faulted", error: { message, detail: { reason: "restart_limit" } }, @@ -1544,7 +1943,7 @@ export class TaskScheduler { continue; } snapshot ??= this.#registry.snapshot(); - const resolution = this.#resolve(runnable, snapshot); + const resolution = this.#resolve(record, snapshot); if (resolution.kind === "blocked") { // An abandoned task waits for its definition, so its abort handler can clean up. if (mode === "abort" && record.abortReason === undefined) { @@ -1552,7 +1951,7 @@ export class TaskScheduler { } continue; } - if (resolution.record !== runnable || record.state.status !== "running") { + if (resolution.record !== record || record.state.status !== "running") { tx.setTask( withState(resolution.record, { status: "running", @@ -1563,7 +1962,7 @@ export class TaskScheduler { // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a // candidate again. this.#runnable.delete(id); - const invocation = this.#createInvocation(runnable, mode, installerOf(snapshot, resolution.task)); + const invocation = this.#createInvocation(record, mode, installerOf(snapshot, resolution.task)); reservations.push({ invocation, task: resolution.task, snapshot }); } }, this.#context); @@ -1577,7 +1976,7 @@ export class TaskScheduler { } for (const { record, message } of faulted) { this.#report(new Error(message), { - type: "task_faulted", + kind: "task_faulted", ...this.#aboutTask(recordTask(record)), reason: "restart_limit", }); @@ -1588,6 +1987,46 @@ export class TaskScheduler { return reservations; } + /** + * A reservation pass's decision region: the candidates it drops (gone, holding, waiting, asleep, or with an + * invocation), whether one waits for a cascade, and the runnable records to resolve. Iterating the candidates is not + * keyed: a pass that misses a task a pending commit makes runnable only delays it, as does skipping one with an + * invocation without reading its record, or a waiter's check whether what it waits on is still live. + */ + #planReservation(): { + readonly dropped: readonly TaskId[]; + readonly cascade: boolean; + readonly ready: readonly RunnableTaskRecord[]; + } { + const dropped: TaskId[] = []; + const ready: RunnableTaskRecord[] = []; + let cascade = false; + for (const id of this.#runnable) { + if (this.#invocations.has(id)) { + dropped.push(id); + continue; + } + const record = this.#liveRecord(id); + // It returns once what it waits for ends, its record changes, or the work below it ends; a sleep at the first pass + // after its time comes, which the timer, `resume()`, or any change starts. + if ( + record === undefined || + record.state.status === "completing" || + this.#waitingOn(record, true).length > 0 || + this.#notDue(record) + ) { + dropped.push(id); + continue; + } + if (!record.abortRequested && !record.background && this.#belowCancelled(parentOf(record))) { + cascade = true; + continue; + } + ready.push(record as RunnableTaskRecord); + } + return { dropped, cascade, ready }; + } + /** * Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since * abort runs bottom-up, otherwise the live part of the `on` of a wait. @@ -1618,7 +2057,7 @@ export class TaskScheduler { queueMicrotask(() => this.#notifyBlocked()); this.#scheduleWake(); const extension = installerOf(snapshot, fit.task); - this.#report(error, { type: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) }); + this.#report(error, { kind: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) }); return { kind: "blocked", reason: "migration_failed" }; } } @@ -1635,7 +2074,7 @@ export class TaskScheduler { /** Sizes of the ownership indexes, for tests that bound the scheduler's memory. */ get indexSizes(): Readonly< - Record<"live" | "settled" | "edges" | "below" | "roots" | "unloaded" | "dropQueue", number> + Record<"live" | "settled" | "edges" | "below" | "roots" | "unloaded" | "dropQueue" | "pins", number> > { let below = 0; for (const children of this.#below.values()) below += children.size; @@ -1647,6 +2086,33 @@ export class TaskScheduler { roots: this.#roots.size, unloaded: this.#unloaded.size, dropQueue: this.#dropQueue.size, + pins: this.#pins.size, + }; + } + + /** + * The kept contexts' idle state, for tests (a diagnostic, computed on demand): how many are kept and how many are + * idle, how many disagree with a full recompute (an idle period without `#idle()`, or `#idle()` without one; an + * expiry order that differs from the idle periods, or is not sorted), and how many idle checks covered every scope. + */ + get idleState(): Readonly> { + let mismatches = 0; + let idle = 0; + for (const [conversationId, kept] of this.#contexts) { + if (kept.idleSince !== undefined) idle++; + if ((kept.idleSince !== undefined) !== this.#idle(conversationId)) mismatches++; + if (this.#idleOrder.get(conversationId) !== kept.idleSince) mismatches++; + } + let previous = Number.NEGATIVE_INFINITY; + for (const [conversationId, idleSince] of this.#idleOrder) { + if (idleSince < previous || this.#contexts.get(conversationId)?.idleSince !== idleSince) mismatches++; + previous = idleSince; + } + return { + contexts: this.#contexts.size, + idleContexts: idle, + idleMismatches: mismatches, + fullIdleChecks: this.#fullIdleChecks, }; } @@ -1658,6 +2124,7 @@ export class TaskScheduler { async inspect( snapshot: RegistrySnapshot, ): Promise<{ scheduling: HarnessInspection["scheduling"]; tasks: TaskInspection[] }> { + await this.#published(); await this.#loadScopes(false); const tasks: TaskInspection[] = []; for (const record of this.#live.values()) tasks.push({ record, state: this.#inspectTask(record, snapshot) }); @@ -1671,13 +2138,12 @@ export class TaskScheduler { const on = this.#waitingOn(record); if (on.length > 0) return { kind: "waiting", on }; if (this.#notDue(record)) return { kind: "sleeping", until: record.state.until }; - if (this.#parked(record)) return { kind: "blocked", reason: "restart_limit" }; const fit = this.#fit(record, snapshot.task(record.kind)); if ("reason" in fit) return { kind: "blocked", ...fit }; if (fit.migrates && fit.task.definition.migrate === undefined) { return { kind: "blocked", reason: "migration_failed", error: missingMigration(record, erased(fit.task)) }; } - return { kind: "ready", migrates: fit.migrates }; + return { kind: "pending", migrates: fit.migrates }; } #createInvocation(record: RunnableTaskRecord, mode: "run" | "abort", extension: string | undefined): Invocation { @@ -1710,7 +2176,7 @@ export class TaskScheduler { if (invocation.mode === "run") await this.#run(reservation); else await this.#runAbort(reservation); } catch (error) { - this.#report(error, { type: "task_error", ...this.#aboutTask(invocation) }, invocation.context); + this.#report(error, { kind: "task_error", ...this.#aboutTask(invocation) }, invocation.context); } finally { this.#end(invocation); invocation.record = undefined; @@ -1728,9 +2194,9 @@ export class TaskScheduler { const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined as ReportedTask }; const phase: Phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined }; const runtime = this.#runtime(invocation, phase); - let current = await this.#step(invocation, (tx, next) => - this.#decide(tx, next, undefined, state, invocation.extension, invocation.context), - ); + const progress = (tx: Transaction, current: ErasedRunningTask) => + this.#progress(tx, current, state, invocation.extension, invocation.context); + let current = await this.#step(invocation, (task) => this.#rules(task, undefined), progress); while (current !== undefined && !this.#closing) { const task = current; current = await this.#phase(invocation, task, async () => { @@ -1744,9 +2210,7 @@ export class TaskScheduler { } catch (error) { previous = { checkpoint, failure: { error } }; } - return this.#step(invocation, (tx, next) => - this.#decide(tx, next, previous, state, invocation.extension, invocation.context), - ); + return this.#step(invocation, (next) => this.#rules(next, previous), progress); }); } } @@ -1818,53 +2282,56 @@ export class TaskScheduler { } /** - * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, `waiting`, or `sleeping`) and 2 - * (closing) are applied by `#step`. Returns whether the invocation continues with the next phase. + * Precedence rules for a run invocation, a decision region of its step. Rules 1 (terminal, `completing`, `waiting`, or + * `sleeping`) and 2 (closing) are applied by `#step`. Progress (rule 5) is decided by `#progress()` once the decision + * stands, as it takes a registry snapshot. */ - #decide( - tx: Transaction, - current: ErasedRunningTask, - previous: PhaseResult | undefined, - state: { task: AnyTask; snapshot: RegistrySnapshot; reported: ReportedTask }, - extension: string | undefined, - context: Context, - ): Decision { + #rules(current: ErasedRunningTask, previous: PhaseResult | undefined): StepPlan { // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone. - if (current.abortRequested) return false; + if (current.abortRequested) return END; // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back. - if (!current.background && this.#belowCancelled(parentOf(current))) { - this.#cascadePending = true; - this.#scheduleReconcile(); - return false; - } - if (previous === undefined) return true; + if (!current.background && this.#belowCancelled(parentOf(current))) return CASCADE; + if (previous === undefined) return CONTINUE; // 4. uncaught error. - if (previous.failure !== undefined) return { fault: previous.failure.error }; + if (previous.failure !== undefined) return { kind: "fault", error: previous.failure.error }; // 6. no durable progress. if (jsonEqual(current.state.checkpoint, previous.checkpoint)) { const message = `Task ${current.kind} phase ${previous.checkpoint.phase} returned without durable progress`; - return { fault: new Error(message) }; + return { kind: "fault", error: new Error(message) }; } - // 5. progress: refresh the snapshot; hand over to a replacement definition that can take the task. + return PROGRESS; + } + + /** + * Rule 5, progress: refresh the snapshot; hand over to a replacement definition that can take the task, writing it + * back as `pending`, or keep running under the old one, reported once per resolved definition. Returns whether the + * invocation continues. + */ + #progress( + tx: Transaction, + current: ErasedRunningTask, + state: { task: AnyTask; snapshot: RegistrySnapshot; reported: ReportedTask }, + extension: string | undefined, + context: Context, + ): boolean { state.snapshot = this.#registry.snapshot(); const next = state.snapshot.task(current.kind); - if (next !== state.task) { - if (next !== undefined && canReserve(next, current)) { - tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint })); - return false; - } - if (state.reported === undefined || state.reported.task !== next) { - state.reported = { task: next }; - this.#report( - new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), - { - type: "task_outdated", - ...this.#aboutTask({ ...recordTask(current), extension }), - reason: next === undefined ? "missing_task" : "incompatible_task", - }, - context, - ); - } + if (next === state.task) return true; + if (next !== undefined && canReserve(next, current)) { + tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint })); + return false; + } + if (state.reported === undefined || state.reported.task !== next) { + state.reported = { task: next }; + this.#report( + new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), + { + kind: "task_outdated", + ...this.#aboutTask({ ...recordTask(current), extension }), + reason: outdatedReason(next, current), + }, + context, + ); } return true; } @@ -1888,31 +2355,44 @@ export class TaskScheduler { failure = { error }; } const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`; - await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) })); + await this.#step(invocation, () => ({ kind: "fault", error: failure?.error ?? new Error(message) })); }); } /** - * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`, - * `waiting`, or `sleeping`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may - * stage a write and returns whether the invocation continues. Ending happens inside the callback, before a fault's - * Harness cleanup. A rejected step, such as admission after close, also ends the invocation. + * One decision on the Session line, before a phase: a decision region (`#region()`) that reads the task's record. A + * task that is no longer running (rule 1: terminal, `completing`, `waiting`, or `sleeping`) or a closing Harness (rule + * 2) ends the invocation without a write; otherwise `plan` decides, and a decision that stands is applied: the + * invocation continues, with `progress` for rule 5, or ends, with a fault written. Ending happens inside the callback, + * before a fault's Harness cleanup. A rejected step, such as admission after close, also ends the invocation. Its + * outcome goes to the invocation only, so it is a scheduler commit (`commitInternal()`). */ async #step( invocation: Invocation, - decide: (tx: Transaction, current: ErasedRunningTask) => Decision, + plan: (current: ErasedRunningTask) => StepPlan, + progress?: (tx: Transaction, current: ErasedRunningTask) => boolean, ): Promise { let fault: { readonly error: unknown } | undefined; try { - const continued = await this.#session.commitWith(async (tx) => { - const found = this.#live.get(invocation.taskId); - const current = found?.state.status === "running" ? (found as ErasedRunningTask) : undefined; - const decision = current !== undefined && !this.#closing ? decide(tx, current) : false; - if (decision === true) return current; + const continued = await this.#session.commitInternal(async (tx) => { + const decided = await this.#region(() => { + const found = this.#liveRecord(invocation.taskId); + if (found?.state.status !== "running" || this.#closing) return { plan: END }; + const current = found as ErasedRunningTask; + return { current, plan: plan(current) }; + }); + const { current } = decided; + const step = decided.plan; + if (step.kind === "continue") return current; + if (step.kind === "progress" && progress!(tx, current!)) return current; this.#end(invocation); - if (decision !== false) { - fault = { error: decision.fault }; - const message = errorMessage(decision.fault); + if (step.kind === "cascade") { + this.#cascadePending = true; + this.#scheduleReconcile(); + } + if (step.kind === "fault") { + fault = { error: step.error }; + const message = errorMessage(step.error); invocation.fault = { status: "faulted", error: { message } }; await this.#terminate(tx, current!, invocation.fault); } @@ -1922,7 +2402,7 @@ export class TaskScheduler { if (fault !== undefined) this.#report( fault.error, - { type: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, + { kind: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context, ); return continued; @@ -1941,7 +2421,7 @@ export class TaskScheduler { */ async #terminate(tx: Transaction, record: AnyTaskRecord, outcome: SchedulerOutcome): Promise { await this.#loadScopes(false); - if (this.#hasOwnedLive(record.id, overlayOf(tx))) { + if (await this.#region(() => this.#hasOwnedLive(record.id, overlayOf(tx)), this.#reloadScopes)) { tx.setTask(withState(record, { status: "completing", outcome })); return; } @@ -1966,7 +2446,7 @@ export class TaskScheduler { const overlay = overlayOf(tx); await this.#loadScopes(false); for (const record of overlay.tasks.values()) await this.#loadChain(parentOf(record), overlay); - if (this.#hasOwnedLive(current.id, overlay)) { + if (await this.#region(() => this.#hasOwnedLive(current.id, overlayOf(tx)), this.#reloadScopes)) { tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome })); return; } @@ -1993,37 +2473,71 @@ export class TaskScheduler { policy: string, ): Promise { if (invocation.mode === "abort") throw new Error(`Abort handler of task ${current.id} cannot wait`); - const overlay = overlayOf(tx); + // Owned work is found by walks down, which need the chains of every live task loaded. + await this.#loadScopes(false); + // Tasks in `on` that are not live, read from Storage: terminal, or missing. + const ended = new Map(); + for (;;) { + const checked = await this.#region( + () => this.#checkWait(current, on, policy, overlayOf(tx), ended), + this.#reloadScopes, + ); + if (checked === undefined) return; + if ("missing" in checked) { + for (const id of checked.missing) + ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null); + continue; + } + throw checked.error; + } + } + + /** + * `#validateWait()`'s decision region: why the wait is refused, `undefined` when it may be stored, or the tasks in + * `on` it needs from Storage first, when `ended` lacks them. + */ + #checkWait( + current: ErasedRunningTask, + on: readonly TaskId[], + policy: string, + overlay: Overlay, + ended: ReadonlyMap, + ): { readonly missing: readonly TaskId[] } | { readonly error: Error } | undefined { const targets: TaskId[] = []; + const missing: TaskId[] = []; for (const id of on) { - const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context)); - if (member === undefined) throw new Error(`Task ${id} does not exist`); + const member = overlay.tasks.get(id) ?? this.#liveRecord(id) ?? ended.get(id); + if (member === undefined) { + missing.push(id); + continue; + } + if (member === null) return { error: new Error(`Task ${id} does not exist`) }; if (policy === "failFast" && member.owner !== current.id) { - throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`); + return { error: new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`) }; } // A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A // terminal task has no edges. if (member.state.status !== "terminal" && member.owner !== current.id) targets.push(id); } - if (targets.length === 0) return; - const cycle = await this.#waitCycle(current.id, targets, overlay); - if (cycle !== undefined) { - throw new Error( + if (missing.length > 0) return { missing }; + if (targets.length === 0) return undefined; + const cycle = this.#waitCycle(current.id, targets, overlay); + if (cycle === undefined) return undefined; + return { + error: new Error( `Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`, - ); - } + ), + }; } /** * The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1). * An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live - * ordinary owned work of X, - * which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the - * live work it reaches. + * ordinary owned work of X, which holds X as `completing`. Each task is visited once and each subtree walked once, + * so a check is bounded by the live work it reaches. Part of a decision region: it records every record it reads. + * Owned work is found by walks down, which need the chains of every live task loaded. */ - async #waitCycle(task: TaskId, on: readonly TaskId[], overlay: Overlay): Promise { - // Owned work is found by walks down, which need the chains of every live task loaded. - await this.#loadScopes(false); + #waitCycle(task: TaskId, on: readonly TaskId[], overlay: Overlay): TaskId[] | undefined { const from = new Map(); const walked = new Set(); const pending: TaskId[] = []; @@ -2040,7 +2554,7 @@ export class TaskScheduler { for (const id of on) reach(id, undefined); for (let id = pending.pop(); id !== undefined; id = pending.pop()) { if (id === task) return path(id); - const record = overlay.tasks.get(id) ?? this.#live.get(id); + const record = overlay.tasks.get(id) ?? this.#liveRecord(id); if (record === undefined || record.state.status === "terminal") continue; // An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`). if (record.state.status === "waiting" && !record.abortRequested) { @@ -2079,6 +2593,8 @@ export class TaskScheduler { /** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */ #idle(conversationId: ConversationId | undefined): boolean { const scope: Scope = conversationId === undefined ? { roots: true } : { conversation: conversationId }; + // In a decision region, global idle reads the roots it starts from. + if (conversationId === undefined) this.#record(ROOTS_KEY); // Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up // decides otherwise. for (const id of this.#unloaded) { @@ -2119,7 +2635,7 @@ export class TaskScheduler { if (invocation.controller.signal.aborted) throw error; this.#report( error, - { type: "extension_error", extension, task: reportTask(invocation) }, + { kind: "extension_error", extension, task: reportTask(invocation) }, invocation.context, ); } @@ -2145,8 +2661,15 @@ export class TaskScheduler { get registry() { return phase.snapshot(); }, - commit: async (change, context) => { + commit: async (change, context, options) => { let checkpoint: string | undefined; + let interrupted: + | { + readonly message: string; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; + } + | undefined; await this.#gated( invocation, async (tx, current) => { @@ -2155,9 +2678,35 @@ export class TaskScheduler { if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) { checkpoint = next.checkpoint.phase; } + // A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal. + if (next?.status === "terminal") { + const restarts = current.restarts ?? 0; + const reason = next.outcome.status === "failed" ? interruptionOf(next.outcome.error) : undefined; + if (reason !== undefined && next.outcome.status === "failed") { + interrupted = { message: next.outcome.error.message, reason, restarts }; + } else { + // A task that ended for crashes with another outcome says so itself, as a generation whose run + // answered ends `completed` without its after-run tasks. + const message = endedByCrashesStates.get(next); + if (message !== undefined) interrupted = { message, reason: "restart_limit", restarts }; + } + } }, context, + options, ); + if (interrupted !== undefined) { + this.#report( + new Error(interrupted.message), + { + kind: "task_interrupted", + ...this.#aboutTask(invocation), + reason: interrupted.reason, + restarts: interrupted.restarts, + }, + invocation.context, + ); + } if (checkpoint !== undefined) { invocation.context.telemetry.addEvent("pi.durable.checkpoint", { phase: checkpoint, @@ -2197,7 +2746,7 @@ export class TaskScheduler { )) as ErasedRuntime["snapshotAsOf"], getTask: ((id: TaskId, context: Context) => this.#read(invocation, () => - this.#session.readOnLine(() => this.#storage.task(id, context)), + this.#session.readOnLine((storage) => storage.task(id, context)), )) as ErasedRuntime["getTask"], waitForTask: ((id: TaskId, context: Context) => this.#read(invocation, () => @@ -2210,11 +2759,15 @@ export class TaskScheduler { if (invocation.ended) throw endedError(invocation); context.abortSignal?.throwIfAborted(); // Every live task is listed under its parent node, its owner task when it has one. - const owned: AnyTaskRecord[] = []; - for (const node of this.#below.get(invocation.taskId) ?? []) { - const record = typeof node === "string" ? undefined : this.#live.get(node); - if (record?.owner === invocation.taskId) owned.push(record); - } + const owned = await this.#region(() => { + this.#record(ownedKey(invocation.taskId)); + const records: AnyTaskRecord[] = []; + for (const node of this.#below.get(invocation.taskId) ?? []) { + const record = typeof node === "string" ? undefined : this.#liveRecord(node); + if (record?.owner === invocation.taskId) records.push(record); + } + return records; + }); // Copies, as from storage: the scheduler reads its own. return owned .sort((a, b) => a.id - b.id) @@ -2224,7 +2777,7 @@ export class TaskScheduler { abortOwned: (id, context) => this.#read(invocation, async () => { const bound = withAbortSignal(invocation.controller.signal, context); - const record = await this.#session.readOnLine(() => this.#storage.task(id, bound)); + const record = await this.#session.readOnLine((storage) => storage.task(id, bound)); if (record?.owner !== invocation.taskId) { throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`); } @@ -2234,10 +2787,10 @@ export class TaskScheduler { }), outcomes: ((ids: readonly TaskId[], context: Context) => this.#read(invocation, () => - this.#session.readOnLine(async () => { + this.#session.readOnLine(async (storage) => { const outcomes: TaskOutcome[] = []; for (const id of ids) { - const state = (await this.#storage.task(id, context))?.state; + const state = (await storage.task(id, context))?.state; if (state?.status !== "terminal") throw new Error(`Task ${id} is not terminal`); outcomes.push(state.outcome); } @@ -2263,16 +2816,24 @@ export class TaskScheduler { ? [undefined, args[0] as EntryId, args[1] as Context] : [args[0] as { readonly kind: string }, args[1] as EntryId, args[2] as Context]; return this.#read(invocation, async () => { - const found = await this.#session.readOnLine(() => - this.#storage.entry(invocation.conversationId, id, context), - ); - const entry: EntryRecord | undefined = found?.entry; + // An entry of the conversation's kept context range is committed and visible: no Storage read. + const kept = this.#contexts.get(invocation.conversationId); + const entry = + (kept === undefined ? undefined : rangeEntry(kept.range, id)) ?? + (await this.#session.readOnLine((storage) => storage.entry(invocation.conversationId, id, context))) + ?.entry; return token === undefined || entry?.kind === token.kind ? entry : undefined; }); }) as ErasedRuntime["entry"], + entries: (query, limit, cursor, context) => + this.#read(invocation, () => + this.#session.readOnLine((storage) => + storage.scanEntries(conversationEntries(invocation.conversationId, query), limit, cursor, context), + ), + ), context: (conversationId, context, options) => this.#read(invocation, async () => { - const { view, range } = await readContextFrom( + const read = await readContextFrom( this.#session, this.#storage, conversationId, @@ -2280,6 +2841,14 @@ export class TaskScheduler { options?.at, this.#contexts.get(conversationId)?.range, ); + const { view } = read; + // Checked again here, synchronously up to the install below: a commit published since the read returned + // was not seen by `#advanceContexts`, so a range that missed it must not be advanced. + const range = + read.range?.current === true && + this.#session.knownLatestEntryId(conversationId) !== read.range.bounds.tail + ? { ...read.range, current: false } + : read.range; // Keep it unless the invocation ended or a concurrent read already kept a newer range. const kept = this.#contexts.get(conversationId); if ( @@ -2291,8 +2860,16 @@ export class TaskScheduler { // A read of another, idle conversation starts or continues its retention period. if (!this.#idle(conversationId)) { this.#contexts.set(conversationId, { range, idleSince: undefined }); + this.#idleOrder.delete(conversationId); } else if (this.#contextRetentionMs() > 0) { - this.#contexts.set(conversationId, { range, idleSince: kept?.idleSince ?? this.#now() }); + // An idle period that already started keeps its expiry and its place; a new one starts now. + const idleSince = kept?.idleSince ?? this.#now(); + const expiresAt = + kept?.idleSince !== undefined && kept.expiresAt !== undefined + ? kept.expiresAt + : idleSince + this.#contextRetentionMs(); + this.#contexts.set(conversationId, { range, idleSince, expiresAt }); + if (kept?.idleSince === undefined) this.#markIdle(conversationId, idleSince); this.#scheduleExpiry(); } } @@ -2322,16 +2899,25 @@ export class TaskScheduler { this.#report( error, from === undefined - ? { type: "task_error", ...this.#aboutTask(invocation) } - : from.type === "extension" - ? { type: "extension_error", task, ...optionalExtension(from.extension) } - : { type: "host_error", callback: from.callback, task }, + ? { kind: "task_error", ...this.#aboutTask(invocation) } + : from.kind === "extension" + ? { kind: "extension_error", task, ...optionalExtension(from.extension) } + : { kind: "host_error", callback: from.callback, task }, invocation.context, ); }, }; } + /** + * On the line: wait until every commit admitted before is published, for a decision that reads what cannot be keyed, + * every live task (open, inspection) or every task a conversation abort reaches. + */ + async #published(): Promise { + const gate = this.#session.touch(ALL_KEYS); + if (gate !== undefined) await gate; + } + /** Run a committed-state read unless the invocation has ended. */ async #read(invocation: Invocation, read: () => Promise): Promise { if (invocation.ended) throw endedError(invocation); @@ -2343,23 +2929,30 @@ export class TaskScheduler { invocation: Invocation, change: (tx: Transaction, current: ErasedRunningTask) => T | Promise, context: Context, + options?: CommitOptions, ): Promise { if (invocation.ended) return Promise.reject(endedError(invocation)); return this.#session.commitWith( async (tx) => { if (invocation.ended) throw endedError(invocation); if (this.#closing) throw closedError(this.#session); - const found = this.#live.get(invocation.taskId); - if (found === undefined) throw new Error(`Task ${invocation.taskId} is terminal`); - if (found.state.status !== "running") throw new Error(`Task ${invocation.taskId} is ${found.state.status}`); - const current = found as ErasedRunningTask; - if (invocation.mode === "run" && current.abortRequested) { - throw new Error(`Task ${invocation.taskId} has a durable abort mark`); - } + const current = await this.#region(() => { + const found = this.#liveRecord(invocation.taskId); + if (found === undefined) throw new Error(`Task ${invocation.taskId} is terminal`); + if (found.state.status !== "running") + throw new Error(`Task ${invocation.taskId} is ${found.state.status}`); + if (invocation.mode === "run" && found.abortRequested) { + throw new Error(`Task ${invocation.taskId} has a durable abort mark`); + } + return found as ErasedRunningTask; + }); + // The step that ends it may have run while the region waited. + if (invocation.ended) throw endedError(invocation); return change(tx, current); }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId }, + options, ); } @@ -2377,9 +2970,10 @@ export class TaskScheduler { } } -const REGISTRY: ReportInfo = { type: "host_error", callback: "registry" }; -const SETTINGS: ReportInfo = { type: "host_error", callback: "settings" }; -const ON_WAKE: ReportInfo = { type: "host_error", callback: "onWake" }; +const REGISTRY: ReportInfo = { kind: "host_error", callback: "registry" }; +const SETTINGS: ReportInfo = { kind: "host_error", callback: "settings" }; +const ON_WAKE: ReportInfo = { kind: "host_error", callback: "onWake" }; +const TELEMETRY: ReportInfo = { kind: "host_error", callback: "telemetry" }; /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */ function installerOf(snapshot: RegistrySnapshot, task: AnyTask | undefined): string | undefined { @@ -2405,6 +2999,26 @@ function recordTask(record: Pick(); + +/** `state`, marked as ending its task for crashes (`task_interrupted` with `message`), whatever its outcome. */ +export function endedByCrashes(state: T, message: string): T { + endedByCrashesStates.set(state, message); + return state; +} + +function interruptionOf(error: TaskOutcomeError): "restart_limit" | "interrupted" | undefined { + const detail = error.detail; + if (typeof detail !== "object" || detail === null || !("reason" in detail)) return undefined; + return detail.reason === "restart_limit" || detail.reason === "interrupted" ? detail.reason : undefined; +} + function withAbortMark(record: AnyTaskRecord, reason?: "restart"): AnyTaskRecord { const { abortReason: _reason, restarts: _restarts, ...rest } = record; return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) } as AnyTaskRecord; @@ -2415,6 +3029,19 @@ function cancellationIntent(record: AnyTaskRecord): boolean { return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record)); } +/** + * Whether replacing `previous` (absent: a new task) with `next` changes the cancellation intent set or what it means: a + * mark set, upgraded, or cleared, or a held or ended outcome other than `completed`. + */ +function intentChanges(previous: AnyTaskRecord | undefined, next: AnyTaskRecord): boolean { + return ( + (previous?.abortRequested ?? false) !== next.abortRequested || + previous?.abortReason !== next.abortReason || + (previous !== undefined && failedOutcome(previous)) !== failedOutcome(next) || + (previous !== undefined && cancellationIntent(previous)) !== cancellationIntent(next) + ); +} + /** Whether the record holds or ends with an outcome other than `completed`. */ function failedOutcome(record: AnyTaskRecord): boolean { const state = record.state; @@ -2493,6 +3120,15 @@ function withState(record: AnyTaskRecord, state: TaskState } /** Whether a definition can take the task at reservation: same version, or newer with a migration. */ +/** + * Why a replacement definition cannot take a running task, in `#fit()`'s words: none is registered, it is older than + * the task, or it is newer without a `migrate`. + */ +function outdatedReason(task: AnyTask | undefined, record: AnyTaskRecord): BlockedReason { + if (task === undefined) return "missing_task"; + return task.definition.version < record.version ? "task_too_old" : "migration_failed"; +} + function canReserve(task: AnyTask, record: AnyTaskRecord): boolean { const definition = task.definition; return ( diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 0cf838a93..bac428f44 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -6,17 +6,9 @@ import { type TelemetryContext, type TelemetrySpan, } from "@earendil-works/pi-telemetry"; -import { ConversationBusy } from "../errors.ts"; -import type { SessionImpl } from "../session/session.ts"; -import type { - CommitPublication, - ConversationId, - JsonObject, - Storage, - SubmissionId, - SubmissionRecord, - Tx, -} from "../types.ts"; +import { ConversationBusy, ConversationIdle, ReadAfterWrite } from "../errors.ts"; +import type { ObservedCommit, SessionImpl } from "../session/session.ts"; +import type { ConversationId, JsonObject, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; import { startRun } from "./generation.ts"; import { appendInput, @@ -42,7 +34,6 @@ type Admission = { readonly id: SubmissionId; readonly status: "queued" | "place export class Submissions { readonly #telemetry: TelemetryContext; readonly #session: SessionImpl; - readonly #storage: Storage; readonly #now: () => number; /** Read at each admission, on the Session line. */ readonly #queueModes: () => QueueModes; @@ -53,7 +44,6 @@ export class Submissions { constructor( session: SessionImpl, - storage: Storage, now: () => number, queueModes: () => QueueModes, resume: () => void, @@ -61,11 +51,10 @@ export class Submissions { ) { this.#telemetry = telemetry; this.#session = session; - this.#storage = storage; this.#now = now; this.#queueModes = queueModes; this.#resume = resume; - session.observeCommits((publication) => this.#observe(publication)); + session.followCommits((publication) => this.#observe(publication)); session.subscribeClose(() => { this.#closed = true; this.#waiters.rejectAll(closedError(this.#session)); @@ -74,8 +63,40 @@ export class Submissions { /** Admit a submission in one commit; see `admitSubmission()`. */ async submit(conversationId: ConversationId, draft: SubmissionDraft, context: Context): Promise { + const { id } = await this.#traced(conversationId, draft, context, (span) => + this.#session.commitWith( + (tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), + withTelemetry(span, context), + ), + ); + return new SubmissionHandle(id, this); + } + + /** + * Admit a submission inside the caller's commit `tx`, with the rules of `submit()`. The admission is stored or lost + * with that commit. It must come before the commit's first table write, another admission's included. + */ + async admit(tx: Tx, conversationId: ConversationId, draft: SubmissionDraft): Promise { + if (!this.#session.ownsTransaction(tx)) throw new Error("The transaction belongs to another Session"); + // Enforced on every path: admission reads tables only on some (request IDs, an idle boundary, a steer's trace). + if (tx.hasTableWrite) throw new ReadAfterWrite("submit(tx, draft)"); + const admission = await this.#traced(conversationId, draft, tx.context, (span) => + admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), + ); + return admission.id; + } + + /** Run one admission in its `pi.durable.submit` span, asking for progress first, as the run it starts needs it. */ + #traced( + conversationId: ConversationId, + draft: SubmissionDraft, + context: Context, + admit: (span: TelemetrySpan) => Promise, + ): Promise { this.#resume(); - const parent = draft.type === "input" ? this.#telemetry : context.telemetry; + // Inputs and continues start runs: root traces linked to their callers. + const startsRun = draft.type !== "write"; + const parent = startsRun ? this.#telemetry : context.telemetry; return createTypedSpanStarter(parent, [DURABLE_TELEMETRY_SCHEMA])( "pi.durable.submit", { @@ -83,14 +104,11 @@ export class Submissions { "conversation.id": conversationId, }, async (span) => { - if (draft.type === "input") span.addLink({ context: context.telemetry }); + if (startsRun) span.addLink({ context: context.telemetry }); try { - const { id, status } = await this.#session.commitWith( - (tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), - withTelemetry(span, context), - ); - span.setAttributes({ "submission.id": id, result: status }); - return new SubmissionHandle(id, this); + const admission = await admit(span); + span.setAttributes({ "submission.id": admission.id, result: admission.status }); + return admission; } catch (error) { // Error text may contain caller content. Record only the status. span.setStatus({ status: "error" }); @@ -102,12 +120,12 @@ export class Submissions { /** Handle for an existing submission, or `undefined`. */ async get(id: SubmissionId, context: Context): Promise { - const record = await this.#session.readOnLine(() => this.#storage.submission(id, context)); + const record = await this.#session.readOnLine((storage) => storage.submission(id, context)); return record === undefined ? undefined : new SubmissionHandle(record.id, this); } async status(id: SubmissionId, context: Context): Promise { - const record = await this.#session.readOnLine(() => this.#storage.submission(id, context)); + const record = await this.#session.readOnLine((storage) => storage.submission(id, context)); if (record === undefined) throw new Error(`Submission ${id} does not exist`); return record; } @@ -115,8 +133,8 @@ export class Submissions { async wait(id: SubmissionId, context: Context): Promise { this.#resume(); // Check and register on the line so no settling publication falls between them. - const found = await this.#session.readOnLine(async () => { - const record = await this.#storage.submission(id, context); + const found = await this.#session.readOnLine(async (storage) => { + const record = await storage.submission(id, context); if (record === undefined) throw new Error(`Submission ${id} does not exist`); if (isSettled(record)) return { promise: Promise.resolve(record) }; // Close rejects registered waiters synchronously and may begin during the read. @@ -142,7 +160,9 @@ export class Submissions { }, context); } - #observe(publication: CommitPublication): void { + #observe(publication: ObservedCommit): void { + // A commit held in memory changes documents only. + if (publication.kind === "memory") return; for (const change of publication.changes) { if (change.type !== "submission" || !isSettled(change.value)) continue; this.#waiters.resolve(change.value.id, change.value); @@ -189,14 +209,16 @@ function checkInputEntry(entry: InputEntry): void { } /** - * Admit a submission inside a commit (spec §6); `Conversation.submit()` and conversation-owned compactions share it. A - * known request ID returns its existing submission without writing. A display-only write (only `kind` and `data`) is - * appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a queued reset it - * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues anything else in - * `pi.inbox`, or rejects `whenBusy: "reject"` input with `ConversationBusy`. An idle conversation with queued items - * queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's - * own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head reaches - * before the active range. + * Admit a submission inside a commit (spec §6); both `Conversation.submit()` forms and conversation-owned compactions + * share it. A known request ID returns its existing submission without writing. A display-only write (only `kind` and + * `data`) is appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a + * queued reset it queues, so it lands after the reset instead of before it in the context the reset ends. A busy + * conversation queues anything else in `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with + * `ConversationBusy`; an idle one rejects `whenIdle: "reject"` input with `ConversationIdle`. An idle continue places + * queued items and starts a run with them and itself, placing no entry of its own. An idle conversation with queued + * items queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the + * input's own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head + * reaches before the active range. */ export async function admitSubmission( tx: Tx, @@ -238,6 +260,18 @@ export async function admitSubmission( if (run?.trace !== undefined) span.addLink({ context: span.propagation.extract(run.trace) }); } if (busy && draft.type === "input" && draft.whenBusy === "reject") throw new ConversationBusy(conversationId); + if (!busy && draft.type === "input" && draft.whenIdle === "reject") throw new ConversationIdle(conversationId); + if (draft.type === "continue") { + // Never queued: whether a run can continue depends on how the current one ends. Rejected before any write. + if (busy) throw new ConversationBusy(conversationId); + const boundary = await prepareBoundary(tx, conversationId, queueModes); + const { id } = await tx.createSubmission({ ...fields, type: "continue", status: "placed" }); + // Items an earlier failed run left queued (`afterFailedRun: "keep"`) are placed first and join the run. Its history + // starts with the failed run's, so `afterRun` hooks see what that run did. + const { users, first } = await applyBoundary(tx, boundary, "final", now); + await startRun(tx, conversationId, live, { inputs: [...users, id], first: live.runFailure?.first ?? first }); + return { id, status: "placed" }; + } // A boundary reads the table, so it is prepared before the first table write; a busy one needs none. const boundary = busy ? undefined : await prepareBoundary(tx, conversationId, queueModes); if (boundary === undefined || boundary.inbox.items.length > 0) { @@ -259,8 +293,8 @@ export async function admitSubmission( items.push({ id, mode, content, ...entry }); } if (boundary === undefined) return { id, status: "queued" }; - const { users } = await applyBoundary(tx, boundary, "final", now); - if (users.length > 0) await startRun(tx, conversationId, live, users); + const { users, first } = await applyBoundary(tx, boundary, "final", now); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); return { id, status: draft.type === "write" ? "done" : users.at(-1) === id ? "placed" : "queued" }; } if (draft.type === "write") { @@ -278,6 +312,6 @@ export async function admitSubmission( const entry = await appendInput(tx, conversationId, draft.content, draft.entry, now); const input = { ...fields, type: "input", status: "placed", entry: entry.id } as const; const { id } = await tx.createSubmission(input); - await startRun(tx, conversationId, live, [id]); + await startRun(tx, conversationId, live, { inputs: [id], first: entry.id }); return { id, status: "placed" }; } diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index ec4e55a23..c7fbcb824 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,11 +1,20 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchCall, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; -import { contained, type SessionImpl } from "../session/session.ts"; +import { + closeEndOf, + closeEnds, + contained, + guardObserver, + type ObservedCommit, + observation, + type SessionImpl, +} from "../session/session.ts"; import type { - CommitPublication, ConversationId, JoinPolicy, + Observation, + Observer, Storage, TaskId, TaskOutcome, @@ -14,7 +23,7 @@ import type { import { type BlockedReason, type BlockedSubject, blockedSubject } from "./scheduler.ts"; import { closedError, scanAll } from "./util.ts"; -/** A live task's durable status without its checkpoint and outcome payloads (spec §9.5). */ +/** A live task's durable status without its checkpoint and outcome payloads (spec §9.4). */ export type TaskGraphState = | { readonly status: "pending" | "running"; readonly phase: string } | { @@ -39,14 +48,14 @@ export type TaskGraphNode = { readonly state: TaskGraphState; /** * Why it cannot run here, when it cannot: no registered definition takes it (`missing_task`, `task_too_old`, - * `migration_failed`), or the restart limit parked it (`restart_limit`). Follows registry changes. + * `migration_failed`). Follows registry changes. */ readonly blocked?: BlockedReason; /** Conversations this task owns, in ID order. */ readonly conversations: readonly ConversationId[]; }; -/** Every live task of the Session (spec §9.5). */ +/** Every live task of the Session (spec §9.4). */ export type TaskGraph = { /** Every live task, keyed by its decimal ID. */ readonly tasks: Readonly>; @@ -55,10 +64,10 @@ export type TaskGraph = { export type TaskGraphWatch = WatchHandle; type AnyTaskRecord = TaskRecord; -type Observer = Watch; type Mount = { value: TaskGraph; - readonly observers: Set; + /** Watches and guarded host observers (`observeTaskGraph()`). */ + readonly observers: Set>; /** What the blocked reason reads of each live task, so a registry change can derive it again without payloads. */ readonly records: Map; }; @@ -78,25 +87,25 @@ const SCAN_PAGE_SIZE = 256; */ export class TaskGraphView { readonly #session: SessionImpl; - readonly #storage: Storage; readonly #blocking: TaskGraphBlocking; #mount: Mount | undefined; #closed = false; - constructor(session: SessionImpl, storage: Storage, blocking: TaskGraphBlocking) { + constructor(session: SessionImpl, blocking: TaskGraphBlocking) { this.#session = session; - this.#storage = storage; this.#blocking = blocking; - session.observeCommits((publication, context) => { - if (this.#mount === undefined) return; + session.followCommits((publication, context) => { + // A commit held in memory changes documents only, which the graph does not show. + if (this.#mount === undefined || publication.kind === "memory") return; advance(this.#mount, publication, context, blocking.blocked, (error) => session.report(error)); }); blocking.subscribeBlocked(() => this.#refreshBlocked()); session.subscribeClose(() => { this.#closed = true; + const ends = closeEnds(this.#session.failure); for (const observer of [...(this.#mount?.observers ?? [])]) { contained( - () => observer.end(this.#session.failure), + () => observer.end(closeEndOf(observer, ends)), (error) => session.report(error), ); } @@ -104,7 +113,25 @@ export class TaskGraphView { }); } - /** A serialized watch of the graph; cancelling `context` stops it. */ + /** + * Attach a synchronous observer of the graph (`Harness.observeTaskGraph()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe(observer: Observer, context: Context): Promise> { + let value!: TaskGraph; + const attached = await this.#attach((current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.#session.report(error)); + }, context); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the graph, an observer of its mount; aborting `context` ends it `aborted`. */ async watch(context: Context): Promise { const { observer } = await this.#attach( (value, release) => new Watch(value, release, (error) => this.#session.report(error)), @@ -112,20 +139,23 @@ export class TaskGraphView { ); const signal = context.abortSignal; if (signal?.aborted) { - observer.cancel(); + observer.abort(); throw signal.reason; } - if (signal !== undefined) observer.observeCancellation(signal); + if (signal !== undefined) observer.observeAbort(signal); return observer; } /** Register an observer created from the current revision, atomically on the Session line. */ - #attach( + #attach>( create: (value: TaskGraph, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { - return this.#session.readOnLine(async () => { - const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set() }; + return this.#session.readOnLine(async (storage) => { + const mount = this.#mount ?? { + ...(await this.#build(storage, context)), + observers: new Set>(), + }; const detach = (): void => { mount.observers.delete(observer); if (mount.observers.size === 0 && this.#mount === mount) this.#mount = undefined; @@ -161,19 +191,18 @@ export class TaskGraphView { .catch(() => {}); } - async #build(context: Context): Promise> { + /** Read the graph on the line, through the line's `storage`. */ + async #build(storage: Storage, context: Context): Promise> { const tasks: Record = {}; const live = new Map(); const records: AnyTaskRecord[] = []; for (const status of LIVE_STATUSES) { - records.push( - ...(await scanAll((cursor) => this.#storage.scanTasks({ status }, SCAN_PAGE_SIZE, cursor, context))), - ); + records.push(...(await scanAll((cursor) => storage.scanTasks({ status }, SCAN_PAGE_SIZE, cursor, context)))); } records.sort((a, b) => a.id - b.id); for (const record of records) { const owned = await scanAll((cursor) => - this.#storage.scanConversations({ ownerTaskId: record.id }, SCAN_PAGE_SIZE, cursor, context), + storage.scanConversations({ ownerTaskId: record.id }, SCAN_PAGE_SIZE, cursor, context), ); const conversations = owned.map((conversation) => conversation.id).sort((a, b) => a - b); const subject = blockedSubject(record); @@ -190,7 +219,7 @@ export class TaskGraphView { */ function advance( mount: Mount, - publication: CommitPublication, + publication: ObservedCommit, context: Context, blocked: TaskGraphBlocking["blocked"], report: (error: unknown) => void, @@ -239,8 +268,8 @@ function publish(mount: Mount, ops: Op[], context: Context, report: (error: unkn if (ops.length === 0) return; mount.value = applyImmutable(mount.value, ops); const frameContext = withoutAbortSignal(context); - for (const observer of [...mount.observers]) - contained(() => observer.advance(mount.value, ops, frameContext), report); + const call: WatchCall = { kind: "value", value: mount.value, ops, retired: false }; + for (const observer of [...mount.observers]) contained(() => observer.advance(call, frameContext), report); } function nodeOf( diff --git a/packages/durable/src/harness/telemetry.ts b/packages/durable/src/harness/telemetry.ts index ce8fa0bd6..baebaae0d 100644 --- a/packages/durable/src/harness/telemetry.ts +++ b/packages/durable/src/harness/telemetry.ts @@ -11,11 +11,11 @@ const reportAttributes = { type: { type: "string", required: true, - description: "ReportInfo type.", + description: "Report kind (`Report.kind`).", values: [ - "session_failed", + "storage_failed", "task_faulted", - "task_parked", + "task_interrupted", "migration_failed", "task_outdated", "task_error", @@ -28,7 +28,7 @@ const reportAttributes = { type: "string", required: false, description: "Fixed report reason, when present.", - values: ["threw", "restart_limit", "missing_task", "incompatible_task"], + values: ["threw", "restart_limit", "interrupted", "missing_task", "task_too_old", "migration_failed"], }, extension: { type: "string", required: false, description: "Registered extension name, when present." }, "task.id": { type: "number", required: false, description: "Task ID, when present.", cardinality: "high" }, @@ -52,13 +52,13 @@ export const DURABLE_TELEMETRY_SCHEMA = defineTelemetrySchema({ spans: { "pi.durable.submit": { description: - "Public submission admission and its commit. Inputs start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", + "Public submission admission: with its commit for submit(draft, context); the admission only for submit(tx, draft), whose result is provisional (a later throw or storage failure of the caller's commit leaves no submission) and whose commit events go to the caller's commit context. Inputs and continues start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", parents: { kind: "any" }, startAttributes: { "submission.type": { type: "string", required: true, - values: ["input", "write"], + values: ["input", "write", "continue"], description: "Submission type.", }, "conversation.id": { type: "number", required: true, description: "Conversation ID.", cardinality: "high" }, @@ -117,7 +117,7 @@ export const DURABLE_TELEMETRY_SCHEMA = defineTelemetrySchema({ metrics: { "pi.durable.reports": { kind: "counter", - description: "Reports raised, grouped by their fixed ReportInfo type.", + description: "Reports raised, grouped by their fixed kind.", attributes: { type: reportAttributes.type }, }, "pi.durable.phase.duration": { diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index 215e8f063..dc9e72ca2 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -11,7 +11,8 @@ import { } from "@earendil-works/chord/output"; import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai"; import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation"; -import { Compile } from "typebox/compile"; +import { compile, type Validator } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import { defineDocFamily } from "../documents.ts"; import { AssistantEntry, ToolResultEntry } from "../entries.ts"; import type { ExecutionEnv } from "../env/index.ts"; @@ -34,11 +35,15 @@ import { toolExtension } from "./agent.ts"; import { assignJson } from "./json.ts"; import { clearProgress, + cutOffCount, + cutOffTooOften, finishSlot, LiveDoc, type NestedToolSlot, type NestedToolSummary, + PROGRESS_COMMIT, removeNestedSlots, + salvageable, type ToolSlot, toolSlot, } from "./live.ts"; @@ -60,7 +65,13 @@ import { optionalExtension } from "./util.ts"; * whose call the input carries because no entry holds it. */ export type ToolTaskInput = - | { kind: "model"; assistant: EntryId; callId: string } + | { + kind: "model"; + assistant: EntryId; + callId: string; + /** A call to a tool its request did not offer: settles `tool_unavailable` without resolving the name. */ + unoffered?: true; + } | { kind: "nested"; parent: TaskId; @@ -124,25 +135,28 @@ type Content = (TextContent | ImageContent)[]; */ export const ToolTask = defineTask({ name: "pi.tool", - version: 2, + version: 3, initial: () => ({ phase: "call" }), - // Version 1 had only model-issued calls. - migrate: (input, checkpoint) => ({ - input: { kind: "model", ...(input as { assistant: EntryId; callId: string }) }, + // Version 1 had only model-issued calls, without `kind`; version 3 added `unoffered`, absent in version 2 inputs. + migrate: (input, checkpoint, fromVersion) => ({ + input: + fromVersion === 1 + ? { kind: "model", ...(input as { assistant: EntryId; callId: string }) } + : (input as ToolTaskInput), checkpoint: checkpoint as ToolTaskCheckpoint, }), phases: { call: async (task, runtime, context) => { const call = await readCall(runtime, task.input, context); - const tool = await resolveTool(runtime, task.input, call.name, context); + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); + const tool = isUnoffered(task.input) ? undefined : await resolveTool(runtime, task.input, call.name, context); if (tool === undefined) { const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`); - return settle(runtime, task.input, call, COMPLETED, () => error, context); + return reject(runtime, task.input, call, undefined, error, context); } const prepared = prepare(tool, call.arguments as JsonObject); const checked = "error" in prepared ? prepared : validate(tool, call, prepared.args); - if ("error" in checked) - return settle(runtime, task.input, call, COMPLETED, () => invalid(checked.error), context); + if ("error" in checked) return reject(runtime, task.input, call, tool, invalid(checked.error), context); let args = checked.args; let block: string | undefined; await runtime.hooks.each("beforeTool", async (hook) => { @@ -158,20 +172,27 @@ export const ToolTask = defineTask blocked, context); + return reject(runtime, task.input, call, tool, blocked, context); } const validated = validate(tool, call, args); - if ("error" in validated) - return settle(runtime, task.input, call, COMPLETED, () => invalid(validated.error), context); + if ("error" in validated) return reject(runtime, task.input, call, tool, invalid(validated.error), context); // Strict JSON: repair may set optional properties to undefined, which the slot and checkpoint cannot hold. const final = copyJson(validated.args, { omitUndefinedProperties: true }) as JsonObject; - await runtime.commit(async (tx) => { - const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot !== undefined) slot.status = "running"; - // A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed. - // Compared with the input, which the slot got at admission: reading the draft would track every leaf. - const changed = task.input.kind === "nested" && !jsonEqual(task.input.call.arguments as JsonValue, final); - if (changed && slot !== undefined && "parentTaskId" in slot) slot.arguments = final; + // The slot's status and arguments are presentation, stored like progress (`PROGRESS_COMMIT`); the intent is + // durable before the tool runs. Parallel calls of one round then share no immediate write to `pi.live`. + await runtime.commit( + async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot !== undefined) slot.status = "running"; + // A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed. + // Assigned whole: an equal value records no change, and a rerun corrects what an earlier attempt showed. + if (slot !== undefined && "parentTaskId" in slot) slot.arguments = final; + return undefined; + }, + context, + PROGRESS_COMMIT, + ); + await runtime.commit(() => { const intent = { phase: "execute", arguments: final, replay: tool.replay ?? "unsafe" } as const; return { status: "running", checkpoint: intent }; }, context); @@ -181,12 +202,19 @@ export const ToolTask = defineTask { const { arguments: args, replay } = task.state.checkpoint; const call = await readCall(runtime, task.input, context); + // Also before an interruption result: its `afterTool` hooks may be what cut the task off. + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); const tool = await resolveTool(runtime, task.input, call.name, context); if (replay === "safe" && tool?.replay === "safe") { - // The rerun reports from scratch; clear what the interrupted attempt published. + // The rerun reports from scratch: clear what the interrupted attempt published, and show the slot running with + // the arguments it runs with again, as the deferred commit that showed them may have been lost with the crash. await runtime.commit(async (tx) => { const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot !== undefined) clearProgress(slot); + if (slot !== undefined) { + clearProgress(slot); + slot.status = "running"; + if ("parentTaskId" in slot) slot.arguments = args; + } return undefined; }, context); return run(runtime, task.input, call, tool, args, context); @@ -194,23 +222,116 @@ export const ToolTask = defineTask fromSlot(slot, "interrupted", message), context); + // Not cut off too often (checked above), so the hooks run. + await settleFromSlot(runtime, task.input, call, { tool }, ending, "interrupted", message, context); }, }, abort: async (task, runtime, context) => { const call = await readCall(runtime, task.input, context); - // Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run. - const [code, message] = - task.abortReason !== "restart" + // Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run. An unoffered call + // could not run in any case, and its result says so. + const [code, message] = isUnoffered(task.input) + ? ["tool_unavailable", `Tool ${call.name} is not available`] + : task.abortReason !== "restart" ? ["aborted", `Tool ${call.name} was aborted`] : task.state.checkpoint.phase === "execute" ? ["interrupted", `Tool ${call.name} was interrupted by a restart and may have partially run`] : ["abandoned", `Tool ${call.name} was not started: its caller ended with a restart`]; - const ending = { status: "aborted" } as const; - await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, code, message), context); + // The scheduler runs an abort handler only while its restarts are below the limit, and faults the task without a + // result when the last attempt is cut off. That attempt skips the hooks, which may be what cuts it off. + const hooks = + (task.restarts ?? 0) < runtime.settings.restartLimit - 1 + ? { tool: await toolForLimits(runtime, task.input, call.name, context) } + : undefined; + await settleFromSlot(runtime, task.input, call, hooks, { status: "aborted" }, code, message, context); }, }); +/** + * End a call cut off `settings.restartLimit` times in a row (spec §5.4) with an error result, built from the slot like + * an interrupted call's, instead of running it again. `failed`, like an interruption, so its owned work is aborted. + */ +async function endCutOff( + runtime: Runtime, + task: { readonly restarts?: number; readonly input: ToolTaskInput }, + call: ToolHookCall, + context: Context, +): Promise { + const message = `Tool ${call.name} was interrupted ${cutOffCount(task)}; it is not run again`; + const ending = { status: "failed", message, reason: "restart_limit" } as const; + // Past the limit the hooks may be what keeps cutting the task off (`salvageable`). + const hooks = salvageable(task, runtime) + ? { tool: await toolForLimits(runtime, task.input, call.name, context) } + : undefined; + await settleFromSlot(runtime, task.input, call, hooks, ending, "restart_limit", message, context); +} + +/** + * Settle a result the Harness decided in the `call` phase, without running the tool: the tool did not resolve, the + * arguments are invalid, or `beforeTool` blocked the call. `afterTool` applies as to an executed call's result. + */ +async function reject( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + tool: ToolRegistration | undefined, + result: ToolExecutionResult, + context: Context, +): Promise { + const settled = await afterHarnessResult(runtime, input, call, tool, result, undefined, context); + await settle(runtime, input, call, COMPLETED, settled, context); +} + +/** + * Settle an error result built from the call's slot (`fromSlot`): an interruption, the restart limit, or an abort. + * The tool is not running, so the slot read before the commit is still current when it commits. With `hooks`, the + * result passes `afterTool` and is bounded with the limits of `hooks.tool` (the defaults when it is `undefined`); + * without, it is settled as built, when the hooks may be what cuts the task off. + */ +async function settleFromSlot( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + hooks: { readonly tool: ToolRegistration | undefined } | undefined, + ending: Ending, + code: string, + message: string, + context: Context, +): Promise { + const live = await runtime.snapshot(LiveDoc, runtime.conversationId, context); + const { result, truncation } = fromSlot( + live === undefined ? undefined : toolSlot(live, runtime.taskId), + code, + message, + ); + const settled = + hooks !== undefined + ? await afterHarnessResult(runtime, input, call, hooks.tool, result, truncation, context) + : truncation === undefined + ? result + : { ...result, diagnostics: [...(result.diagnostics ?? []), truncation] }; + await settle(runtime, input, call, ending, settled, context); +} + +/** + * The tool whose output limits bound a hooks' result: the called tool, or none when it, or the agent, fails to resolve, + * and for an unoffered call, which never resolves its name. + */ +async function toolForLimits( + runtime: Runtime, + input: ToolTaskInput, + name: string, + context: Context, +): Promise { + if (isUnoffered(input)) return undefined; + return resolveTool(runtime, input, name, context).catch(() => undefined); +} + +/** A model-issued call to a tool its request did not offer (spec §8.3). */ +function isUnoffered(input: ToolTaskInput): boolean { + return input.kind === "model" && input.unoffered === true; +} + /** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */ async function resolveTool( runtime: Runtime, @@ -222,6 +343,15 @@ async function resolveTool( return (input.kind === "model" ? agent.tools : agent.callable).find((tool) => tool.name === name); } +/** The tool's output limits, the defaults filled in; all defaults when the tool did not resolve. */ +function outputLimits(tool: ToolRegistration | undefined): OutputLimits { + return { + maxBytes: tool?.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES, + maxLines: tool?.outputLimits?.maxLines ?? DEFAULT_MAX_LINES, + retain: tool?.outputLimits?.retain ?? "head", + }; +} + /** The call: from the input for a nested call, which hooks see with its parent, else from the assistant entry. */ async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise { if (input.kind === "nested") { @@ -290,11 +420,7 @@ async function run( args: JsonObject, context: Context, ): Promise { - const limits: OutputLimits = { - maxBytes: tool.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES, - maxLines: tool.outputLimits?.maxLines ?? DEFAULT_MAX_LINES, - retain: tool.outputLimits?.retain ?? "head", - }; + const limits = outputLimits(tool); const reported: Reported = { output: new OutputBuffer(limits), limits, diagnostics: [], details: undefined }; const streams = input.kind === "model" || input.progress !== false; const progress = publishProgress(runtime, reported, streams, context); @@ -358,12 +484,16 @@ async function run( committed.catch(() => {}); return awaitWithContext(committed, detailsContext); }, - commit: async (change, commitContext) => { + commit: async (change, commitContext, options) => { let result: Awaited> | undefined; - await runtime.commit(async (tx) => { - result = await change(tx); - return undefined; - }, commitContext); + await runtime.commit( + async (tx) => { + result = await change(tx); + return undefined; + }, + commitContext, + options, + ); return result as Awaited>; }, memo: runtime.memo, @@ -449,7 +579,7 @@ async function run( const pending = await progress.stop(); try { const settled = await finalResult(runtime, input, call, tool, result, reported, context); - await settle(runtime, input, call, ending, () => settled, context, durationMs); + await settle(runtime, input, call, ending, settled, context, durationMs); } catch (error) { for (const waiter of pending) waiter.reject(error); throw error; @@ -478,7 +608,7 @@ function progressClock(runtime: Runtime): ProgressClock { /** * Throttled commits of what the tool reported into its slot, in `pi.live.tools` or `pi.live.nestedTools`, each writing - * only what changed since the last one; none when `enabled` is false. + * only what changed since the last one, deferred (`PROGRESS_COMMIT`); none when `enabled` is false. */ function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, context: Context): Progress { let written = { text: "", details: undefined as JsonValue | undefined, diagnostics: 0 }; @@ -501,24 +631,28 @@ function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, } if (detailsChanged) bytes += utf8ByteLength(JSON.stringify(current.details ?? null)); if (added.length > 0) bytes += utf8ByteLength(JSON.stringify(added)); - await runtime.commit(async (tx) => { - const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot === undefined) return undefined; - // REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an - // append for a sliding tail; replacing the slot object would record the whole window on every commit. - if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text; - if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes; - if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines; - // Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed. - if (detailsChanged && current.details !== undefined) { - assignJson(slot as unknown as Record, "details", current.details); - } - if (added.length > 0) { - if (slot.diagnostics === undefined) slot.diagnostics = []; - for (const diagnostic of added) slot.diagnostics.push(diagnostic); - } - return undefined; - }, context); + await runtime.commit( + async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot === undefined) return undefined; + // REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an + // append for a sliding tail; replacing the slot object would record the whole window on every commit. + if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text; + if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes; + if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines; + // Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed. + if (detailsChanged && current.details !== undefined) { + assignJson(slot, "details", current.details); + } + if (added.length > 0) { + if (slot.diagnostics === undefined) slot.diagnostics = []; + for (const diagnostic of added) slot.diagnostics.push(diagnostic); + } + return undefined; + }, + context, + PROGRESS_COMMIT, + ); written = current; return bytes; }, @@ -579,7 +713,7 @@ async function finalResult( (agent) => toolExtension(agent, tool.name), () => undefined, ); - runtime.report(new Error(broken), { type: "extension", ...optionalExtension(extension) }); + runtime.report(new Error(broken), { kind: "extension", ...optionalExtension(extension) }); } } // Programs get the tool's own value: without a schema, its output. The model reads only `output`, so model-issued @@ -602,7 +736,42 @@ async function finalResult( return { ...final, output: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] }; } -const structuredValidators = new WeakMap>(); +/** + * A result the Harness wrote itself, without executing the tool, after the `afterTool` chain, normalized like an + * executed call's: the output bounded with the tool's limits (the defaults when it did not resolve), a truncation + * diagnostic when that dropped something, and the hooks' `structuredOutput` dropped, since the tool never produced + * one. A nested call that the hooks turned into a success gets the bounded output as its value, as a schema-less + * tool's success does, unless the tool declares a schema. `usage` is kept and recorded; `control` applies only when + * the task completes. `truncation` describes what the result's own output lost before (a slot's retained output); it + * is kept only while the hooks keep that output. + */ +async function afterHarnessResult( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + tool: ToolRegistration | undefined, + result: ToolExecutionResult, + truncation: ToolDiagnostic | undefined, + context: Context, +): Promise { + let final = result; + await runtime.hooks.each("afterTool", async (hook) => { + final = (await hook(call, final, runtime, context)) ?? final; + }); + const harness: ToolDiagnostic[] = []; + if (truncation !== undefined && final.output === result.output) harness.push(truncation); + const limits = outputLimits(tool); + const bounded = boundContent(final.output ?? [], limits); + if (bounded.droppedBytes > 0) harness.push(truncated(bounded, limits.retain)); + const { structuredOutput: _dropped, ...rest } = final; + const value = + input.kind === "nested" && final.isError !== true && tool?.structuredOutputSchema === undefined + ? { structuredOutput: outputValue(bounded.content) as JsonValue } + : {}; + return { ...rest, ...value, output: bounded.content, diagnostics: [...(rest.diagnostics ?? []), ...harness] }; +} + +const structuredValidators = new WeakMap(); /** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResult): string | undefined { @@ -618,12 +787,17 @@ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResu } let validator = structuredValidators.get(schema); if (validator === undefined) { - validator = Compile(schema); + validator = compile(schema, { formats, unresolvedRefs: "reject" }); structuredValidators.set(schema, validator); } - if (validator.Check(value)) return undefined; - const [first] = validator.Errors(value); - const path = first?.instancePath.replace(/^\//, "").replace(/\//g, ".") || "root"; + if (validator.check(value)) return undefined; + const [first] = validator.errors(value); + const path = + first?.path + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join(".") || "root"; return `Tool ${tool.name} returned structuredOutput that does not match its schema: ${path}: ${first?.message ?? "invalid"}`; } @@ -631,15 +805,14 @@ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResu * Commit the tool's terminal state and mark its slot done. A model-issued call appends its result entry and ends with * the entry ID; a nested call writes its result to the caller's `NestedCallDoc`, records its usage, and ends with a * small receipt. Nested calls this call left running are aborted first, so they report into their slots before the - * slots below the call leave `pi.live`. `build` receives the slot so interruption and abort can report the durable - * partial output. + * slots below the call leave `pi.live`. `control` applies only when the task completes. */ async function settle( runtime: Runtime, input: ToolTaskInput, call: ToolHookCall, ending: Ending, - build: (slot: Readonly | undefined) => ToolExecutionResult, + result: ToolExecutionResult, context: Context, durationMs?: number, ): Promise { @@ -656,7 +829,6 @@ async function settle( const conversationId = runtime.conversationId; const live = await tx.doc(LiveDoc, conversationId); const slot = toolSlot(live, runtime.taskId); - const result = build(slot); let settled: ToolTaskResult; if (input.kind === "nested") { const nested = nestedResult(runtime.taskId, result, durationMs); @@ -690,7 +862,10 @@ async function settle( if (madeNested) removeNestedSlots(live, runtime.taskId); if (ending.status === "aborted") return { status: "terminal", outcome: { status: "aborted", result: settled } }; if (ending.status === "failed") { - const error = { message: ending.message }; + const error = { + message: ending.message, + ...(ending.reason === undefined ? {} : { detail: { reason: ending.reason } }), + }; return { status: "terminal", outcome: { status: "failed", error, result: settled } }; } return { status: "terminal", outcome: { status: "completed", result: settled } }; @@ -817,26 +992,30 @@ function fallbackResult(taskId: TaskId, name: string, outcome: TaskOutcome | undefined, code: string, message: string, -): ToolExecutionResult { - const diagnostics = [...(slot?.diagnostics ?? [])]; +): { readonly result: ToolExecutionResult; readonly truncation?: ToolDiagnostic } { const droppedBytes = slot?.droppedBytes ?? 0; - if (droppedBytes > 0) diagnostics.push(truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 })); - diagnostics.push(toolDiagnostic(code, message)); - return { + const result: ToolExecutionResult = { output: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }], isError: true, ...(slot?.details === undefined ? {} : { details: slot.details }), - diagnostics, + diagnostics: [...(slot?.diagnostics ?? []), toolDiagnostic(code, message)], }; + if (droppedBytes === 0) return { result }; + return { result, truncation: truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 }) }; } /** An error result the Harness writes itself: no content and one `error` diagnostic with `code`. */ diff --git a/packages/durable/src/harness/transcript.ts b/packages/durable/src/harness/transcript.ts new file mode 100644 index 000000000..d2463b0d9 --- /dev/null +++ b/packages/durable/src/harness/transcript.ts @@ -0,0 +1,285 @@ +import type { Context } from "@earendil-works/chord"; +import { idFromNumber } from "../ids.ts"; +import type { Cursor, EntryId, EntryQuery, EntryRecord, Page } from "../types.ts"; +import type { ConversationView, ConversationWindow } from "./view.ts"; + +/** One row of a client's transcript. */ +export type TranscriptItem = + | { readonly kind: "entry"; readonly entry: EntryRecord } + /** Entries strictly between `after` and `upTo` are missing: render "load more" and call `fillGap()`. */ + | { readonly kind: "gap"; readonly after: EntryId; readonly upTo: EntryId } + /** The transcript continues above: render "load earlier" and call `loadEarlier()`. Always the first item. */ + | { readonly kind: "earlier" }; + +/** The conversation view as a renderer sees it: `items`, the client's whole transcript, instead of the window. */ +export type TranscriptView = Omit & { + readonly items: readonly TranscriptItem[]; +}; + +/** Pages a conversation's entries, as `Conversation.entries()` does without a cursor. */ +export type EntryLoader = ( + query: Omit, + limit: number, + context: Context, +) => Promise>; + +export type ConversationTranscriptOptions = { + /** Entries `loadEarlier()` and `fillGap()` load per call; default 50. */ + readonly pageSize?: number; +}; + +type Gap = Extract; +type EntryItem = Extract; + +const EARLIER: TranscriptItem = Object.freeze({ kind: "earlier" }); + +/** + * What a client has of one conversation: every entry it got from a view or fetched, and the gaps between (spec §9.3). + * A layer over any `ConversationView`, wherever it comes from: call `merge()` with every value of the view; it returns + * the view with the client's whole transcript as `items` instead of the window. Entries never change once committed, + * so the transcript only grows; a window that dropped its oldest entries keeps them here. + * + * A window whose `before` lies past the newest entry the client has does not continue it, after a stall, an overflow, + * or a reconnect: the entries in between become a gap. Nothing counts entries or computes ranges from entry IDs, which + * have holes; only exclusive bounds are turned into the inclusive bounds of `EntryQuery`. After a drop, a new watch + * with `resume` as its window continues the transcript. + * + * A merge costs O(items) and never re-sorts or re-fetches; a merge, `loadEarlier()`, or `fillGap()` that adds nothing + * returns the previous `items` array. Concurrent calls of `loadEarlier()`, or of `fillGap()` for the same gap, share one + * load, so a placeholder that stays visible does not fetch twice. + */ +export class ConversationTranscript { + readonly #load: EntryLoader; + readonly #pageSize: number; + /** Every entry held, in ID order, as items. */ + #entries: EntryItem[] = []; + readonly #ids = new Set(); + /** Gaps in ID order; each is an immutable item, replaced when it shrinks. */ + #gaps: Gap[] = []; + /** Whether anything was merged yet. */ + #started = false; + /** Exclusive lower bound of the known range: entries at or below it are not loaded; `-Infinity` at the start. */ + #low = Number.NEGATIVE_INFINITY; + /** Inclusive upper bound of the known range: the newest entry, or a later empty window's `before`. */ + #tail = Number.NEGATIVE_INFINITY; + #items: readonly TranscriptItem[] = []; + #last: ConversationView | undefined; + #view: TranscriptView | undefined; + #earlier: Promise | undefined; + readonly #filling = new Map>(); + + constructor(load: EntryLoader, options?: ConversationTranscriptOptions) { + const pageSize = options?.pageSize ?? 50; + if (!Number.isSafeInteger(pageSize) || pageSize < 1) { + throw new RangeError(`Page size must be an integer >= 1, got ${pageSize}`); + } + this.#load = load; + this.#pageSize = pageSize; + } + + /** + * The ID the transcript reaches: its newest entry, or the `before` of a later window that was empty. Undefined before + * the first merge and for an empty transcript. + */ + get newest(): EntryId | undefined { + return this.#tail === Number.NEGATIVE_INFINITY ? undefined : idFromNumber(this.#tail); + } + + /** + * The window that resumes this transcript after a drop: `{ from }` right after `newest`, so a new watch returns only + * what was missed when it fits its window, otherwise its newest entries and a gap; `{}` for an empty transcript; + * undefined before the first merge. `conversation.watch(context, transcript.resume)`. + */ + get resume(): ConversationWindow | undefined { + if (!this.#started) return undefined; + const newest = this.newest; + return newest === undefined ? {} : { from: idFromNumber(newest + 1) }; + } + + /** Take one value of the view: its window's entries join the transcript. */ + merge(view: ConversationView): TranscriptView { + this.#last = view; + const { entries, before } = view; + const hi = entries.at(-1)?.id ?? before ?? Number.NEGATIVE_INFINITY; + this.#cover(before ?? Number.NEGATIVE_INFINITY, hi, entries); + return this.#result(); + } + + /** Load the page before the oldest entry held. Without anything earlier, returns the current view. */ + loadEarlier(context: Context): Promise { + const last = this.#last; + if (last === undefined) return Promise.reject(new Error("ConversationTranscript: merge a view first")); + if (this.#low === Number.NEGATIVE_INFINITY) return Promise.resolve(this.#result()); + if (this.#earlier !== undefined) return this.#earlier; + const low = this.#low; + // The loader runs in a later microtask, so even its synchronous throw settles a promise that is already shared. + const loading = Promise.resolve().then(async () => { + const query = { maxEntryId: idFromNumber(low), order: "descending" } as const; + const page = await this.#load(query, this.#pageSize, context); + const items = [...page.items].reverse(); + const oldest = items[0]; + // No next page, or an empty one: the transcript starts here. + if (page.next === undefined || oldest === undefined) this.#cover(Number.NEGATIVE_INFINITY, low, items); + else this.#cover(oldest.id - 1, low, items); + return this.#result(); + }); + this.#earlier = loading; + const settled = (): void => { + if (this.#earlier === loading) this.#earlier = undefined; + }; + loading.then(settled, settled); + return loading; + } + + /** + * Load the next page of `gap`, a gap item of `items`, shrinking it from its start, or removing it once nothing is + * missing. A gap that has shrunk since loads the gap it became; one that is no longer there returns the current view. + */ + fillGap(gap: Gap, context: Context): Promise { + const current = + this.#gaps.find((each) => each.after === gap.after && each.upTo === gap.upTo) ?? + this.#gaps.find((each) => each.after < gap.upTo && gap.after < each.upTo); + if (current === undefined) { + const last = this.#last; + if (last === undefined) return Promise.reject(new Error("ConversationTranscript: merge a view first")); + return Promise.resolve(this.#result()); + } + // Keyed by the gap the call resolved to, so a stale descriptor and the current one share the load. + const key = `${current.after}:${current.upTo}`; + const running = this.#filling.get(key); + if (running !== undefined) return running; + const filling = Promise.resolve().then(async () => { + const query = { + minEntryId: idFromNumber(current.after + 1), + maxEntryId: idFromNumber(current.upTo - 1), + order: "ascending", + } as const; + const page = await this.#load(query, this.#pageSize, context); + const newest = page.items.at(-1); + // No next page, or an empty one: nothing else is missing. + if (page.next === undefined || newest === undefined) this.#cover(current.after, current.upTo - 1, page.items); + else this.#cover(current.after, newest.id, page.items); + return this.#result(); + }); + this.#filling.set(key, filling); + const settled = (): void => { + if (this.#filling.get(key) === filling) this.#filling.delete(key); + }; + filling.then(settled, settled); + return filling; + } + + /** + * Record that every entry with an ID in (`lo`, `hi`] is known, `added` being those entries in ID order: hold the new + * ones, shrink the gaps they fill, extend the known range, and open a gap where the range does not continue it. + */ + #cover(lo: number, hi: number, added: readonly EntryRecord[]): void { + const fresh = added.filter((entry) => !this.#ids.has(entry.id)); + for (const entry of fresh) this.#ids.add(entry.id); + const held = this.#entries; + // New entries past every held one: the items grow at their end. + const appended = fresh.length === 0 || held.length === 0 || fresh[0]!.id > held.at(-1)!.entry.id; + const items = fresh.map((entry): EntryItem => ({ kind: "entry", entry })); + if (items.length > 0) this.#entries = appended ? held.concat(items) : mergeById(held, items); + let rebuild = false; + if (!this.#started || (held.length === 0 && this.#gaps.length === 0 && lo >= this.#tail)) { + // The first range, or one after an empty transcript: it is all the client knows. + rebuild = !this.#started || lo !== this.#low; + this.#started = true; + this.#low = lo; + this.#tail = Math.max(this.#tail, hi); + } else if (lo > this.#tail) { + // Past the newest known ID: what lies between is missing. + const upTo = added[0]?.id ?? lo + 1; + this.#gaps = [...this.#gaps, gapOf(this.#tail, upTo)]; + this.#tail = hi; + rebuild = true; + } else if (hi !== Number.NEGATIVE_INFINITY && hi < this.#low) { + // Below the oldest known ID: what lies between is missing. + const upTo = held.find((item) => item.entry.id > hi)?.entry.id ?? this.#low + 1; + this.#gaps = [gapOf(hi, upTo), ...this.#gaps]; + this.#low = lo; + rebuild = true; + } else { + const gaps = this.#subtract(lo, hi); + if (gaps !== this.#gaps) { + this.#gaps = gaps; + rebuild = true; + } + if (lo < this.#low) { + this.#low = lo; + rebuild = true; + } + if (hi > this.#tail) this.#tail = hi; + } + if (rebuild || !appended) this.#items = this.#build(); + else if (items.length > 0) this.#items = this.#items.concat(items); + } + + /** The gaps without (`lo`, `hi`]; the same array when none changes, as when the range is empty. */ + #subtract(lo: number, hi: number): Gap[] { + if (hi <= lo) return this.#gaps; + let changed = false; + const gaps: Gap[] = []; + for (const gap of this.#gaps) { + // Missing: IDs in (after, upTo); known now: IDs in (lo, hi]. + if (hi <= gap.after || lo + 1 >= gap.upTo) { + gaps.push(gap); + continue; + } + changed = true; + if (lo > gap.after) { + const firstKnown = this.#entries.find((item) => item.entry.id > lo)?.entry.id ?? lo + 1; + gaps.push(gapOf(gap.after, Math.min(gap.upTo, firstKnown))); + } + if (hi + 1 < gap.upTo) gaps.push(gapOf(hi, gap.upTo)); + } + return changed ? gaps : this.#gaps; + } + + /** Items in ID order: "earlier" until the start is known, then entries with each gap before the entry after it. */ + #build(): TranscriptItem[] { + const items: TranscriptItem[] = this.#low === Number.NEGATIVE_INFINITY ? [] : [EARLIER]; + const gaps = this.#gaps; + let next = 0; + for (const item of this.#entries) { + while (next < gaps.length && gaps[next]!.after < item.entry.id) items.push(gaps[next++]!); + items.push(item); + } + while (next < gaps.length) items.push(gaps[next++]!); + return items; + } + + /** The view with the current items; the previous one when nothing changed. */ + #result(): TranscriptView { + const last = this.#last!; + const previous = this.#view; + if ( + previous !== undefined && + previous.items === this.#items && + previous.docs === last.docs && + previous.conversation === last.conversation + ) { + return previous; + } + this.#view = { conversation: last.conversation, docs: last.docs, items: this.#items }; + return this.#view; + } +} + +function gapOf(after: number, upTo: number): Gap { + return Object.freeze({ kind: "gap", after: idFromNumber(after), upTo: idFromNumber(upTo) }); +} + +/** Two ID-ordered item lists merged in one pass. */ +function mergeById(left: readonly EntryItem[], right: readonly EntryItem[]): EntryItem[] { + const merged: EntryItem[] = []; + let i = 0; + let j = 0; + while (i < left.length && j < right.length) { + merged.push(left[i]!.entry.id < right[j]!.entry.id ? left[i++]! : right[j++]!); + } + while (i < left.length) merged.push(left[i++]!); + while (j < right.length) merged.push(right[j++]!); + return merged; +} diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 66d8cb75f..e312758b9 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -1,36 +1,41 @@ import type { Context, JsonValue, WatchHandle } from "@earendil-works/chord"; import type { + Api, AssistantMessage, CacheRetention, ClientIdentity, Message, + Model, Models, ModelThinkingLevel, - Static, Tool, ToolCall, ToolResultMessage, Transport, - TSchema, Usage, UserMessage, } from "@earendil-works/pi-ai"; +import type { Static } from "@earendil-works/pi-schema"; import type { TelemetryContext } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv, ShellOutputSkip, ShellOutputWindow } from "../env/index.ts"; import type { + CommitOptions, ConversationId, ConversationOwnership, ConversationRecord, Cursor, DocumentObserver, DocumentReader, + Entry, EntryDraft, EntryId, EntryQuery, EntryRecord, JsonObject, + Observation, + Observer, Page, - ReportInfo, + Report, Session, SetTimer, SubmissionId, @@ -41,10 +46,11 @@ import type { TaskRecord, TaskState, Tx, + TypedEntry, } from "../types.ts"; -import type { TaskGraphWatch } from "./task-graph.ts"; +import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts"; import type { UsageState } from "./usage.ts"; -import type { ConversationView } from "./view.ts"; +import type { ConversationView, ConversationWindow } from "./view.ts"; /** Provider and model ID resolved through pi-ai `Models`. */ export type ModelRef = { @@ -60,7 +66,11 @@ export type UserInput = UserMessage["content"]; */ export type InputEntry = { readonly kind: string; readonly data?: JsonValue }; -/** Host submission: user input that may start a run, or a passive entry write. */ +/** + * Host submission: user input that may start a run, a passive entry write, or a continue: a run without a new user + * message, for an idle conversation whose context the model still owes an answer (it ends in a user message or tool + * results, for example after a run that ended without an answer). + */ export type SubmissionDraft = { readonly requestId?: string; } & ( @@ -68,6 +78,12 @@ export type SubmissionDraft = { readonly type: "input"; readonly content: UserInput; readonly whenBusy?: "steer" | "followUp" | "reject"; + /** + * `reject`: an idle conversation (no run) rejects the input with `ConversationIdle` and writes nothing, so the input + * never starts a run of its own. With `whenBusy: "steer"` it steers the current run or is refused; a run that + * answered and waits for its `afterRun` tasks is still busy, and such an input queues and starts the next run. + */ + readonly whenIdle?: "reject"; /** Place the input as this entry; default `pi.user` without data. The kind must not start with `pi.`. */ readonly entry?: InputEntry; } @@ -76,6 +92,21 @@ export type SubmissionDraft = { readonly entry: EntryDraft; readonly content?: never; readonly whenBusy?: never; + readonly whenIdle?: never; + } + | { + /** + * Start a run on the context as it is, without a user message. Only when idle: a busy conversation rejects it with + * `ConversationBusy`, and it never queues. Items kept in `pi.inbox` are placed first and join the run. A context + * with nothing to answer (empty, or ending in an answer, also a text answer cut off at its output limit) ends the + * run without a request, `unanswered` with `nothing_to_answer`. A cut-off answer with tool calls is continued: + * its calls never ran, and the context ends in their error results (`missing_result`). + */ + readonly type: "continue"; + readonly content?: never; + readonly entry?: never; + readonly whenBusy?: never; + readonly whenIdle?: never; } ); @@ -124,12 +155,16 @@ export type ConversationAbortOptions = { export type HooksOf = K extends Task ? H : never; /** - * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends; passive entries are - * written with ordinary transaction writes instead. + * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends. `submit` takes any + * submission, in both forms of `Conversation.submit()`: a subagent's conversation is an ordinary one, so a tool may also + * continue it, and a task may admit a submission in the same commit as its own writes (`runtime.commit()`); entries + * the admission appends there carry the task's `byTaskId`. The transaction form also rejects once the invocation is + * aborted. */ export interface ConversationHandle { readonly id: ConversationId; - submit(submission: InputSubmissionDraft, context: Context): Promise; + submit(submission: SubmissionDraft, context: Context): Promise; + submit(tx: Tx, submission: SubmissionDraft): Promise; /** `Conversation.abort()`: withdraw queued inputs, abort the ordinary ownership scope, and wait until it is idle. */ abort(context: Context, options?: ConversationAbortOptions): Promise; /** Resolve when the conversation's ordinary ownership scope has no live non-background task. */ @@ -141,6 +176,10 @@ export type ToolControl = { readonly addTools?: readonly string[]; /** End the run without another model request when every result of the round sets it to `true`; `false` is a no-op. */ readonly terminate?: boolean; + /** + * Start a new context with this text as its user message, as `reset(handoff)` writes, and continue the run there. + * The last handoff in call order wins. + */ readonly handoff?: string; }; @@ -240,7 +279,8 @@ export interface ToolExecutionApi extend diagnostic(diagnostic: ToolDiagnostic): void; /** Replace running details; the last value becomes the result details when the result omits `details`. */ details(value: TDetails, context: Context): Promise; - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** A commit of this call's task; `options` may defer storing it (`CommitPersistence`). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; /** @@ -292,12 +332,22 @@ export interface ToolExecutionApi extend ): Promise; } +/** + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `ToolRegistration`. + */ +export type ToolArgs = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + /** * Executable tool registered in a registry. Only pi-ai `Tool` fields enter the transcript. `args` are typed by - * `parameters`, which the Harness validates them against before `execute()`; `defineTool()` infers both generics. + * `parameters` (any JSON Schema object; see {@link ToolArgs}), which the Harness validates them against before + * `execute()`; `defineTool()` infers both generics. */ export type ToolRegistration< - TParameters extends TSchema = TSchema, + TParameters extends object = object, TDetails extends JsonValue = JsonValue, > = Tool & { /** Whether an interrupted execution may rerun on recovery. Default `unsafe`. */ @@ -309,12 +359,12 @@ export type ToolRegistration< * pure and must not mutate `args`: it runs again when a call is retried before its intent is recorded. Its result is * still validated against `parameters`. */ - prepareArguments?(args: unknown): Static; + prepareArguments?(args: unknown): ToolArgs; /** * Schema of `structuredOutput`, which programs that call the tool receive. Omitted: they receive the result's bounded * output, success or error, as `NestedToolExecutionResult.structuredOutput` describes. */ - readonly structuredOutputSchema?: TSchema; + readonly structuredOutputSchema?: object; /** * Who may call the tool: the model, other tools through `executeTool()`, or both (default). A code mode tool is * `["model"]`, so scripts cannot start scripts; tools behind code mode, such as MCP tools, can be `["tools"]`, so the @@ -332,7 +382,7 @@ export type ToolRegistration< readonly retain?: "head" | "tail"; }; execute( - args: Static, + args: ToolArgs, api: ToolExecutionApi, context: Context, ): Promise>; @@ -429,6 +479,11 @@ export type AgentState = { instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ cwd?: string; + /** + * Tool rounds one run takes at most (`pi.live.run.turns`): the round that reaches it runs its tools, then the run ends + * `unanswered` with `turn_limit`. An integer, `1` or more; unset, no limit. Task-owned children and forks inherit it. + */ + maxTurns?: number; }; /** A change to `pi.agent`: a given field replaces the stored one, `null` clears it, `undefined` changes nothing. */ @@ -447,6 +502,8 @@ export type AgentChange = { | null; readonly instructions?: string | null; readonly cwd?: string | null; + /** `configure()` throws a `RangeError` unless it is an integer `1` or more (or `null`). */ + readonly maxTurns?: number | null; }; /** A conversation's agent resolved against a registry snapshot and the settings. */ @@ -464,6 +521,11 @@ export type Agent = { readonly sections: readonly PromptSection[]; readonly instructions?: string; readonly cwd?: string; + /** + * Tool rounds one run takes at most (`pi.live.run.turns`): the round that reaches it runs its tools, then the run ends + * `unanswered` with `turn_limit`. An integer, `1` or more; unset, no limit. Task-owned children and forks inherit it. + */ + readonly maxTurns?: number; }; /** @@ -521,14 +583,21 @@ export type CompactionSettings = Partial & { }; /** - * How often running progress is committed. Each progress commit is a storage write; a host whose storage is remote can - * commit less often, so live answers and tool output appear in larger steps. + * How often running progress is committed, and how long it may wait in memory before it is stored. Progress commits are + * deferred (`CommitPersistence`): watches see each at once, and Storage gets them together, at the run's next phase + * boundary or after `persistWithinMs`. A host can commit less often, so live answers and tool output appear in larger steps. */ export type ProgressPolicy = { /** Minimum pause between commits of the answer being generated. */ partialIntervalMs: number; /** Minimum pause between commits of running tool output; large commits also pause in proportion to their size. */ outputIntervalMs: number; + /** + * How long these commits, and other deferred commits without their own `withinMs`, may stay in memory before they are + * stored (`CommitPersistence`); a crash loses at most about this much of an answer or of tool output. Default 1,000; + * `Infinity` stores them only at the next commit that reads or writes their document and at close. + */ + persistWithinMs: number; }; /** Why a compaction runs: `compact()`, a threshold in generation preparation, or a context overflow. */ @@ -563,20 +632,27 @@ export type HarnessSettings = { */ readonly documentRetentionMs?: number; /** - * What a run that fails (`model_error`, `no_model`) does with the inputs queued behind it: `keep` leaves them - * queued until the next input or queued write, `continue` starts the next run with them, as an answer would, and - * `withdraw` settles them `unanswered` with reason `run_failed`. Default `keep`. + * What a run that fails (`model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, + * `turn_limit`) does with the inputs queued behind it: `keep` leaves them queued until the next input or queued write, `continue` starts the + * next run with them, as an answer would, and `withdraw` settles them `unanswered` with reason `run_failed`. Default + * `keep`. */ readonly afterFailedRun?: AfterFailedRun; /** * How many times in a row a task may be cut off by a crash, eviction, or reset (a close does not count) without - * changing its state before it is parked as blocked (`restart_limit`) instead of run again; `Harness.retryTask()` - * runs it again. Default 5; `Infinity` never parks. + * changing its state before it ends instead of running again: a tool call with an error result, a compaction failed + * into `pi.live.compactionFailure`, a run with `pi.live.runFailure` reason `restart_limit`, any other task `faulted` + * with detail `{ reason: "restart_limit" }`. A generation's request or poll the host died during is a failed attempt + * under `retry` first; the limit bounds it only when charging that attempt keeps being cut off. Default 5; + * `Infinity` never ends a task for it. */ readonly restartLimit?: number; }; -/** What a failed run does with the inputs queued behind it (`HarnessSettings.afterFailedRun`). */ +/** + * What a failed run does with the inputs queued behind it (`HarnessSettings.afterFailedRun`): a run that ends with + * `model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, or `turn_limit`. + */ export type AfterFailedRun = "keep" | "continue" | "withdraw"; /** Resolved settings: every field over its built-in default, object fields merged. */ @@ -648,18 +724,19 @@ export type HarnessOptions = { * Wakes the process for sleeping tasks instead of a `setTimer` timer. Without it, while scheduling runs, a timer that * keeps the process alive resumes sleeping tasks when they are due. With it, the Harness arms no timer for them: it * calls `onWake` after open and whenever `Harness.wake()` changes, and the host arms its own wakeup, such as a - * Durable Object alarm, and calls `resume()` when it fires. Called off the Session line, after the commit that - * changed the value, so a host arms a recovery wakeup before calls that can start work. It may call `wake()` and - * `resume()`, and must not block. A throw is reported, and the value is offered again at the next change. + * Durable Object alarm, and calls `resume()` when it fires. Called in a microtask the Session line never waits for, + * after the commit that changed the value was adopted, so a host arms a recovery wakeup before calls that can start + * work. It may call `wake()` and `resume()`, and must not block. A throw is reported, and the value is offered again + * at the next change. */ readonly onWake?: (wake: Wake) => void; /** * Receives failures that do not fail the calling operation, such as a throwing hook or listener, a task that faulted, - * and the error that fails the Session, once. `info` says what happened: a host can tell a failed Session - * (`session_failed`, often a platform reset) from a faulted task (`task_faulted`) or a parked one (`task_parked`). - * Its own throw or rejection is dropped. + * and the error that fails the Session, once: one `Report`, the `error` and what happened (`kind` and the fields of + * its `ReportInfo`). A host can tell a failed Session (`storage_failed`, often a platform reset) from a faulted task + * (`task_faulted`). Its own throw or rejection is dropped. */ - readonly onReport?: (error: unknown, info: ReportInfo) => void; + readonly onReport?: (report: Report) => void; /** * Open for reading only, as a viewer of a session that another process drives, for example a synced copy on a device * that only shows it. Open recovers nothing: a `running` task stays as committed. Every commit with writes rejects @@ -685,7 +762,7 @@ export type TaskInspection = { readonly state: /** An invocation is active. */ | { readonly kind: "running" } /** The next scheduling pass reserves it; `migrates` when its definition is newer and has `migrate`. */ - | { readonly kind: "ready"; readonly migrates: boolean } + | { readonly kind: "pending"; readonly migrates: boolean } /** * Waits for these live tasks: the live part of its `on`, or, when abort-marked, its live ordinary owned work, which * must end before its abort handler starts. @@ -695,13 +772,10 @@ export type TaskInspection = { | { readonly kind: "sleeping"; readonly until: number } /** Outcome held until its ordinary owned work drains. */ | { readonly kind: "completing" } - /** - * Cannot run here: no registered definition can take it (aborting it settles it as `orphaned`), or it is parked by - * the restart limit (`retryTask()` runs it again). - */ + /** Cannot run here: no registered definition can take it (aborting it settles it as `orphaned`). */ | { readonly kind: "blocked"; - readonly reason: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; readonly error?: unknown; }; }; @@ -736,12 +810,25 @@ export interface Conversation { configure(change: AgentChange, context: Context): Promise; /** - * Durably admit user input or a passive entry write. A busy conversation, or one with queued items, queues input and - * writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with `ConversationBusy` instead - * and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) is appended at once, - * even mid-run, as it changes nothing the model sees. + * Durably admit user input, a passive entry write, or a continue. A busy conversation, or one with queued items, + * queues input and writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with + * `ConversationBusy` instead and writes nothing. An idle conversation rejects `whenIdle: "reject"` input with + * `ConversationIdle` and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) + * is appended at once, even mid-run, as it changes nothing the model sees. A continue starts a run on the context as + * it is, after placing queued items; a busy conversation rejects it with `ConversationBusy`, as it never queues. */ submit(submission: SubmissionDraft, context: Context): Promise; + /** + * Admit a submission inside the caller's commit `tx` (`Harness.commit()`, `Conversation.commit()`, a task's + * `runtime.commit()`), with the same rules, and return its ID: the admission is stored with that commit, or not at all + * when the callback throws later or the commit fails. Call it before the commit's first table write (`appendEntry()`, + * `createTask()`, `createSubmission()`, another admission): otherwise it throws `ReadAfterWrite`, on every path. + * Documents may be read and written before and after it. A rejection (`ConversationBusy`, `ConversationIdle`) throws + * before any write, so the callback may catch it. Do not call `Submission` or other Harness reads inside the callback: + * they wait for the commit holding the line. Its `pi.durable.submit` span covers the admission only, so its `result` + * is provisional. Throws for a transaction of another Session. + */ + submit(tx: Tx, submission: SubmissionDraft): Promise; /** * Admit a write of a `pi.reset` entry that starts a new context, carrying `handoff` as a user message when given. * Resolves after admission; while busy, it is placed at the next boundary. @@ -753,8 +840,8 @@ export interface Conversation { */ compact(instructions: string | undefined, context: Context): Promise>; - /** Session commit whose `tx.createTask()` defaults to this conversation. */ - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** Session commit whose `tx.createTask()` defaults to this conversation; `options` as for `Session.commit()`. */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; /** * Committed raw active transcript and model context. With `at`, the context as of that visible entry: the same view * `fork(at)` would start with, without creating a conversation. @@ -778,8 +865,12 @@ export interface Conversation { * owned, transitively, by its non-background tasks. */ waitForIdle(context: Context): Promise; - /** The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. */ - watch(context: Context): Promise; + /** + * The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. Its + * `entries` are a window over the transcript, as `window` says: the newest 50 by default, then appends, dropping the + * oldest past 100. Built on the same mount as `Harness.observeConversation()`. + */ + watch(context: Context, window?: ConversationWindow): Promise; } export type ConversationWatch = WatchHandle; @@ -825,11 +916,6 @@ export interface Harness extends Session { * definition cannot take it settles as `orphaned` instead. */ abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">; - /** - * Run a task parked by `settings.restartLimit` again: clears its restart count and enables scheduling. Does nothing - * for a task that is not parked; rejects for an unknown one. - */ - retryTask(id: TaskId, context: Context): Promise; /** Resolve with the terminal receipt; cancelling `context` cancels only this wait. */ waitForTask(id: TaskId, context: Context): Promise>; /** Resolve when the ordinary ownership scope of every ownerless conversation has no live non-background task. */ @@ -837,13 +923,34 @@ export interface Harness extends Session { /** Session total: every conversation's `pi.usage` summed. */ usage(context: Context): Promise; /** - * Every live task with its owner edge, status, and owned conversations (spec §9.5), as a serialized watch; commits + * Every live task with its owner edge, status, and owned conversations (spec §9.4), as a serialized watch; commits * made while its listener runs arrive folded. */ watchTaskGraph(context: Context): Promise; + /** + * Attach a synchronous `Observer` to a conversation's structural view (spec §9.3), atomically on the Session line, + * for adapters: `advance` gets each next revision with its operations as memory adopts the commit, inside Storage's + * `adopt` for a stored one and before it is stored for a deferred one, so it only records it. `window` as for + * `Conversation.watch()`; each observer has its own. `context` governs only the attachment. + */ + observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + window?: ConversationWindow, + ): Promise>; + /** + * Attach a synchronous `Observer` to the task graph (spec §9.4), as `observeConversation()` does to a view. A + * `blocked` reason that changes without a commit (a registry change, a failed migration) advances it on the Session + * line outside any commit and `adopt`, with the background context. + */ + observeTaskGraph(observer: Observer, context: Context): Promise>; } -/** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */ +/** + * What a hook may use: committed reads and the asking task's memos, which hooks and the task share. Hooks run off the + * Session line, so reads see what is committed when they run; a hook rerun after a crash may see more. + */ export interface HookApi extends DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; @@ -851,6 +958,17 @@ export interface HookApi extends DocumentReader { readonly models: Models; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; + /** Committed entry visible from the task's conversation, as `TaskRuntime.entry()`. */ + entry(id: EntryId, context: Context): Promise; + /** Undefined when the entry is absent, not visible, or has another kind. */ + entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** Committed raw history of the task's conversation, as `TaskRuntime.entries()`. */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; } export type HookResult = T | undefined | Promise; @@ -863,14 +981,50 @@ export interface GenerationHooks { api: HookApi, context: Context, ): HookResult<{ readonly messages: readonly Message[] }>; + /** + * After pi-ai builds the provider request body, before it is sent; the result replaces the body for that request only. + * Runs again whenever pi-ai rebuilds the body, such as a retry within one attempt. The body is in the provider API's + * own format (`request.model.api`), so a hook checks the API before changing it. + */ + beforeProviderRequest( + request: { readonly model: Model; readonly payload: unknown }, + api: HookApi, + context: Context, + ): HookResult<{ readonly payload: unknown }>; /** Every terminal provider message, before classification. */ afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise; /** A final answer; the first `continue` appends a user message and continues the run. */ onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>; /** After every tool of the round is terminal; `results` are the round's result entries in call order. */ afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; + /** + * After a run answered, its answer committed and its inputs settled `done`: a final answer, or a `terminate` round; + * not when an `onYield` continuation is taken, and never after a failure or an abort. The requested tasks of every + * hook are created together as children of the generation, which keeps the run, and the conversation busy, until + * they are terminal; then queued items are placed and the run ends (spec §8.3). A crash before they are created asks + * again; once created, never again for this run. + */ + afterRun(run: RunSummary, api: HookApi, context: Context): HookResult<{ readonly tasks: readonly RunEndTask[] }>; } +/** How a run ended, for `afterRun` hooks. Read the run's history with `api.entries({ minEntryId: first, maxEntryId: last })`. */ +export type RunSummary = { + /** A final answer, or a tool round whose every call asked to `terminate`. */ + readonly reason: "answer" | "terminate"; + /** + * The run's first entry (`LiveRun.first`): the first one the boundary that started it placed, for a continue the failed + * run's it continues, otherwise its first tool-calling answer; the answer itself for a run that has none of these. + */ + readonly first: EntryId; + /** The answer, or the terminating round's tool-calling answer, whose results follow it. */ + readonly answer: EntryId; + /** The run's last entry: the answer; for a `terminate` round its newest result, or the reset of its handoff. */ + readonly last: EntryId; +}; + +/** A task an `afterRun` hook asks for, built by `runEndTask()`. */ +export type RunEndTask = { readonly task: AnyTask; readonly input: JsonValue }; + /** A call as tool hooks see it; `parent` is set for a nested call, one a running tool made through `executeTool()`. */ export type ToolHookCall = ToolCall & { readonly parent?: { readonly taskId: TaskId; readonly callId: string }; @@ -884,7 +1038,18 @@ export interface ToolHooks { api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result is committed; replaces the result. */ + /** + * Before the result is committed; replaces the result. Runs on every result the tool task writes: an executed + * call's, and the error results the Harness writes itself, told apart by their diagnostic code: a tool that does not + * resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent + * them), a block (`blocked`), and, rebuilt from the call's partial output, an interruption (`interrupted`), the + * restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`). Once the task was cut off more than + * `settings.restartLimit` times the hooks no longer run, since they may be what cuts it off, nor in the last attempt + * the scheduler gives an abort handler (restarts at the limit minus one). A result's `structuredOutput` is dropped + * when the Harness wrote it (a nested call a hook turns into a success gets the bounded output as a schema-less tool's + * does), and its `control` applies only to a call that completes (not to an interrupted, cut-off, or aborted one). + * An abort's hooks run inside the abort handler, which cannot be aborted: a slow hook delays `Conversation.abort()`. + */ afterTool( call: ToolHookCall, result: ToolExecutionResult, diff --git a/packages/durable/src/harness/util.ts b/packages/durable/src/harness/util.ts index cf2222a5a..f3c8ea374 100644 --- a/packages/durable/src/harness/util.ts +++ b/packages/durable/src/harness/util.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { SessionFailed } from "../errors.ts"; -import type { Cursor, Page } from "../types.ts"; +import type { ConversationId, Cursor, EntryQuery, Page } from "../types.ts"; /** Pending waits by key. Each settles once: through `resolve`, `rejectAll`, or cancellation of its context. */ export class Waiters { @@ -64,3 +64,16 @@ export function closedError(session: { readonly failure: { readonly error: unkno export function optionalExtension(extension: string | undefined): { readonly extension?: string } { return extension === undefined ? {} : { extension }; } + +/** The scan of conversation `conversationId`'s history that `query` asks for, without fields an untyped caller added. */ +export function conversationEntries( + conversationId: ConversationId, + query: Omit, +): EntryQuery { + return { + conversationId, + ...(query.minEntryId === undefined ? {} : { minEntryId: query.minEntryId }), + ...(query.maxEntryId === undefined ? {} : { maxEntryId: query.maxEntryId }), + ...(query.order === undefined ? {} : { order: query.order }), + }; +} diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index d47d2e913..130d1c459 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,47 +1,84 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { + type Context, + type JsonValue, + Watch, + type WatchCall, + type WatchEnd, + type WatchHandle, +} from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; -import { contained, type SessionImpl } from "../session/session.ts"; +import { idFromNumber } from "../ids.ts"; +import { + closeEndOf, + closeEnds, + contained, + guardObserver, + type ObservedCommit, + observation, + type SessionImpl, +} from "../session/session.ts"; import type { - CommitPublication, ConversationDocToken, ConversationId, ConversationRecord, + Cursor, DocumentId, + EntryId, + EntryQuery, EntryRecord, JsonObject, + Observation, + Observer, Storage, } from "../types.ts"; import { AgentDoc } from "./agent.ts"; -import { activeEntries, captureContextBounds } from "./context.ts"; import { InboxDoc } from "./inbox.ts"; import { LiveDoc } from "./live.ts"; import { ProviderDoc } from "./provider.ts"; import { UsageDoc } from "./usage.ts"; import { closedError } from "./util.ts"; -/** Structural mount of one conversation's active transcript and built-in documents (spec §9.3). */ +/** Structural view of one conversation: a window over its transcript and its built-in documents (spec §9.3). */ export type ConversationView = { readonly conversation: ConversationRecord; - /** Raw active entries, as `ContextView.entries`: the head marker, then the non-head entries from its head. */ + /** + * A window over the transcript, its fork-aware history (`Conversation.entries()`), in entry ID order. Head markers + * such as compaction summaries sit where they were written; their `head` tells where the entries they summarize end. + */ readonly entries: readonly EntryRecord[]; + /** ID of the entry just before the window; absent when the window starts the transcript. */ + readonly before?: EntryId; /** Built-in conversation documents keyed by kind; absent documents are absent. */ readonly docs: Readonly>; }; -/** Receives each next revision of a mount, and the Session's close. */ +/** + * Which entries a view's window holds (spec §9.3). The window holds between `min` and `max` entries: it starts with + * the newest `min` entries, takes appends, and once it would hold more than `max`, drops its oldest in one splice down + * to `min`. With `from`, it starts at `from` instead when at most `max` entries are at or after it. + */ +export type ConversationWindow = { + /** + * Where the first value's window starts, inclusive, such as right after the newest entry a client has + * (`ConversationTranscript.resume`). It selects only that first value: every entry committed after the attachment is + * appended, whatever its ID. A `from` past the newest entry starts the window empty, with `before` the newest entry. + */ + readonly from?: EntryId; + /** Default 50; an integer, at least 1. */ + readonly min?: number; + /** Default `2 * min`; an integer, at least `min`. */ + readonly max?: number; +}; + +/** Default `ConversationWindow.min`. */ +const DEFAULT_WINDOW_MIN = 50; + +/** Receives each next revision of one observer's window, and the Session's close. */ export type ViewObserver = { - advance?(value: ConversationView, ops: readonly Op[], context: Context): void; - /** Every publication, after the mount took it; `ops` are the mount's, possibly none. */ - publication?( - before: ConversationView, - after: ConversationView, - ops: readonly Op[], - publication: CommitPublication, - context: Context, - ): void; - /** The Session closed, or failed with `failure`. */ - end(failure: { readonly error: unknown } | undefined): void; + advance(call: WatchCall, context: Context): void; + /** The Session closed (`ended`), or failed (`failed`). */ + end(end: WatchEnd): void; }; const MOUNTED = [ @@ -52,13 +89,26 @@ const MOUNTED = [ UsageDoc, ] as unknown as readonly ConversationDocToken[]; const MOUNTED_KINDS: ReadonlySet = new Set(MOUNTED.map((token) => token.definition.kind)); +const SCAN_PAGE_SIZE = 256; -/** One conversation's mount: its current revision, the document incarnations it shows, and its observers. */ -type Mount = { +/** One observer of a mount and its window: the bounds and the revision it was last given. */ +type Windowed = { + readonly min: number; + readonly max: number; value: ConversationView; + readonly observer: ViewObserver; +}; + +/** + * One conversation's mount: the documents it shows, shared by every observer, and the observers, each with its own + * window over the entries. + */ +type Mount = { + readonly conversation: ConversationRecord; + docs: Readonly>; /** Mounted incarnation and definition version per kind; another incarnation or version is set whole. */ - readonly docs: Map; - readonly observers: Set; + readonly incarnations: Map; + readonly observers: Set; }; const VIEWS = new WeakMap(); @@ -70,30 +120,48 @@ export function conversationViews(harness: object): ConversationViews { return views; } +/** A window with its defaults applied. */ +type WindowBounds = { readonly from: EntryId | undefined; readonly min: number; readonly max: number }; + +/** `window` with its defaults; throws a `RangeError` for bounds that are not integers or `max` below `min`. */ +function resolveWindow(window: ConversationWindow | undefined): WindowBounds { + const min = window?.min ?? DEFAULT_WINDOW_MIN; + if (!Number.isSafeInteger(min) || min < 1) throw new RangeError(`Window min must be an integer >= 1, got ${min}`); + const max = window?.max ?? 2 * min; + if (!Number.isSafeInteger(max) || max < min) { + throw new RangeError(`Window max must be an integer >= min (${min}), got ${max}`); + } + const from = window?.from; + if (from !== undefined && !Number.isSafeInteger(from)) { + throw new RangeError(`Window from must be an entry ID, got ${from}`); + } + return { from, min, max }; +} + /** * The Harness's conversation view mounts: at most one per conversation, built on the Session line by its first - * observer and dropped with its last. Each mount advances from the Session's commit publications, which are durable. + * observer and dropped with its last. Each mount advances from the Session's commits as memory adopts them, deferred ones + * (`CommitPersistence`) before they are stored, and hands each observer the next revision of its own window. */ export class ConversationViews { readonly #session: SessionImpl; - readonly #storage: Storage; readonly #mounts = new Map(); #closed = false; - constructor(session: SessionImpl, storage: Storage) { + constructor(session: SessionImpl) { this.#session = session; - this.#storage = storage; VIEWS.set(session, this); - session.observeCommits((publication, context) => { + session.followCommits((publication, context) => { for (const [id, mount] of this.#mounts) advance(id, mount, publication, context, (error) => this.report(error)); }); session.subscribeClose(() => { this.#closed = true; + const ends = closeEnds(this.#session.failure); for (const mount of this.#mounts.values()) - for (const observer of [...mount.observers]) { + for (const windowed of [...mount.observers]) { contained( - () => observer.end(this.#session.failure), + () => windowed.observer.end(closeEndOf(windowed.observer, ends)), (error) => this.report(error), ); } @@ -106,53 +174,90 @@ export class ConversationViews { this.#session.report(error); } - /** A serialized watch of the view; cancelling `context` stops it. */ - async watch(id: ConversationId, context: Context): Promise> { + /** + * Attach a synchronous observer of the view (`Harness.observeConversation()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe( + id: ConversationId, + observer: Observer, + context: Context, + window?: ConversationWindow, + ): Promise> { + let value!: ConversationView; + const attached = await this.attach( + id, + window, + (current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.report(error)); + }, + context, + ); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the view, an observer of its mount; aborting `context` ends it `aborted`. */ + async watch( + id: ConversationId, + context: Context, + window?: ConversationWindow, + ): Promise> { const { observer } = await this.attach( id, + window, (value, release) => new Watch(value, release, (error) => this.#session.report(error)), context, ); const signal = context.abortSignal; if (signal?.aborted) { - observer.cancel(); + observer.abort(); throw signal.reason; } - if (signal !== undefined) observer.observeCancellation(signal); + if (signal !== undefined) observer.observeAbort(signal); return observer; } /** - * Register an observer created from the current revision, atomically on the Session line: it sees every later - * publication and nothing earlier. `create` may read committed Storage, still on the line. `release` drops it, and - * the mount with its last observer. + * Register an observer created from its window's first revision, atomically on the Session line: it sees every later + * publication and nothing earlier. `release` drops it, and the mount with its last observer. */ - attach( + async attach( id: ConversationId, - create: (value: ConversationView, release: () => void, storage: Storage) => O | Promise, + window: ConversationWindow | undefined, + create: (value: ConversationView, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { - return this.#session.readOnLine(async () => { - const mount = this.#mounts.get(id) ?? (await this.#build(id, context)); + const bounds = resolveWindow(window); + return this.#session.readOnLine(async (storage) => { + const mount = this.#mounts.get(id) ?? (await this.#build(storage, id, context)); + const { entries, before } = await loadWindow(storage, id, bounds, context); + const value = view(mount.conversation, entries, before, mount.docs); + let windowed: Windowed | undefined; const detach = (): void => { - mount.observers.delete(observer); + if (windowed !== undefined) mount.observers.delete(windowed); if (mount.observers.size === 0 && this.#mounts.get(id) === mount) this.#mounts.delete(id); }; - const observer = await create(mount.value, detach, this.#storage); + const observer = create(value, detach); // Close or cancellation may begin while the mount hydrates; register nothing then. if (this.#closed) throw closedError(this.#session); context.abortSignal?.throwIfAborted(); + windowed = { min: bounds.min, max: bounds.max, value, observer }; this.#mounts.set(id, mount); - mount.observers.add(observer); + mount.observers.add(windowed); return { observer, detach }; }); } - async #build(id: ConversationId, context: Context): Promise { - const conversation = await this.#storage.conversation(id, context); + /** Hydrate a mount on the line, through the line's `storage`. */ + async #build(storage: Storage, id: ConversationId, context: Context): Promise { + const conversation = await storage.conversation(id, context); if (conversation === undefined) throw new Error(`Conversation ${id} does not exist`); - const bounds = await captureContextBounds(this.#storage, id, context); - const entries = await activeEntries(this.#storage, id, bounds, context); const docs: Record = {}; const incarnations = new Map(); for (const token of MOUNTED) { @@ -161,70 +266,114 @@ export class ConversationViews { docs[token.definition.kind] = loaded.value; incarnations.set(token.definition.kind, { id: loaded.record.id, version: loaded.version }); } - return { value: { conversation, entries, docs }, docs: incarnations, observers: new Set() }; + return { conversation, docs, incarnations, observers: new Set() }; } } +/** A view revision; `before` is a key only when set. */ +function view( + conversation: ConversationRecord, + entries: readonly EntryRecord[], + before: EntryId | undefined, + docs: Readonly>, +): ConversationView { + return before === undefined ? { conversation, entries, docs } : { conversation, entries, before, docs }; +} + /** - * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that - * throws is reported through `report`, and the others still get it. + * A new observer's window, read on the Session line with bounded scans: the entries from `from` when at most `max` are + * at or after it, otherwise the newest `min`, and the entry before them. + */ +async function loadWindow( + storage: Storage, + conversationId: ConversationId, + bounds: WindowBounds, + context: Context, +): Promise<{ entries: readonly EntryRecord[]; before: EntryId | undefined }> { + const { from, min, max } = bounds; + if (from !== undefined) { + const since = await scanUpTo(storage, { conversationId, minEntryId: from, order: "ascending" }, max + 1, context); + if (since.length <= max) { + const query = { conversationId, maxEntryId: idFromNumber(from - 1) }; + const previous = (await storage.scanEntries(query, 1, undefined, context)).items[0]; + return { entries: since, before: previous?.id }; + } + } + const newest = await scanUpTo(storage, { conversationId }, min + 1, context); + const before = newest.length > min ? newest.pop()!.id : undefined; + return { entries: newest.reverse(), before }; +} + +/** Up to `limit` entries of `query`, following cursors past short pages. */ +async function scanUpTo(storage: Storage, query: EntryQuery, limit: number, context: Context): Promise { + const entries: EntryRecord[] = []; + let cursor: Cursor | undefined; + do { + const page = await storage.scanEntries(query, Math.min(SCAN_PAGE_SIZE, limit - entries.length), cursor, context); + entries.push(...page.items); + cursor = page.next; + } while (cursor !== undefined && entries.length < limit); + return entries; +} + +/** + * Derive the mount's operations from one publication and hand each observer the next revision of its window; one that + * throws is reported through `report`, and the others still get theirs. Documents are applied once and shared; each + * window takes the appended entries and, past its `max`, drops its oldest down to its `min` in one splice. */ function advance( id: ConversationId, mount: Mount, - publication: CommitPublication, + publication: ObservedCommit, context: Context, report: (error: unknown) => void, ): void { const docOps: Op[] = []; - const entryOps: Op[] = []; - let entries = mount.value.entries; // Entry writes are published in ID order. + const appended: EntryRecord[] = []; for (const change of publication.changes) { if (change.type === "entry" && change.value.conversationId === id) { - const entry = change.value; - const value = entry as unknown as JsonValue; - if (entry.head === undefined) { - entryOps.push(["p", ["entries"], entries.length, 0, [value]]); - entries = [...entries, entry]; - continue; - } - // A head marker keeps the non-head entries from its head, which are always a suffix, and goes in front. - const target = entry.head; - let kept = entries.findIndex((candidate) => candidate.head === undefined && candidate.id >= target); - if (kept < 0) kept = entries.length; - entryOps.push(["p", ["entries"], 0, kept, [value]]); - entries = [entry, ...entries.slice(kept)]; + appended.push(change.value); continue; } if (change.type !== "document" || change.conversationId !== id) continue; const kind = change.record.kind; if (!MOUNTED_KINDS.has(kind) || change.record.key !== undefined) continue; const path: NonEmptyPath = ["docs", kind]; - const mounted = mount.docs.get(kind); - if (change.value === null) { + const mounted = mount.incarnations.get(kind); + if (change.kind === "retire") { if (mounted?.id !== change.record.id) continue; - mount.docs.delete(kind); + mount.incarnations.delete(kind); docOps.push(["d", path]); } else if (mounted?.id === change.record.id && mounted.version === change.version) { for (const op of change.ops) docOps.push(prefixed(op, path)); } else { - mount.docs.set(kind, { id: change.record.id, version: change.version! }); + mount.incarnations.set(kind, { id: change.record.id, version: change.version }); docOps.push(["s", path, change.value]); } } - const before = mount.value; - const ops = [...docOps, ...entryOps]; + if (docOps.length === 0 && appended.length === 0) return; + if (docOps.length > 0) mount.docs = applyImmutable({ docs: mount.docs }, docOps).docs; const frameContext = withoutAbortSignal(context); - if (ops.length > 0) { - const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps); - mount.value = entries === value.entries ? value : { ...value, entries }; - for (const observer of [...mount.observers]) { - contained(() => observer.advance?.(mount.value, ops, frameContext), report); + const inserted = appended as unknown as JsonValue[]; + for (const windowed of [...mount.observers]) { + const previous = windowed.value; + let entries = previous.entries; + let before = previous.before; + let ops: Op[] = docOps; + if (appended.length > 0) { + ops = [...docOps, ["p", ["entries"], entries.length, 0, inserted]]; + entries = entries.concat(appended); + if (entries.length > windowed.max) { + const drop = entries.length - windowed.min; + before = entries[drop - 1]!.id; + entries = entries.slice(drop); + ops.push(["p", ["entries"], 0, drop, []], ["s", ["before"], before]); + } } - } - for (const observer of [...mount.observers]) { - contained(() => observer.publication?.(before, mount.value, ops, publication, frameContext), report); + windowed.value = view(mount.conversation, entries, before, mount.docs); + const call: WatchCall = { kind: "value", value: windowed.value, ops, retired: false }; + contained(() => windowed.observer.advance(call, frameContext), report); } } diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 912d8f528..9d35a9378 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -1,4 +1,4 @@ -export type { WatchEnd, WatchHandle } from "@earendil-works/chord"; +export type { NativeWatchCall, WatchCall, WatchEnd, WatchHandle } from "@earendil-works/chord"; export { defineDoc, defineDocFamily } from "./documents.ts"; export { AssistantEntry, @@ -11,7 +11,7 @@ export { } from "./entries.ts"; export { ConversationBusy, - LogInterruptedError, + ConversationIdle, ReadAfterWrite, ReadOnlySession, SessionFailed, @@ -31,15 +31,7 @@ export { CompactionTask, type SummaryRequest, } from "./harness/compaction.ts"; -export { defineExtension, defineTool, hook, section, wrapSection, wrapTool } from "./harness/define.ts"; -export { - type AgentEvent, - type AgentEventStream, - type MessageChange, - type SnapshotEvent, - type ToolEventCall, - watchEvents, -} from "./harness/events.ts"; +export { defineExtension, defineTool, hook, runEndTask, section, wrapSection, wrapTool } from "./harness/define.ts"; export { type GenerationCheckpoint, type GenerationInput, @@ -52,13 +44,17 @@ export { type CompactionFailure, type CompactionStatus, LiveDoc, + type LiveRun, type LiveState, type NestedToolSlot, type NestedToolSummary, + type RunFailure, + type RunFailureReason, type ToolSlot, } from "./harness/live.ts"; export { ProviderDoc, type ProviderState } from "./harness/provider.ts"; export { createRegistry } from "./harness/registry.ts"; +export { registryReader } from "./harness/registry-reader.ts"; export type { BlockedReason } from "./harness/scheduler.ts"; export type { TaskGraph, TaskGraphNode, TaskGraphState, TaskGraphWatch } from "./harness/task-graph.ts"; export { DURABLE_TELEMETRY_SCHEMA } from "./harness/telemetry.ts"; @@ -70,6 +66,13 @@ export { type ToolTaskInput, type ToolTaskResult, } from "./harness/tool.ts"; +export { + ConversationTranscript, + type ConversationTranscriptOptions, + type EntryLoader, + type TranscriptItem, + type TranscriptView, +} from "./harness/transcript.ts"; export type { AfterFailedRun, Agent, @@ -111,6 +114,8 @@ export type { Registry, RegistryReader, RegistrySnapshot, + RunEndTask, + RunSummary, Settings, SettledSubmissionRecord, SettledTask, @@ -130,14 +135,19 @@ export type { Wrap, } from "./harness/types.ts"; export { UsageDoc, type UsageState } from "./harness/usage.ts"; -export type { ConversationView } from "./harness/view.ts"; +export type { ConversationView, ConversationWindow } from "./harness/view.ts"; export { createSession } from "./session/session.ts"; export { MemoryStorage } from "./storage/memory.ts"; export { defineTask } from "./tasks.ts"; export type { CheckpointInfo, CommitChange, + CommitObservation, + CommitObserver, + CommitOptions, + CommitPersistence, CommitPublication, + CommitResult, CommonDocDefinition, ContextEdit, ConversationDocFamilyToken, @@ -175,12 +185,12 @@ export type { JoinPolicy, JsonObject, LatestConversationSemantics, - LogRecord, - LogWrite, NextTaskState, + Observation, + Observer, Page, PhaseHandler, - ReplayableStorage, + Report, ReportFrom, ReportInfo, ReportTask, diff --git a/packages/durable/src/search.ts b/packages/durable/src/search.ts index eb89772a9..9dc6ed5ec 100644 --- a/packages/durable/src/search.ts +++ b/packages/durable/src/search.ts @@ -3,7 +3,115 @@ * ranking: durable keeps its own copy of what it needs from the coding agent. */ -export type SearchDocument = { readonly name: string; readonly text: string }; +/** A tool as the ranker sees it: its name, its text, and the text of its namespace. */ +export type SearchDocument = { + readonly name: string; + /** Description and parameter names and descriptions; not the name, which the ranker reads from `name`. */ + readonly text: string; + /** Name, description, and instructions of the tool's namespace. */ + readonly namespaceText?: string; +}; + +/** + * Okapi BM25 (k1 1.2, b 0.75) over `text`, with the name and the namespace as separate fields: the names of the best + * matches, best first; ties keep document order. + * - A tool that matches more of the query's terms, in any field, ranks above one that matches fewer, whatever their + * scores: for `screenshot window`, a tool whose description has both words beats `window` whose name has one. + * - A query term in the name scores more than any number of mentions in the text. + * - A tool whose whole local name (after the last `__`) is in the query gets a bonus worth one more match of every + * query term, so among tools matching as many terms, `peekaboo app` finds `mcp__peekaboo__app` over tools that + * only mention the words. + * - A query that is exactly a tool's name, or its name after a `__`, ranks that tool first, before coverage. + * + * Only `text` is length-normalized, so a long name or long namespace instructions do not lower a tool's score. + */ +export function rankDocuments(query: string, documents: readonly SearchDocument[], limit: number): string[] { + const queryTerms = new Set(tokenize(query)); + if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return []; + const indexed = documents.map(indexDocument); + const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1; + const idf = new Map( + [...queryTerms].map((term) => { + const frequency = indexed.filter( + (document) => + document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term), + ).length; + return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const; + }), + ); + const fullNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); + const exactQuery = normalizeName(query); + const matches: { name: string; score: number; covered: number; exact: boolean }[] = []; + for (const document of indexed) { + const norm = K1 * (1 - B + (B * document.textLength) / averageLength); + let score = 0; + let covered = 0; + for (const term of queryTerms) { + const count = document.textCounts.get(term) ?? 0; + const weight = + (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + + (count * (K1 + 1)) / (count + norm); + if (weight > 0) covered++; + score += idf.get(term)! * weight; + } + if (score === 0) continue; + const local = document.localNameTerms; + if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += fullNameBonus; + matches.push({ name: document.name, score, covered, exact: isExactNameQuery(exactQuery, document.name) }); + } + return matches + .sort( + (left, right) => + Number(right.exact) - Number(left.exact) || right.covered - left.covered || right.score - left.score, + ) + .slice(0, limit) + .map((match) => match.name); +} + +const K1 = 1.2; + +const B = 0.75; + +/** + * A query term in the name counts this many times its IDF. The text part of a term is at most `K1 + 1` (2.2) times + * its IDF and about 1 for a single mention, so a name match beats any number of mentions in the text. + */ +const NAME_WEIGHT = 3; + +/** + * A query term in the namespace text counts this many times its IDF. Every tool of the namespace shares it, so it helps + * find the namespace but not the tool in it. + */ +const NAMESPACE_WEIGHT = 0.5; + +/** A document split into the fields the ranker weighs differently. */ +type IndexedDocument = { + name: string; + /** Terms of the name: `mcp`, `app`, `thread`, `read` for `mcp__app__thread_read`. */ + nameTerms: ReadonlySet; + /** Terms of the name after its last `__`, the part models say: `thread`, `read`. */ + localNameTerms: readonly string[]; + textCounts: ReadonlyMap; + textLength: number; + namespaceTerms: ReadonlySet; +}; + +function indexDocument(document: SearchDocument): IndexedDocument { + const textCounts = new Map(); + for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1); + const segments = document.name.split("__"); + return { + name: document.name, + nameTerms: new Set(tokenize(document.name)), + localNameTerms: [...new Set(tokenize(segments[segments.length - 1]!))], + textCounts, + textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0), + namespaceTerms: new Set(tokenize(document.namespaceText ?? "")), + }; +} + +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); const STOP_WORDS: ReadonlySet = new Set([ "a", @@ -29,6 +137,24 @@ const STOP_WORDS: ReadonlySet = new Set([ "with", ]); +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ +export function tokenize(text: string): string[] { + const normalized = text + .normalize("NFC") + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); +} + /** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */ function stem(term: string): string { if (term.length > 4 && term.endsWith("ies")) return `${term.slice(0, -3)}y`; @@ -37,49 +163,19 @@ function stem(term: string): string { return term; } -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ -export function tokenize(text: string): string[] { - return text - .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") +/** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */ +function normalizeName(name: string): string { + return name + .trim() .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^a-z0-9_$]/g, "_"); } -/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */ -export function rankDocuments(query: string, documents: readonly SearchDocument[], limit: number): string[] { - const k1 = 1.2; - const b = 0.75; - const queryTerms = [...new Set(tokenize(query))]; - if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; - const termCounts = documents.map((document) => { - const counts = new Map(); - for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); - return counts; - }); - const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); - const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; - const idf = new Map( - queryTerms.map((term) => { - const frequency = termCounts.filter((counts) => counts.has(term)).length; - return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; - }), - ); - const matches: { name: string; score: number }[] = []; - documents.forEach((document, index) => { - let score = 0; - for (const term of queryTerms) { - const count = termCounts[index]!.get(term); - if (!count) continue; - const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength); - score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm)); - } - if (score > 0) matches.push({ name: document.name, score }); - }); - return matches - .sort((left, right) => right.score - left.score) - .slice(0, limit) - .map((match) => match.name); +/** + * Whether the whole query is the tool's name: its full name or a suffix after a `__` separator, so `thread_read` and + * `app__thread_read` both name `mcp__app__thread_read`. + */ +function isExactNameQuery(query: string, name: string): boolean { + const normalized = normalizeName(name); + return normalized === query || normalized.endsWith(`__${query}`); } diff --git a/packages/durable/src/session/cached-storage.ts b/packages/durable/src/session/cached-storage.ts new file mode 100644 index 000000000..5087a2834 --- /dev/null +++ b/packages/durable/src/session/cached-storage.ts @@ -0,0 +1,378 @@ +// Write-through cache of hot Storage record reads for the Session that owns the Storage. +import type { Context, JsonValue } from "@earendil-works/chord"; +import type { + CommitResult, + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../types.ts"; + +type AnyTaskRecord = TaskRecord; +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** + * Storage that serves hot record reads from memory: conversations, tasks, and written submissions, each table bounded + * by a count, the least recently used record leaving first, except live tasks, which are kept until they end, and the + * ended tasks a task waits on, kept until it moved on from its wait: a task's record is read again at each of its state + * changes, a waiter reads the outcomes of what it waited on once it resumes (`runtime.outcomes()`), and both are + * bounded by live work, which the scheduler keeps in memory too. Without that, a burst of more tasks than the count + * would make each of those reads a Storage read on the line. Every commit goes through it, and one process owns a + * Storage, so it applies each batch as it is published and never goes stale: `commit()`'s batches when the Session publishes + * them (`published()`), not at their durability receipts. A read + * result is kept only when no commit published since the read started wrote its record, as the read may not observe + * that commit; versions are kept only for records with a read in flight. Cached values are frozen and shared. Entries, + * scans, and documents pass through: the Session keeps documents and the newest entry per conversation itself, and + * context ranges hold the entries a run reads. + * + * Not for a read-only Session: another process writes its Storage. + */ +export class CachedStorage implements Storage { + readonly #storage: Storage; + readonly #conversations = new Lru(4096); + /** Live (non-terminal) task records, until their task ends. */ + readonly #liveTasks = new Map(); + /** + * Tasks some task waits on, or waited on until its wait ended and it has not moved on since: how many such waits name + * each, and its ended record once it ended. + */ + readonly #awaited = new Map(); + /** Per task that waits, or left its wait and has not changed since: what it waited on, and whether it left. */ + readonly #waits = new Map(); + // Ended task records carry inputs and outcomes, such as a tool call's arguments, so fewer are kept. + readonly #tasks = new Lru(256); + readonly #submissions = new Lru(256); + /** Per record with a read in flight: the reads, and how many published commits wrote it since the first began. */ + readonly #reads = new Map, { count: number; version: number }>(); + + constructor(storage: Storage) { + this.#storage = storage; + } + + /** The backend's own scope for a publication (`Storage.publishing`), when it has one. */ + get publishing(): Storage["publishing"] { + return scopeOf(this.#storage); + } + + /** Pass the batches through; the Session applies each one with `published()` when it publishes it. */ + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#storage.commit(batches, receive, context); + } + + /** + * Apply a batch `commit()` stored, when the Session publishes it. Its records are frozen in place and kept as + * they are, no copies: the Session's components that follow the publication, such as the scheduler's live tasks, + * hold the same objects, so a live task's record is in memory once. + */ + published(writes: readonly StorageWrite[]): void { + for (const write of writes) { + switch (write.type) { + case "conversation": + this.#wrote(write.value.id); + this.#conversations.set(write.value.id, freeze(write.value)); + break; + case "task": + this.#wrote(write.value.id); + this.#followWait(write.value); + this.#keepTask(freeze(write.value)); + break; + case "submission": + this.#wrote(write.value.id); + this.#submissions.set(write.value.id, freeze(write.value)); + break; + } + } + } + + mintId>(): Promise { + return this.#storage.mintId(); + } + + conversation(id: ConversationId, context: Context): Promise { + const cached = this.#conversations.get(id); + if (cached !== undefined) return Promise.resolve(cached); + return this.#keep( + id, + () => this.#storage.conversation(id, context), + (record) => this.#conversations.set(id, record), + ); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanConversations(query, limit, cursor, context); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + first: EntryId | ConversationId, + second: EntryId | Context, + third?: Context, + ): Promise { + if (third === undefined) return this.#storage.entry(first as EntryId, second as Context); + return this.#storage.entry(first as ConversationId, second as EntryId, third); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#storage.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanEntries(query, limit, cursor, context); + } + + task(id: TaskId, context: Context): Promise { + const cached = this.#liveTasks.get(id) ?? this.#awaited.get(id)?.record ?? this.#tasks.get(id); + if (cached !== undefined) return Promise.resolve(cached); + return this.#keep( + id, + () => this.#storage.task(id, context), + (record) => this.#keepTask(record), + ); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanTasks(query, limit, cursor, context); + } + + /** Cached as written only: this process writes every submission it settles, and a read one is usually settled. */ + submission(id: SubmissionId, context: Context): Promise { + const cached = this.#submissions.get(id); + return cached === undefined ? this.#storage.submission(id, context) : Promise.resolve(cached); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanSubmissions(query, limit, cursor, context); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + return this.#storage.submissionByRequest(conversationId, requestId, context); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#storage.findDocument(address, at, context); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + return this.#storage.document(id, at, context); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanDocuments(query, limit, cursor, context); + } + + close(context: Context): Promise { + return this.#storage.close(context); + } + + /** Records kept per table, for tests that bound memory. */ + get sizes(): { + readonly conversations: number; + readonly tasks: number; + readonly liveTasks: number; + readonly awaited: number; + readonly waits: number; + readonly submissions: number; + } { + return { + conversations: this.#conversations.size, + tasks: this.#tasks.size, + liveTasks: this.#liveTasks.size, + awaited: this.#awaited.size, + waits: this.#waits.size, + submissions: this.#submissions.size, + }; + } + + /** Keep a frozen task record: a live one until its task ends, an ended one in the bounded table. */ + #keepTask(record: AnyTaskRecord): void { + if (record.state.status === "terminal") { + this.#liveTasks.delete(record.id); + const awaited = this.#awaited.get(record.id); + if (awaited !== undefined) awaited.record = record; + else this.#tasks.set(record.id, record); + return; + } + this.#tasks.delete(record.id); + this.#liveTasks.set(record.id, record); + } + + /** + * Follow a task's waits from its written record: a wait holds the ended records of its `on` until the task, after + * leaving the wait, changes again (or ends), as the phase that resumes reads their outcomes before its next commit. + */ + #followWait(record: AnyTaskRecord): void { + const wait = this.#waits.get(record.id); + if (record.state.status === "waiting") { + if (wait !== undefined) this.#release(wait.on); + const on = record.state.on; + this.#waits.set(record.id, { on, left: false }); + for (const id of on) { + let awaited = this.#awaited.get(id); + if (awaited === undefined) { + // One that ended already moves over from the bounded table. + const ended = this.#tasks.get(id); + this.#tasks.delete(id); + awaited = ended === undefined ? { count: 0 } : { count: 0, record: ended }; + this.#awaited.set(id, awaited); + } + awaited.count++; + } + return; + } + if (wait === undefined) return; + if (!wait.left && record.state.status !== "terminal") { + wait.left = true; + return; + } + this.#waits.delete(record.id); + this.#release(wait.on); + } + + /** A wait on `on` is over: what no other wait names goes back to the bounded table. */ + #release(on: readonly TaskId[]): void { + for (const id of on) { + const awaited = this.#awaited.get(id)!; + if (--awaited.count > 0) continue; + this.#awaited.delete(id); + if (awaited.record !== undefined) this.#tasks.set(id, awaited.record); + } + } + + /** A published commit wrote record `id`: a read of it in flight may have missed that. */ + #wrote(id: Id): void { + const reads = this.#reads.get(id); + if (reads !== undefined) reads.version++; + } + + /** Resolve with a read's result, frozen; keep a found record unless a commit published since wrote it. */ + async #keep( + id: Id, + read: () => Promise, + keep: (value: T) => void, + ): Promise { + let reads = this.#reads.get(id); + if (reads === undefined) { + reads = { count: 0, version: 0 }; + this.#reads.set(id, reads); + } + reads.count++; + const version = reads.version; + let value: T | undefined; + try { + value = await read(); + } finally { + if (--reads.count === 0) this.#reads.delete(id); + } + if (value === undefined) return undefined; + freeze(value); + if (reads.version === version) keep(value); + return value; + } + + /** Records with a read in flight, for tests that bound memory. */ + get readsInFlight(): number { + return this.#reads.size; + } +} + +/** `storage`'s publication scope (`Storage.publishing`), for a wrapper to expose as its own; none when it has none. */ +export function scopeOf(storage: Storage): Storage["publishing"] { + if (storage.publishing === undefined) return undefined; + return (publish, context) => storage.publishing!(publish, context); +} + +/** A Map bounded to `capacity` entries that drops the least recently used one. */ +class Lru { + readonly #map = new Map(); + readonly #capacity: number; + + constructor(capacity: number) { + this.#capacity = capacity; + } + + get(key: K): V | undefined { + const value = this.#map.get(key); + if (value === undefined) return undefined; + this.#map.delete(key); + this.#map.set(key, value); + return value; + } + + get size(): number { + return this.#map.size; + } + + delete(key: K): void { + this.#map.delete(key); + } + + set(key: K, value: V): void { + this.#map.delete(key); + this.#map.set(key, value); + if (this.#map.size > this.#capacity) this.#map.delete(this.#map.keys().next().value!); + } +} + +function freeze(value: T): T { + if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) freeze(child); + return Object.freeze(value); +} diff --git a/packages/durable/src/session/conversation-tails.ts b/packages/durable/src/session/conversation-tails.ts new file mode 100644 index 000000000..2064f77b0 --- /dev/null +++ b/packages/durable/src/session/conversation-tails.ts @@ -0,0 +1,115 @@ +// The newest entry and newest head marker of each recently used conversation, kept current from the Session's commits. +import type { ConversationId, EntryId, EntryRecord, StorageWrite } from "../types.ts"; + +/** An entry that carries a `head`: the marker of a context range. */ +export type HeadMarker = EntryRecord & { readonly head: EntryId }; + +/** Conversations whose newest entry ID is kept: IDs only, so many. */ +const LATEST_CAPACITY = 4096; + +/** Conversations whose newest head marker is kept: whole entries (a compaction summary is one), so fewer. */ +const MARKER_CAPACITY = 256; + +/** + * The newest entry ID and newest head marker per conversation, for the Session that owns its Storage: every entry is + * written by one of its commits, so `apply` keeps both current, and a value read from Storage stays valid until the + * next entry of that conversation. Read only on the Session line after touching the conversation's appends, so no + * commit that appends to it is unpublished and a value read from Storage can be kept. The newest entry visible from a conversation is its own newest, or for a fork without entries + * of its own the parent's entry at the fork point: IDs grow, so a conversation's own new entry is always the newest. + * Each map is bounded by a count, the least recently used conversation leaving first. Kept markers are frozen: entry + * records are immutable (`Tx.appendEntry()`). + */ +export class ConversationTails { + /** Newest entry ID per conversation, `null` for none. */ + readonly #latest = new Map(); + /** Newest head marker per conversation, `null` for none. */ + readonly #markers = new Map(); + + /** The newest entry ID visible from `conversationId`, read with `read` when not known. */ + async latest( + conversationId: ConversationId, + read: () => Promise, + ): Promise { + const known = use(this.#latest, conversationId); + if (known !== undefined) return known ?? undefined; + const latest = await read(); + put(this.#latest, conversationId, latest ?? null, LATEST_CAPACITY); + return latest; + } + + /** + * The newest visible head marker at or before `atOrBefore` (any when `undefined`), read with `read` when not known. + * The newest marker answers every cutoff at or after it, and no marker answers every cutoff. + */ + async marker( + conversationId: ConversationId, + atOrBefore: EntryId | undefined, + read: () => Promise, + ): Promise { + const marker = use(this.#markers, conversationId); + if (marker === null) return undefined; + if (marker !== undefined && (atOrBefore === undefined || atOrBefore >= marker.id)) return marker; + const latest = this.#latest.get(conversationId); + const found = await read(); + // Only a read that no newer entry can change: no cutoff, or one at or after the newest entry. + if (atOrBefore === undefined || (latest !== undefined && (latest ?? 0) <= atOrBefore)) { + put(this.#markers, conversationId, found === undefined ? null : freeze(found), MARKER_CAPACITY); + } + return found; + } + + /** The newest entry ID of `conversationId` if known, without reading: `null` for none, `undefined` when unknown. */ + peek(conversationId: ConversationId): EntryId | null | undefined { + return this.#latest.get(conversationId); + } + + /** At a commit's publication: apply a durable batch. */ + apply(writes: readonly StorageWrite[]): void { + for (const write of writes) { + if (write.type === "conversation") { + // A new root conversation sees no entries; a fork sees its parent's up to the fork point. + if (write.value.parent !== undefined) continue; + put(this.#latest, write.value.id, null, LATEST_CAPACITY); + put(this.#markers, write.value.id, null, MARKER_CAPACITY); + } else if (write.type === "entry") { + const entry = write.value; + put(this.#latest, entry.conversationId, entry.id, LATEST_CAPACITY); + if (entry.head !== undefined) { + put(this.#markers, entry.conversationId, freeze(entry as HeadMarker), MARKER_CAPACITY); + } + } + } + } + + /** How many conversations have their newest entry kept, for tests that bound memory. */ + get size(): number { + return this.#latest.size; + } + + /** How many conversations have their newest head marker kept, for tests that bound memory. */ + get markers(): number { + return this.#markers.size; + } +} + +/** `map`'s value for `key`, made the most recently used. */ +function use(map: Map, key: K): V | undefined { + const value = map.get(key); + if (value === undefined) return undefined; + map.delete(key); + map.set(key, value); + return value; +} + +/** Set `key` as the most recently used, dropping the least recently used beyond `capacity`. */ +function put(map: Map, key: K, value: V, capacity: number): void { + map.delete(key); + map.set(key, value); + if (map.size > capacity) map.delete(map.keys().next().value!); +} + +function freeze(value: T): T { + if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) freeze(child); + return Object.freeze(value); +} diff --git a/packages/durable/src/session/keys.ts b/packages/durable/src/session/keys.ts new file mode 100644 index 000000000..85fa8f2d3 --- /dev/null +++ b/packages/durable/src/session/keys.ts @@ -0,0 +1,274 @@ +// The keys of the Session line: what a commit writes and a job reads, so a read of an unpublished write waits for it. +import type { Context, JsonValue } from "@earendil-works/chord"; +import { addressId } from "../documents.ts"; +import type { + CommitResult, + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../types.ts"; + +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** A task record. */ +export function taskKey(id: TaskId): string { + return `task:${id}`; +} + +/** A conversation record. */ +export function conversationKey(id: ConversationId): string { + return `conv:${id}`; +} + +/** An entry record. */ +export function entryKey(id: EntryId): string { + return `entry:${id}`; +} + +/** The newest entries of a conversation: appended blind, read by scans, the newest entry, and head markers. */ +export function appendKey(id: ConversationId): string { + return `entry-append:${id}`; +} + +/** A submission record. */ +export function submissionKey(id: SubmissionId): string { + return `sub:${id}`; +} + +/** The submission a conversation's request ID names. */ +export function requestKey(conversationId: ConversationId, requestId: string): string { + return `req:${conversationId}:${requestId}`; +} + +/** A document address (`addressId()`): its current incarnation, value, and held changes. */ +export function documentKey(addressId: string): string { + return `doc:${addressId}`; +} + +/** A document incarnation, for reads by its ID. */ +export function incarnationKey(id: DocumentId): string { + return `docid:${id}`; +} + +/** The incarnations alive in a scope: created and retired blind, read by scans and the task-document index. */ +export function scopeKey(scope: DocumentRecord["scope"]): string { + switch (scope.kind) { + case "session": + return "docscope:session"; + case "conversation": + return `docscope:c${scope.conversationId}`; + case "task": + return `docscope:t${scope.taskId}`; + } +} + +/** + * A node of the scheduler's ownership tree (a task ID, or `c`): the live work listed below it. Written + * by task creation and a task becoming terminal for every node from the task's parent up to and including the nearest + * live task or the root conversation; read by every walk that expands the node. + */ +export function ownedKey(node: TaskId | string): string { + return `owned:${node}`; +} + +/** The ownerless conversations with live work below them, which global idle walks from. */ +export const ROOTS_KEY = "roots"; + +/** The live tasks with cancellation intent: abort marks, and held outcomes other than `completed`. */ +export const INTENT_KEY = "intent"; + +/** Every unpublished write: what a read that cannot be keyed, such as a scan, touches. */ +export const ALL_KEYS = "*"; + +/** The keys of the table records in `writes`; document writes are keyed by the transaction, which knows addresses. */ +export function tableKeys(writes: readonly StorageWrite[], keys: string[]): void { + for (const write of writes) { + switch (write.type) { + case "conversation": + keys.push(conversationKey(write.value.id)); + break; + case "entry": + keys.push(entryKey(write.value.id), appendKey(write.value.conversationId)); + break; + case "task": + keys.push(taskKey(write.value.id)); + break; + case "submission": + keys.push(submissionKey(write.value.id)); + if (write.value.requestId !== undefined) { + keys.push(requestKey(write.value.conversationId, write.value.requestId)); + } + break; + } + } +} + +/** + * The Storage a job on the line reads through: every read first touches its key (`touch`), which waits while the key + * has an unpublished writer, so the read sees what the line admitted before it. Scans that cannot be keyed touch + * `ALL_KEYS`. Commits and `close()` pass through; the line never commits through it. + */ +export class GatedStorage implements Storage { + readonly #storage: Storage; + readonly #touch: (key: string) => Promise | undefined; + + constructor(storage: Storage, touch: (key: string) => Promise | undefined) { + this.#storage = storage; + this.#touch = touch; + } + + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#storage.commit(batches, receive, context); + } + + mintId>(): Promise { + return this.#storage.mintId(); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#after(conversationKey(id), () => this.#storage.conversation(id, context)); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(ALL_KEYS, () => this.#storage.scanConversations(query, limit, cursor, context)); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + first: EntryId | ConversationId, + second: EntryId | Context, + third?: Context, + ): Promise { + if (third === undefined) { + const id = first as EntryId; + return this.#after(entryKey(id), () => this.#storage.entry(id, second as Context)); + } + const conversationId = first as ConversationId; + const id = second as EntryId; + // Visibility follows the conversation's immutable ancestry; the conversation may be unpublished itself. + return this.#after(conversationKey(conversationId), () => + this.#after(entryKey(id), () => this.#storage.entry(conversationId, id, third)), + ); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#afterEntries(conversationId, () => + this.#storage.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context), + ); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#afterEntries(query.conversationId, () => this.#storage.scanEntries(query, limit, cursor, context)); + } + + task(id: TaskId, context: Context): Promise | undefined> { + return this.#after(taskKey(id), () => this.#storage.task(id, context)); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise, Cursor>> { + return this.#after(ALL_KEYS, () => this.#storage.scanTasks(query, limit, cursor, context)); + } + + submission(id: SubmissionId, context: Context): Promise { + return this.#after(submissionKey(id), () => this.#storage.submission(id, context)); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(ALL_KEYS, () => this.#storage.scanSubmissions(query, limit, cursor, context)); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + return this.#after(requestKey(conversationId, requestId), () => + this.#storage.submissionByRequest(conversationId, requestId, context), + ); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#after(documentKey(addressId(address)), () => this.#storage.findDocument(address, at, context)); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + return this.#after(incarnationKey(id), () => this.#storage.document(id, at, context)); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(scopeKey(query.scope), () => this.#storage.scanDocuments(query, limit, cursor, context)); + } + + close(context: Context): Promise { + return this.#storage.close(context); + } + + /** + * Run a read of a conversation's entries once neither its appends nor its creation has an unpublished writer: Storage + * refuses the read of a conversation it does not have yet. + */ + #afterEntries(conversationId: ConversationId, read: () => Promise): Promise { + return this.#after(conversationKey(conversationId), () => this.#after(appendKey(conversationId), read)); + } + + /** Run `read` once `key` has no unpublished writer. */ + #after(key: string, read: () => Promise): Promise { + const gate = this.#touch(key); + return gate === undefined ? read() : gate.then(read); + } +} diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 4a6563b70..135855515 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,6 +1,7 @@ -import { type Context, type JsonValue, Watch } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchEnd } from "@earendil-works/chord"; import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; -import { type Op, track } from "@earendil-works/chord/delta"; +import { type Op, PendingOps, track } from "@earendil-works/chord/delta"; +import type { SpanAttributes } from "@earendil-works/pi-telemetry"; import { type AnyDocToken, addressId, @@ -14,7 +15,12 @@ import { idFromNumber } from "../ids.ts"; import { hostTimers, type Timers } from "../timers.ts"; import type { CommitChange, + CommitObservation, + CommitObserver, + CommitOptions, + CommitPersistence, CommitPublication, + CommitResult, ConversationDocFamilyToken, ConversationDocToken, ConversationId, @@ -33,6 +39,8 @@ import type { EntryRecord, Id, JsonObject, + Observation, + Observer, Page, ReportInfo, RewindableConversationDocFamilyToken, @@ -56,13 +64,75 @@ import type { TaskRecord, Tx, } from "../types.ts"; -import { type LoadedDocument, Transaction, type TransactionHost, type TransactionScope } from "./transaction.ts"; +import { CachedStorage, scopeOf } from "./cached-storage.ts"; +import { ConversationTails, type HeadMarker } from "./conversation-tails.ts"; +import { ALL_KEYS, appendKey, conversationKey, documentKey, GatedStorage, incarnationKey } from "./keys.ts"; +import { + type AdoptedDocuments, + type HeldDocument, + type LoadedDocument, + storeHeld, + Transaction, + type TransactionHost, + type TransactionScope, +} from "./transaction.ts"; +import { type PublishFailure, Writer } from "./writer.ts"; -/** What a document watch observes: the committed value of its incarnation, `null` once it retired. */ -type ObservedDocumentValue = Readonly | null; +/** + * What the Session's own components observe of one commit (`followCommits()`): a stored commit with its own changes, or + * a deferred one adopted in memory only (`CommitPersistence`), which changes documents only. Memory is never behind: + * a later store of held changes is not observed again. + */ +export type ObservedCommit = + | { readonly kind: "stored"; readonly seq: Seq; readonly changes: readonly CommitChange[] } + | { readonly kind: "memory"; readonly changes: readonly DocumentCommitChange[] }; + +/** A held document (`HeldDocument`) as the Session keeps it: pinned in the cache until stored. */ +type Hold = HeldDocument & { + /** When the earliest of its held commits must be stored, on the Session clock. */ + deadline: number; + readonly unpin: () => void; +}; + +/** The armed store of held documents: a foreground timer for the earliest deadline, `fired` once its job is queued. */ +type HoldTimer = { readonly at: number; disarm: () => void; fired: boolean }; + +const NO_HOLDS: readonly Hold[] = []; + +const NO_OPERATIONS: readonly Op[] = []; + +/** A line job's result: its value, delivered once `after`, the publication of its commit, settles, if it has one. */ +type Ran = { readonly value: T; readonly after?: Promise }; + +/** A job on the line: its index in line order, and whether a touch of it waited for an unpublished writer. */ +type Job = { + readonly index: number; + hit: boolean; + /** Called once its outcome settles: its commit published, or never to be (`onJobEnd()`). */ + ends?: (() => void)[]; +}; + +const NO_KEYS: readonly string[] = []; + +/** + * Jobs admitted to the line whose outcome is not delivered yet, beyond which the line waits. A count, not bytes: what + * waits is bounded by what one commit can hold (tool output by its output limits), times the count. + */ +const MAX_OUTSTANDING = 256; + +const ignore = (): void => {}; + +/** + * Queues a commit's host dispatch as one microtask through `then`: in the same queue as `queueMicrotask()`, which Node + * wraps in an async resource that costs about twice the memory and three times the time per stored commit. + */ +const SETTLED: Promise = Promise.resolve(); /** The default source of a report: a listener, which never fails what ran it. */ -const LISTENER: ReportInfo = { type: "listener_error" }; +const LISTENER: ReportInfo = { kind: "listener_error" }; + +/** A telemetry adapter that threw on a commit's event, which must not fail the durable commit. */ +const TELEMETRY: ReportInfo = { kind: "host_error", callback: "telemetry" }; /** * Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. @@ -71,14 +141,23 @@ const LISTENER: ReportInfo = { type: "listener_error" }; */ export function createSession( storage: Storage, - options?: { readonly now?: () => number; readonly documentRetentionMs?: number; readonly setTimer?: SetTimer }, + options?: { + readonly now?: () => number; + readonly documentRetentionMs?: number; + readonly setTimer?: SetTimer; + /** Default `withinMs` of a deferred commit (`CommitPersistence`); default 1,000. */ + readonly persistWithinMs?: number; + }, ): Session { const retention = options?.documentRetentionMs; + const within = options?.persistWithinMs; return new SessionImpl( storage, options?.now, retention === undefined ? undefined : () => retention, options?.setTimer, + false, + () => within, ); } @@ -98,6 +177,17 @@ export function documentRetention(ms: number | undefined): number { return typeof ms === "number" && ms >= 0 ? ms : defaultRetention; } +/** Default `withinMs` of a deferred commit, `settings.progress.persistWithinMs`. */ +const DEFAULT_PERSIST_WITHIN_MS = 1_000; + +/** + * A deferred commit's default window as given, when it is a number of milliseconds, `0` or more (`Infinity` waits for a + * reading or writing commit or close); otherwise the default. + */ +export function persistWithin(ms: number | undefined): number { + return typeof ms === "number" && ms >= 0 ? ms : DEFAULT_PERSIST_WITHIN_MS; +} + /** * Session kernel: one mutation line, the loaded document tracker cache, and committed publication. * @@ -105,8 +195,19 @@ export function documentRetention(ms: number | undefined): number { * written for the retention period, and the next access loads it from Storage. Eviction runs as a job on the mutation * line, so no transaction ever holds a tracker that leaves. * - * Only committed state is observable. Every commit callback, preparation, Storage settlement, adoption, and - * publication enqueue runs while the line is held; listeners run later. + * Only committed state is observable. A deferred commit (`CommitPersistence`) is committed in memory first: it is + * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, + * for the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, + * which store them. + * + * Group commit: the line runs one job at a time, a commit's callback and settlement; a commit with writes then joins + * the writer's pending batch (logical admission, `Writer`) and the line goes on. The writer stores every commit admitted + * while no call is in flight in one `commit()` call, off the line, and publishes them one by one, in line order, a + * macrotask apart: the read caches, adoption, telemetry events, the Session's own observers, and one microtask later + * `observeCommits()` observers, before every wait the commit resolves and every abort it queues. Host listeners and + * watch listeners have no order. A deferred commit is published when the commits before it are. A job's every read + * touches its key (`touch()`); while an unpublished commit writes that key, the job waits for its publication, so no + * job observes state that is not stored. Outcomes are delivered in line order (`#enqueue()`). * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later @@ -115,7 +216,14 @@ export function documentRetention(ms: number | undefined): number { * Storage's to retry. Reopening recovers from what was committed. */ export class SessionImpl implements Session { - readonly #storage: Storage; + readonly #storage: GuardedStorage; + /** Newest entry and head marker per conversation; none for a read-only Session, whose Storage another process writes. */ + readonly #tails: ConversationTails | undefined; + /** + * The current documents of each live task this Session created, so the commit that ends a task retires them without + * a scan. An entry starts with its task's creation and goes when the task is terminal: bounded by live tasks. + */ + readonly #taskDocuments = new Map>(); readonly #documents = new Map(); /** Cached documents with no observer attached, by address, oldest use first, with the time of that use. */ readonly #idleDocuments = new Map(); @@ -123,6 +231,10 @@ export class SessionImpl implements Session { readonly #attachedDocuments = new Map(); readonly #clock: () => number; readonly #documentRetentionMs: () => number; + readonly #persistWithinMs: () => number; + /** Documents with held changes, by incarnation (record ID); each is pinned in the cache until they are stored. */ + readonly #holds = new Map(); + #holdTimer: HoldTimer | undefined; /** Refuse every commit with writes (`HarnessOptions.readOnly`). */ readonly #readOnly: boolean; readonly #timers: Timers; @@ -131,12 +243,33 @@ export class SessionImpl implements Session { #sweepJobs = 0; /** Set once the host declined a background timer: then none is kept, and the end of each commit evicts what expired. */ #documentTimersUnavailable = false; - readonly #commitListeners = new Set<(publication: CommitPublication, context: Context) => void>(); + /** Host commit observers (`observeCommits()`), each behind `guardObserver()`. */ + readonly #commitListeners = new Set>(); /** The Session's own listeners, such as its scheduler's: they keep memory in step with storage, so a throw fails it. */ - readonly #internalListeners = new Set<(publication: CommitPublication, context: Context) => void>(); + readonly #internalListeners = new Set<(publication: ObservedCommit, context: Context) => void>(); readonly #closeListeners = new Set<() => void>(); readonly #host: TransactionHost; + /** The record cache under the guard, which the writer applies each batch to at its publication. */ + readonly #cached: CachedStorage | undefined; + readonly #writer: Writer; + /** The Storage jobs on the line read through: each read waits while its key has an unpublished writer. */ + readonly #gated: GatedStorage; + /** The end of the last line job: the line runs one job at a time. */ #tail: Promise = Promise.resolve(); + /** Line index of the next job; outcomes are delivered in this order. */ + #nextIndex = 0; + /** The job on the line, while one runs: its index, and whether it waited at a gate. */ + #current: Job | undefined; + /** Settles once every job so far has its outcome delivered. */ + #fence: Promise = Promise.resolve(); + #lastDelivered = -1; + /** Jobs that started on the line and have no outcome delivered yet (`MAX_OUTSTANDING`). */ + #outstanding = 0; + #room: PromiseWithResolvers | undefined; + /** The line index whose outcome the failure is: every later job's outcome is `SessionFailed`, except `#ownOutcomes`. */ + #failAt: number | undefined; + /** Later jobs whose outcome is their own error all the same: the other parts of a storage call that rejected. */ + readonly #ownOutcomes = new Set(); #closing: Promise | undefined; #failure: { readonly error: unknown } | undefined; /** Set when close reaches the backend: calls that fail from then on fail because it closes. */ @@ -163,12 +296,28 @@ export class SessionImpl implements Session { documentRetentionMs: () => number | undefined = () => undefined, setTimer?: SetTimer, readOnly = false, + persistWithinMs: () => number | undefined = () => undefined, ) { // Waits race it; none has to. this.#failed.promise.catch(() => {}); this.#readOnly = readOnly; - this.#storage = new GuardedStorage(storage, this); - this.#timers = hostTimers(setTimer, (error) => this.report(error, { type: "host_error", callback: "setTimer" })); + // A read-only Session shows a Storage another process writes, so nothing it reads stays current. + this.#cached = readOnly ? undefined : new CachedStorage(storage); + this.#storage = new GuardedStorage(this.#cached ?? storage, this); + this.#gated = new GatedStorage(this.#storage, (key) => this.touch(key)); + this.#writer = new Writer({ + commit: (batches, receive, context) => this.#storage.commit(batches, receive, context), + stored: (writes) => { + this.#cached?.published(writes); + this.#tails?.apply(writes); + }, + publishing: (publish, context) => this.#publishing(publish, context), + failure: () => this.#failure, + fail: (error, at, own) => this.fail(error, undefined, at, own), + report: (error, context) => this.report(error, LISTENER, context), + }); + this.#tails = readOnly ? undefined : new ConversationTails(); + this.#timers = hostTimers(setTimer, (error) => this.report(error, { kind: "host_error", callback: "setTimer" })); // Cache bookkeeping must never change the outcome of the operation it follows: a clock or a retention setting // that throws counts as now, and as the default. this.#clock = () => { @@ -185,8 +334,16 @@ export class SessionImpl implements Session { return documentRetention(undefined); } }; + this.#persistWithinMs = () => { + try { + return persistWithin(persistWithinMs()); + } catch { + return persistWithin(undefined); + } + }; this.#host = { - storage: this.#storage, + storage: this.#gated, + touch: (key) => this.touch(key), now, cached: (id) => { const cached = this.#documents.get(id); @@ -202,6 +359,9 @@ export class SessionImpl implements Session { if (this.#documents.get(id)?.record.id === recordId) this.#dropDocument(id); }, conversationCreated: (tx, record) => this.conversationCreated(tx, record), + taskDocuments: (taskId) => this.#taskDocuments.get(taskId)?.values(), + latestEntryId: (conversationId, context) => this.latestEntryId(conversationId, context), + latestHeadMarker: (conversationId, context) => this.latestHeadMarker(conversationId, undefined, context), }; } @@ -210,23 +370,90 @@ export class SessionImpl implements Session { return this.#storage; } - commit(change: (tx: Tx) => T | Promise, context: Context): Promise { - return this.commitWith(change, context); + /** + * Internal, on the Session line: the newest entry ID visible from `conversationId`. Known from the Session's own + * commits when it owns its Storage, so it costs no Storage read after the first. + */ + latestEntryId(conversationId: ConversationId, context: Context): Promise { + const gate = this.#touchEntries(conversationId); + const latest = (): Promise => { + const read = async () => + (await this.#storage.scanEntries({ conversationId }, 1, undefined, context)).items[0]?.id; + return this.#tails === undefined ? read() : this.#tails.latest(conversationId, read); + }; + return gate === undefined ? latest() : gate.then(latest); + } + + /** + * Internal: the newest entry ID of `conversationId` as known from the Session's commits right now, without reading; + * `undefined` when not known (and always for a read-only Session). + */ + knownLatestEntryId(conversationId: ConversationId): EntryId | null | undefined { + return this.#tails?.peek(conversationId); + } + + /** + * Internal, on the Session line: the newest visible head marker of `conversationId` at or before `atOrBefore`, any + * when `undefined`; known from the Session's own commits as `latestEntryId()` is. + */ + latestHeadMarker( + conversationId: ConversationId, + atOrBefore: EntryId | undefined, + context: Context, + ): Promise { + const gate = this.#touchEntries(conversationId); + const marker = (): Promise => { + const read = () => this.#storage.findLatestHeadMarker(conversationId, atOrBefore, context); + return this.#tails === undefined ? read() : this.#tails.marker(conversationId, atOrBefore, read); + }; + return gate === undefined ? marker() : gate.then(marker); + } + + /** + * On the line: wait for what an unpublished commit wrote of a conversation's entries, its appends and its creation, + * as reading them from Storage fails for a conversation not stored yet. + */ + #touchEntries(conversationId: ConversationId): Promise | undefined { + const created = this.touch(conversationKey(conversationId)); + const appended = this.touch(appendKey(conversationId)); + if (created === undefined) return appended; + return appended === undefined ? created : Promise.all([created, appended]).then(ignore); + } + + /** + * Internal, on the line: wait, while `key` has an unpublished writer (for `ALL_KEYS`, while anything is unpublished), + * until it is published, then check that the Session is healthy; `undefined` when there is nothing to wait for. A + * job's every read touches its key first, so it never observes state an earlier job admitted and the writer has not + * published. + */ + touch(key: string): Promise | undefined { + const gate = key === ALL_KEYS ? this.#writer.drained() : this.#writer.gate(key); + if (gate === undefined) return undefined; + if (this.#current !== undefined) this.#current.hit = true; + return gate.then(() => this.#assertHealthy()); + } + + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise { + return this.commitWith(change, context, undefined, options); } /** * Internal: fail the Session with `error`, the first wins, and report it once. Admission ends, and close listeners run * now; they read the error from `failure`. The Storage guard calls this; the Harness also calls it for a throw in its - * scheduler's own commits. + * scheduler's own commits. `at` is the line index of the job whose outcome the failure is, by default the job on the + * line: later jobs' outcomes become `SessionFailed`. */ - fail(error: unknown, context?: Context): void { + fail(error: unknown, context?: Context, at?: number, own?: readonly number[]): void { if (this.#failure !== undefined) return; this.#failure = { error }; + // The job that caused it keeps its own outcome; without one, every outcome not delivered yet is `SessionFailed`. + this.#failAt = at ?? this.#current?.index ?? this.#lastDelivered; + if (own !== undefined) for (const index of own) this.#ownOutcomes.add(index); this.#failed.reject(new SessionFailed(error)); // Seal first, so a report handler that calls back finds the Session failed. Closing runs the close listeners // synchronously; a failing backend close is the caller's to see, and here nobody waits. this.close(BACKGROUND_CONTEXT).catch(() => {}); - this.report(error, { type: "session_failed" }, context); + this.report(error, { kind: "storage_failed" }, context); } /** @@ -234,17 +461,79 @@ export class SessionImpl implements Session { * bootstrap and task replacement. `scope` sets the default `tx.createTask()` conversation and the task attributed to * appended entries. */ - commitWith(change: (tx: Transaction) => T | Promise, context: Context, scope?: TransactionScope): Promise { + commitWith( + change: (tx: Transaction) => T | Promise, + context: Context, + scope?: TransactionScope, + options?: CommitOptions, + ): Promise { + let withinMs: number | undefined; + try { + this.assertUsable(); + withinMs = this.#deferral(options?.persistence); + } catch (error) { + return Promise.reject(error); + } + return this.#enqueue((index) => this.#runCommit(change, context, scope, withinMs, index)); + } + + /** + * Internal: `commitWith()` for a commit of the scheduler's own whose outcome goes only to scheduler code. When it + * writes nothing and waited at no gate, it is equivalent to having run before every unpublished commit, so its + * outcome is delivered at once, ahead of theirs. One that writes is never exempt: its outcome waits for its own + * publication and every earlier outcome, so no invocation starts on a reservation that is not durable. + */ + commitInternal(change: (tx: Transaction) => T | Promise, context: Context): Promise { try { this.assertUsable(); } catch (error) { return Promise.reject(error); } - return this.#enqueue(() => this.#runCommit(change, context, scope)); + return this.#enqueue((index) => this.#runCommit(change, context, undefined, undefined, index), true); } - /** Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. */ - readOnLine(job: () => Promise): Promise { + /** + * Internal, on the line: call `end` once the outcome of the job on the line settles, its commit published or never to + * be; at once off the line. + */ + onJobEnd(end: () => void): void { + const job = this.#current; + if (job === undefined) { + end(); + return; + } + if (job.ends === undefined) job.ends = [end]; + else job.ends.push(end); + } + + /** Internal: the Storage jobs on the line read through (`touch()`); off the line, use the plain one. */ + get lineStorage(): Storage { + return this.#gated; + } + + /** How long a commit may stay in memory before it is stored; `undefined` when it is immediate. */ + #deferral(persistence: CommitPersistence | undefined): number | undefined { + if (persistence?.kind !== "deferred") return undefined; + const ms = persistence.withinMs; + if (ms === undefined) return this.#persistWithinMs(); + // Public options: checked here, as the types cannot. + if (typeof ms !== "number" || !(ms >= 0)) { + throw new TypeError("CommitPersistence.withinMs must be a number of milliseconds, 0 or more"); + } + return ms; + } + + /** Internal: whether `tx` is the transaction of one of this Session's commits. */ + ownsTransaction(tx: Tx): tx is Transaction { + return Transaction.ownedBy(tx, this.#host); + } + + /** + * Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. It reads + * through `storage`, whose reads wait for the publication of what the line admitted before; its outcome comes after + * theirs. + */ + readOnLine(job: (storage: Storage) => Promise): Promise { try { this.assertUsable(); } catch (error) { @@ -252,7 +541,7 @@ export class SessionImpl implements Session { } return this.#enqueue(async () => { this.#assertHealthy(); - return job(); + return { value: await job(this.#gated) }; }); } @@ -314,7 +603,7 @@ export class SessionImpl implements Session { ? cached : await this.#enqueue(async () => { this.#assertHealthy(); - return this.#loadDocument(definition, resolved.id, resolved.address, context); + return { value: await this.#loadDocument(definition, resolved.id, resolved.address, context) }; }); if (loaded === undefined) return undefined; checkRecordScope(definition, loaded.record); @@ -350,35 +639,92 @@ export class SessionImpl implements Session { key: string, context: Context, ): Promise | undefined>; - async watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { - this.assertUsable(); - const definition = token.definition; - const resolved = resolveAddress(definition, args); - const context = args[resolved.nextArgument] as Context; - const signal = context.abortSignal; - let cancelled = signal?.aborted ?? false; - const markCancelled = (): void => { - cancelled = true; - }; - signal?.addEventListener("abort", markCancelled, { once: true }); + watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { + // Not async, so an attachment costs one async frame, `#attachDocument`'s; a throw here still rejects. try { - const watch = await this.#enqueue(async () => { - this.#assertHealthy(); - if (cancelled) throw cancellationError(signal!); - const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); - if (cancelled) throw cancellationError(signal!); - if (loaded === undefined) return undefined; - return this.#attachDocument(definition, loaded); - }); - if (watch === undefined) return undefined; - if (cancelled) { - watch.cancel(); - throw cancellationError(signal!); - } - if (signal !== undefined) watch.observeCancellation(signal); - return watch as DocumentWatch; - } finally { - signal?.removeEventListener("abort", markCancelled); + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const context = args[resolved.nextArgument] as Context; + const signal = context.abortSignal; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const watch = this.#observeDocument(definition, loaded, createWatch, endWatch); + if (signal !== undefined) watch.observeAbort(signal); + return watch; + }, + abortWatch, + ); + } catch (error) { + return Promise.reject(error); + } + } + + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise> | undefined> { + // Not async, as `watchDoc()`; a throw here still rejects. + try { + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const observer = args[resolved.nextArgument] as Observer>; + const context = args[resolved.nextArgument + 1] as Context; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const guarded = this.#observeDocument( + definition, + loaded, + (_value, detach, report) => guardObserver(observer, detach, report), + endGuarded, + ); + return observation(loaded.tracker.value, guarded); + }, + detachObservation, + ); + } catch (error) { + return Promise.reject(error); } } @@ -411,7 +757,7 @@ export class SessionImpl implements Session { const context = args[resolved.nextArgument + 1] as Context; return this.#enqueue(async () => { this.#assertHealthy(); - const storedEntry = await this.#storage.entry(conversationId, at, context); + const storedEntry = await this.#gated.entry(conversationId, at, context); if (storedEntry === undefined) { throw new Error(`Entry ${at} is not visible from conversation ${conversationId}`); } @@ -419,13 +765,13 @@ export class SessionImpl implements Session { ...resolved.address, scope: { kind: "conversation", conversationId: storedEntry.entry.conversationId }, }; - const record = await this.#storage.findDocument(address, storedEntry.commitSeq, context); - if (record === undefined) return undefined; - const stored = await this.#storage.document(record.id, storedEntry.commitSeq, context); + const record = await this.#gated.findDocument(address, storedEntry.commitSeq, context); + if (record === undefined) return { value: undefined }; + const stored = await this.#gated.document(record.id, storedEntry.commitSeq, context); if (stored === undefined) { throw new Error(`Historical document ${record.id} (${record.kind}) cannot be read`); } - return materializeDocument(definition, stored); + return { value: materializeDocument(definition, stored) }; }); } @@ -436,28 +782,46 @@ export class SessionImpl implements Session { this.#closing = Promise.resolve() .then(() => this.beforeClose()) .then(() => - this.#enqueue(async () => { + this.#maintain(async () => { + await this.#drained(); + // Last, after the writes at close (`writeAtClose()`), while the trackers are still here. A failed Session + // writes nothing; a failing store fails it, and the backend still closes. + let unstored: { readonly error: unknown } | undefined; + // Only with something to store: otherwise the backend's close begins in this very job. + if (this.#failure === undefined && this.#holds.size > 0) { + await this.#storeHolds(() => [...this.#holds.values()], cleanup).catch((error: unknown) => { + unstored = { error }; + }); + } + const observers = [...this.#commitListeners]; this.#commitListeners.clear(); + // After the last publication: the commits admitted before the seal have settled, held changes are stored. + // Commit observers are synchronous observers, all of them. + const end = closeEnds(this.#failure).observer; + for (const observer of observers) observer.end(end); this.#internalListeners.clear(); this.#documents.clear(); this.#idleDocuments.clear(); this.#attachedDocuments.clear(); + this.#holds.clear(); + this.#taskDocuments.clear(); try { await this.#storage.close(cleanup); } catch (error) { // A Storage that cannot close is a failed one; an earlier failure stays the cause. if (this.#failure === undefined) { this.#failure = { error }; - this.report(error, { type: "session_failed" }); + this.report(error, { kind: "storage_failed" }); } throw error; } + if (unstored !== undefined) throw unstored.error; }), ) .finally(() => { const failure = this.#failure; this.#closed.resolve( - failure === undefined ? { reason: "closed" } : { reason: "failed", error: failure.error }, + failure === undefined ? { kind: "closed" } : { kind: "failed", error: failure.error }, ); }); this.#notifyClose(); @@ -469,6 +833,8 @@ export class SessionImpl implements Session { #notifyClose(): void { this.#documentSweep?.disarm?.(); this.#documentSweep = undefined; + this.#holdTimer?.disarm(); + this.#holdTimer = undefined; const listeners = [...this.#closeListeners]; this.#closeListeners.clear(); for (const listener of listeners) contained(listener, (error) => this.report(error)); @@ -494,30 +860,61 @@ export class SessionImpl implements Session { * writes nothing. */ protected writeAtClose(writes: () => readonly StorageWrite[]): Promise { - return this.#enqueue(async () => { + return this.#maintain(async (index) => { + await this.#drained(); if (this.#failure !== undefined) return; const batch = writes(); - if (batch.length > 0) await this.#storage.commit(batch, () => {}, BACKGROUND_CONTEXT); + if (batch.length === 0) return; + const context = BACKGROUND_CONTEXT; + await this.#writer.admit({ + index, + writes: batch, + keys: [], + context, + publish: () => undefined, + discard: ignore, + }).outcome; }); } + /** + * Keys a commit writes besides its records' (`Transaction.writeKeys()`), at its admission, on the line: a Harness + * adds the ownership memberships its task changes write. A plain Session adds none. + */ + protected membershipKeys(_tx: Transaction): readonly string[] | Promise { + return NO_KEYS; + } + /** Runs after close seals admission and before the line closes Storage; must not reject. */ protected beforeClose(): Promise { return Promise.resolve(); } - /** Register a synchronous post-adoption listener. It must not block or call Session operations; a throw is reported. */ - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { + /** + * Attach a synchronous observer of every stored commit, with exactly what it stored (`CommitPublication`). Its + * `advance` runs one microtask after the commit's publication, which follows its storage call, in line order, before + * every wait the commit resolves and every abort it queues (in no order with watch listeners); it must not block or + * call Session operations. A throw is reported, detaches it, and ends it `listener_error`; `end` gets the Session's + * close after its last publication. + */ + observeCommits(observer: CommitObserver): CommitObservation { this.assertUsable(); - this.#commitListeners.add(listener); - return () => this.#commitListeners.delete(listener); + const guarded = guardObserver( + observer, + () => this.#commitListeners.delete(guarded), + (error, context) => this.report(error, LISTENER, context), + ); + this.#commitListeners.add(guarded); + return { detach: () => guarded.detach() }; } /** - * Internal: `subscribeCommits()` for the Session's own components, whose state follows each commit. A throw leaves - * that state behind storage, so it fails the Session instead of being reported. They run before host listeners. + * Internal: follow every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state + * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. They run at + * the commit's publication, synchronously, so they call no Storage method, run no task code, and contain every host + * callback; the waits they resolve and the aborts they queue run after host observers. */ - observeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { + followCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { this.assertUsable(); this.#internalListeners.add(listener); return () => this.#internalListeners.delete(listener); @@ -533,29 +930,49 @@ export class SessionImpl implements Session { return () => this.#closeListeners.delete(listener); } - /** Drop every loaded tracker on the mutation line; later access cold-loads from Storage. */ + /** Drop every loaded tracker on the mutation line, held changes stored first; later access cold-loads from Storage. */ unloadDocuments(): Promise { - return this.#enqueue(async () => { + return this.#maintain(async () => { + if (this.#failure === undefined) await this.#storeHolds(() => [...this.#holds.values()], BACKGROUND_CONTEXT); this.#documents.clear(); this.#idleDocuments.clear(); }); } /** - * Internal, for tests: how many documents the cache holds, how many are idle, how many incarnations are pinned, and how - * many sweep jobs were queued on the line so far. + * Internal, for tests: how many documents the cache holds, how many are idle, how many incarnations are pinned, how + * many sweep jobs were queued on the line so far, how many documents have held changes, how many live tasks have + * their documents indexed, and how many conversations have their newest entry kept. */ get documentCacheSizes(): { readonly cached: number; readonly idle: number; readonly attached: number; readonly sweeps: number; + readonly held: number; + readonly taskDocuments: number; + readonly tails: number; } { return { cached: this.#documents.size, idle: this.#idleDocuments.size, attached: this.#attachedDocuments.size, sweeps: this.#sweepJobs, + held: this.#holds.size, + taskDocuments: this.#taskDocuments.size, + tails: this.#tails?.size ?? 0, + }; + } + + /** + * Internal, for tests that bound memory: jobs whose outcome is not delivered yet, keys with an unpublished writer, + * and record reads in flight in the cache. All three are 0 once the line is idle. + */ + get lineSizes(): { readonly outstanding: number; readonly unpublished: number; readonly reads: number } { + return { + outstanding: this.#outstanding, + unpublished: this.#writer.unpublished, + reads: this.#cached?.readsInFlight ?? 0, }; } @@ -588,16 +1005,22 @@ export class SessionImpl implements Session { */ #scheduleDocumentSweep(): void { if (this.#closing !== undefined) return; - const oldest = this.#idleDocuments.values().next(); - if (oldest.done === true) return; - const at = oldest.value + this.#documentRetentionMs(); + // Documents with unpublished changes are not swept: their publication touches them again. + let oldest: number | undefined; + for (const [addressId, since] of this.#idleDocuments) { + if (this.#writer.gate(documentKey(addressId)) !== undefined) continue; + oldest = since; + break; + } + if (oldest === undefined) return; + const at = oldest + this.#documentRetentionMs(); if (this.#documentSweep !== undefined && this.#documentSweep.at <= at) return; this.#documentSweep?.disarm?.(); // The pending sweep stays set until its job runs, so touches while the line is busy queue no more. const sweep: { readonly at: number; disarm: (() => void) | undefined } = { at, disarm: undefined }; const run = (): void => { this.#sweepJobs++; - this.#enqueue(async () => { + this.#maintain(async () => { if (this.#documentSweep === sweep) this.#documentSweep = undefined; if (this.#closing === undefined) this.#sweepDocuments(); }).catch(() => {}); @@ -627,6 +1050,8 @@ export class SessionImpl implements Session { const expired = this.#clock() - this.#documentRetentionMs(); for (const [addressId, since] of this.#idleDocuments) { if (since > expired) break; + // A document with an unpublished change stays until its publication touches it. + if (this.#writer.gate(documentKey(addressId)) !== undefined) continue; this.#dropDocument(addressId); } this.#scheduleDocumentSweep(); @@ -653,8 +1078,10 @@ export class SessionImpl implements Session { async #runCommit( change: (tx: Transaction) => T | Promise, context: Context, - scope?: TransactionScope, - ): Promise { + scope: TransactionScope | undefined, + withinMs: number | undefined, + index: number, + ): Promise> { this.#assertHealthy(); context.abortSignal?.throwIfAborted(); const tx = new Transaction(this.#host, context, scope); @@ -679,7 +1106,7 @@ export class SessionImpl implements Session { if (writes.length === 0) { tx.discard(); this.#sweepDocuments(); - return result; + return { value: result }; } if (this.#readOnly) { // Refused before Storage: the Session stays healthy, and the caller sees why. @@ -687,66 +1114,243 @@ export class SessionImpl implements Session { this.#sweepDocuments(); throw new ReadOnlySession("a commit with writes"); } - // Storage adopts the batch synchronously once it is durable, before its next commit; a throw here would reach - // Storage, so the callback keeps its own error. - let adoption: { readonly documents: DocumentCommitChange[] } | { readonly error: unknown } | undefined; - const adopt = (committed: Seq): void => { - try { - adoption = { documents: tx.adopt(committed) }; - } catch (error) { - adoption = { error }; + const discard = (): void => tx.discard(); + if (withinMs !== undefined) { + const held = tx.holdable(); + if (held !== undefined) { + // Adopted in memory when the commits before it are published, as a part of no storage call. + const part = this.#writer.admit({ + index, + writes: undefined, + keys: tx.writeKeys(), + context, + publish: () => this.#hold(tx, held, withinMs, context), + discard, + }); + this.#sweepDocuments(); + return { value: result, after: part.outcome }; } - }; - let seq: Seq; - try { - // Once admitted, caller cancellation does not interrupt Storage settlement. - seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); - } catch (error) { - // The guard has failed the Session. A rejection after adoption leaves nothing staged to discard. - if (adoption === undefined) tx.discard(); - throw error; } - if (adoption === undefined) { - const error = new Error("Storage committed without adopting the batch"); - this.fail(error); - throw error; + let membership = this.membershipKeys(tx); + if (!Array.isArray(membership)) { + try { + membership = await membership; + } catch (error) { + tx.discard(); + this.#sweepDocuments(); + throw error; + } } - if ("error" in adoption) { - // Storage already committed; a failed adoption leaves memory behind durable state. - this.fail(adoption.error); - throw adoption.error; - } - const documents = adoption.documents; - for (const document of documents) this.#touchDocument(addressId(document.record)); - this.#publish(seq, writes, documents, context); + // Logical admission. Held changes leave their queue only here, once nothing but Storage can fail; a commit never + // stores a hold adopted after it in line order. Taken before the keys, so the documents it stores held changes of + // are its writes too: a later job that reads one waits for it instead of storing them again. + const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); + const keys = tx.writeKeys(); + for (const key of membership as readonly string[]) keys.push(key); + const part = this.#writer.admit({ + index, + writes, + keys, + context, + publish: (seq) => this.#adopt(tx, writes, taken, seq!, context), + discard, + }); this.#sweepDocuments(); - return result; + return { value: result, after: part.outcome }; } + /** + * Publish a stored commit (`Writer`), after the read caches took its writes: adopt it in memory and run its + * publication. A throw of the adoption leaves memory behind durable state; the writer fails the Session with it. + */ + #adopt( + tx: Transaction, + writes: readonly StorageWrite[], + taken: readonly Hold[], + seq: Seq, + context: Context, + ): PublishFailure | undefined { + let documents: AdoptedDocuments; + try { + documents = tx.adopt(seq); + } catch (error) { + return { kind: "adoption", error }; + } + this.#indexTaskDocuments(tx, writes, documents, seq); + const failure = this.#publish(seq, writes, documents, context); + if (taken.length > 0) this.#release(taken); + for (const document of documents.stored) this.#touchDocument(addressId(document.record)); + return failure === undefined ? undefined : { kind: "listener", error: failure.error }; + } + + /** + * At publication: keep `#taskDocuments` in step with a stored commit: the tasks it creates start with no documents, + * task documents it creates join their task, ones it retires leave, and tasks it ends leave the index. + */ + #indexTaskDocuments(tx: Transaction, writes: readonly StorageWrite[], documents: AdoptedDocuments, seq: Seq): void { + for (const id of tx.createdTaskIds()) this.#taskDocuments.set(id, new Map()); + if (this.#taskDocuments.size === 0) return; + for (const change of documents.stored) { + if (change.type !== "document" || change.record.scope.kind !== "task") continue; + const known = this.#taskDocuments.get(change.record.scope.taskId); + if (known === undefined) continue; + if (change.kind === "retire") known.delete(change.record.id); + else if (change.record.createdAt === seq) known.set(change.record.id, change.record); + } + for (const write of writes) { + if (write.type === "task" && write.value.state.status === "terminal") + this.#taskDocuments.delete(write.value.id); + } + } + + /** + * Hold a deferred commit (`Transaction.holdable()`): adopt it in memory, queue its changes per document, pinned, by + * the earliest deadline of the document's held commits, and show it to the Session's own observers. + */ + #hold( + tx: Transaction, + held: NonNullable>, + withinMs: number, + context: Context, + ): PublishFailure | undefined { + let documents: DocumentCommitChange[]; + try { + documents = tx.adoptHeld(); + } catch (error) { + // As a stored commit's: memory could not adopt it; the writer fails the Session. + return { kind: "adoption", error }; + } + const deadline = this.#clock() + withinMs; + for (const { loaded, ops, base } of held) { + const id = loaded.record.id; + let hold = this.#holds.get(id); + if (hold === undefined) { + const unpin = this.#pinDocument(loaded.addressId, id); + hold = { loaded, pending: new PendingOps(), base: false, deadline, unpin }; + this.#holds.set(id, hold); + } + hold.pending.push(ops); + if (base) hold.base = true; + if (deadline < hold.deadline) hold.deadline = deadline; + } + this.#armHoldTimer(); + const publication: ObservedCommit = { kind: "memory", changes: documents }; + let failure: PublishFailure | undefined; + for (const listener of [...this.#internalListeners]) { + // Our own observers are synchronous. + try { + listener(publication, context); + } catch (error) { + failure ??= { kind: "listener", error }; + } + } + return failure; + } + + /** + * Store the held changes `select` returns in one storage transaction, through the writer, from a job on the line. A + * barrier: they are selected once every commit admitted before is published. Memory already has them, so only + * `observeCommits()` observers see it, published as the writer publishes it. A failure fails the Session, as every + * Storage failure does. + */ + async #storeHolds(select: () => readonly Hold[], context: Context): Promise { + await this.#drained(); + const holds = select(); + if (holds.length === 0) return; + const writes: Extract[] = []; + const changes: DocumentCommitChange[] = []; + const keys: string[] = []; + for (const hold of holds) { + const { record, tracker, valueVersion: version } = hold.loaded; + const value = tracker.value; + const stored = storeHeld(hold, NO_OPERATIONS, value, version); + writes.push({ type: "document.change", id: record.id, content: stored.content }); + keys.push(documentKey(hold.loaded.addressId), incarnationKey(record.id)); + // Held documents are conversation documents. + const conversationId = record.scope.kind === "conversation" ? record.scope.conversationId : undefined; + changes.push({ type: "document", kind: "change", record, conversationId, version, value, ops: stored.ops }); + } + const publish = (seq: Seq | undefined): undefined => { + for (const [index, hold] of holds.entries()) { + if (writes[index]!.content.kind === "base") hold.loaded.deltasSinceBase = 0; + else hold.loaded.deltasSinceBase++; + } + if (this.#commitListeners.size > 0) this.#notifyHosts({ seq: seq!, changes }, context); + this.#release(holds); + return undefined; + }; + // Run from a job on the line, which a failure it causes is the outcome of. + const index = this.#current!.index; + await this.#writer.admit({ index, writes, keys, context, publish, discard: ignore }).outcome; + } + + /** After their store: forget `holds`, unpin their documents, and move the timer to the next deadline. */ + #release(holds: readonly Hold[]): void { + for (const hold of holds) { + this.#holds.delete(hold.loaded.record.id); + hold.unpin(); + } + this.#armHoldTimer(); + } + + /** + * Keep a foreground timer (Workers refuse background ones) for the earliest deadline of the held documents. When it + * fires, every held document is stored, in one storage transaction: a deadline is the latest time, and the others + * cost little more in the same transaction. Once fired, it stays until its job ran. + */ + #armHoldTimer(): void { + if (this.#closing !== undefined) return; + let at = Infinity; + for (const hold of this.#holds.values()) if (hold.deadline < at) at = hold.deadline; + const armed = this.#holdTimer; + if (armed !== undefined && (armed.fired || armed.at === at)) return; + armed?.disarm(); + this.#holdTimer = undefined; + if (at === Infinity) return; + const timer: HoldTimer = { at, disarm: () => {}, fired: false }; + const run = (): void => { + timer.fired = true; + this.#maintain(async () => { + if (this.#holdTimer === timer) this.#holdTimer = undefined; + // Close stores what is held itself. + if (this.#closing !== undefined || this.#failure !== undefined) return; + await this.#storeHolds(() => [...this.#holds.values()], BACKGROUND_CONTEXT); + }).catch(() => {}); + }; + this.#holdTimer = timer; + timer.disarm = this.#timers.foreground(run, Math.max(0, at - this.#clock())); + } + + /** + * At publication: publish a stored commit. Its telemetry events first, then the dispatch to `observeCommits()` + * observers, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: + * returns the first error of an own observer, which fails the Session once its storage call settled. + */ #publish( seq: Seq, writes: readonly StorageWrite[], - documents: readonly DocumentCommitChange[], + documents: AdoptedDocuments, context: Context, - ): void { + ): { readonly error: unknown } | undefined { const changes: CommitChange[] | undefined = this.#commitListeners.size > 0 || this.#internalListeners.size > 0 ? [] : undefined; for (const write of writes) { switch (write.type) { case "entry": - context.telemetry.addEvent("pi.durable.entry", { - "entry.id": write.value.id, - "entry.kind": write.value.kind, - }); + this.#event("pi.durable.entry", { "entry.id": write.value.id, "entry.kind": write.value.kind }, context); changes?.push(write); break; case "submission": if (write.value.status === "done" || write.value.status === "unanswered") { - context.telemetry.addEvent("pi.durable.submission", { - "submission.id": write.value.id, - status: write.value.status, - "answer.entry_id": write.value.answer, - }); + this.#event( + "pi.durable.submission", + { + "submission.id": write.value.id, + status: write.value.status, + "answer.entry_id": write.value.answer, + }, + context, + ); } changes?.push(write); break; @@ -755,58 +1359,186 @@ export class SessionImpl implements Session { changes?.push(write); } } - if (changes === undefined) return; - for (const document of documents) changes.push(document); - const publication: CommitPublication = { seq, changes }; - // The commit is durable: a failing listener neither fails the commit nor skips the others. - for (const listener of [...this.#internalListeners]) { - contained( - () => listener(publication, context), - (error) => this.fail(error, context), - ); + if (changes === undefined) return undefined; + const tables = changes.length; + for (const document of documents.observed) changes.push(document); + const observed: ObservedCommit = { kind: "stored", seq, changes }; + if (this.#commitListeners.size > 0) { + // One publication for both, unless the commit also stored held changes, which only host listeners get. + const stored: CommitPublication = + documents.stored === documents.observed + ? observed + : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; + this.#notifyHosts(stored, context); } - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(publication, context), - (error) => this.report(error, LISTENER, context), - ); + // The commit is durable: a failing observer neither fails the commit nor skips the others. + let failure: { readonly error: unknown } | undefined; + for (const listener of [...this.#internalListeners]) { + // Our own observers are synchronous. + try { + listener(observed, context); + } catch (error) { + failure ??= { error }; + } + } + return failure; + } + + /** At publication: record a commit's telemetry event. A throwing adapter is reported, as publication must not throw. */ + #event(name: string, attributes: SpanAttributes, context: Context): void { + try { + context.telemetry.addEvent(name, attributes); + } catch (error) { + this.report(error, TELEMETRY, context); } } /** - * Attach a watch to one committed incarnation: check the definition, then forward this incarnation's committed - * changes, its retirement, and close. The watch's detach removes both subscriptions. + * At publication: call the `observeCommits()` observers with `publication` one microtask from now, and, queued + * before the Session's own observers run, before the waits they resolve and the aborts they queue. The observers are + * those attached when it runs; their guards report a throw. */ - #attachDocument(definition: AnyDocToken["definition"], loaded: LoadedDocument): Watch { + #notifyHosts(publication: CommitPublication, context: Context): void { + void SETTLED.then(() => { + for (const observer of [...this.#commitListeners]) observer.advance(publication, context); + }); + } + + /** + * On the line, as one job: load the document at `resolved` and `attach` to it, so the attachment sees every commit + * after the value it starts from; `undefined` when absent. Cancelling `context` meanwhile rejects, after `cancel` + * undid an attachment made. + */ + async #attachDocument( + definition: AnyDocToken["definition"], + resolved: { readonly id: string; readonly address: DocumentAddress }, + context: Context, + attach: (loaded: LoadedDocument) => A, + cancel: (attached: A) => void, + ): Promise { + const signal = context.abortSignal; + let cancelled = signal?.aborted ?? false; + const markCancelled = (): void => { + cancelled = true; + }; + signal?.addEventListener("abort", markCancelled, { once: true }); + try { + const attached = await this.#enqueue(async () => { + this.#assertHealthy(); + if (cancelled) throw cancellationError(signal!); + const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); + if (cancelled) throw cancellationError(signal!); + return { value: loaded === undefined ? undefined : attach(loaded) }; + }); + if (attached === undefined) return undefined; + if (cancelled) { + cancel(attached); + throw cancellationError(signal!); + } + return attached; + } finally { + signal?.removeEventListener("abort", markCancelled); + } + } + + /** + * Follow the address of the incarnation `loaded`, after checking the definition, with the observer `create` makes + * from its value, the idempotent detach, and the report of a listener error; returns that observer. It gets each + * committed change there, a retirement as a `retired` change with the last value it was given and no ops (a change + * in the retiring commit is not published), and a later incarnation as a root replacement. When the commit makes the + * owner task of a task document terminal, no later commit can change the address: after its changes the commit + * subscription and the pin go, and `ownerEnded` tells the observer, which stays subscribed to the Session's close + * until it ends (its detach drops that). `end` gets the Session's close. The current incarnation is pinned in the + * cache. The detach runs before `end`; once it ran, nothing more of the commit underway reaches the observer or pins + * anything for it. `create` and `ownerEnded` are module functions where they can be, so a watch keeps no closure of + * its caller. + */ + #observeDocument>>( + definition: AnyDocToken["definition"], + loaded: LoadedDocument, + create: (value: Readonly, detach: () => void, report: (error: unknown) => void) => O, + ownerEnded: (observer: O) => void, + ): O { checkRecordScope(definition, loaded.record); checkRecordVersion(definition, loaded.record, loaded.storedVersion); + // The closures below keep only the address, as the cache's ID and first record, which the cache shares while the + // document is cached: `loaded` holds its incarnation's tracker, which a retirement frees. + const addressId = loaded.addressId; + const address: DocumentAddress = loaded.record; + let incarnation: DocumentId | undefined = loaded.record.id; + // The value of the incarnation followed; none while the address is absent, so the Session keeps no retired value. + let value: Readonly | undefined = loaded.tracker.value; + // The definition version the observer's value has: a change of another version replaces it whole. + let version = loaded.valueVersion; + // The incarnation observed stays cached; once it retires or the observer detaches, it is idle again. + let unpin = this.#pinDocument(addressId, incarnation); let unsubscribeCommit = (): void => {}; let unsubscribeClose = (): void => {}; - // Attached, the incarnation stays cached; once its last observer detaches, or it retires, it is idle again. - const unpin = this.#pinDocument(loaded.addressId, loaded.record.id); + let attached = true; const detach = (): void => { + if (!attached) return; + attached = false; unsubscribeCommit(); unsubscribeClose(); unpin(); }; - const observer = new Watch(loaded.tracker.value, detach, (error) => this.report(error)); - const observed = { version: loaded.valueVersion }; - unsubscribeCommit = this.observeCommits((publication, context) => { + const observer = create(loaded.tracker.value, detach, (error) => this.report(error)); + unsubscribeCommit = this.followCommits((publication, context) => { + // Listeners run from a copy of the set: another observer's call may have detached this one. + if (!attached) return; + let ownerEnds = false; for (const change of publication.changes) { - if (change.type !== "document" || change.record.id !== loaded.record.id) continue; - if (change.value === null) { - // A retired incarnation never changes again: its watch delivers the retirement but pins nothing. - unpin(); - observer.retire(context); + if (change.type === "task") { + const scope = address.scope; + if ( + scope.kind === "task" && + change.value.id === scope.taskId && + change.value.state.status === "terminal" + ) { + ownerEnds = true; + } continue; } - const ops = observedOperations(observed, change); - // A migration-only base changes nothing for an observer of the new version. - if (ops.length === 0) continue; - observer.advance(change.value, ops, context); + // The incarnation followed is at the address by definition; only another record needs the address compared. + if (change.type !== "document") continue; + if (change.record.id !== incarnation && !atAddress(change.record, address)) continue; + if (change.kind === "retire") { + if (change.record.id !== incarnation) continue; + // Set with every incarnation. + const last = value!; + incarnation = undefined; + value = undefined; + unpin(); + observer.advance({ kind: "value", value: last, ops: NO_OPERATIONS, retired: true }, context); + } else if (change.record.id !== incarnation || change.version !== version) { + // A new incarnation at the address, or a value of another definition version, whose shape the observer's + // value does not have: shown whole. + if (change.record.id !== incarnation) { + incarnation = change.record.id; + unpin(); + unpin = this.#pinDocument(addressId, incarnation); + } + version = change.version; + value = change.value; + observer.advance({ kind: "value", value, ops: [["r", value]], retired: false }, context); + } else { + // A migration-only base changes nothing for an observer of the new version. + if (change.ops.length === 0) continue; + value = change.value; + observer.advance({ kind: "value", value, ops: change.ops, retired: false }, context); + } + // The observer may detach in its call, or throw and be detached: the rest of the commit is not its. + if (!attached) return; } + if (!ownerEnds) return; + unsubscribeCommit(); + unpin(); + ownerEnded(observer); + }); + unsubscribeClose = this.subscribeClose(() => { + detach(); + observer.end(closeEndOf(observer, closeEnds(this.#failure))); }); - unsubscribeClose = this.subscribeClose(() => observer.end(this.#failure)); return observer; } @@ -816,13 +1548,24 @@ export class SessionImpl implements Session { address: DocumentAddress, context: Context, ): Promise { + // What an unpublished commit wrote at the address is in neither the cache nor what Storage shows yet. + const gate = this.touch(documentKey(addressId)); + if (gate !== undefined) await gate; const cached = this.#documents.get(addressId); // A tracker serves only tokens of the version its value was materialized for; others reload from Storage. if (cached?.valueVersion === definition.version) { this.#touchDocument(addressId); return cached; } - if (cached !== undefined) this.#dropDocument(addressId); + if (cached !== undefined) { + // The reload must find what memory holds: held changes are stored before their tracker goes, after a barrier. + const id = cached.record.id; + await this.#storeHolds(() => { + const hold = this.#holds.get(id); + return hold === undefined ? NO_HOLDS : [hold]; + }, context); + this.#dropDocument(addressId); + } const record = await this.#storage.findDocument(address, "current", context); if (record === undefined) return undefined; const stored = await this.#storage.document(record.id, "current", context); @@ -841,13 +1584,150 @@ export class SessionImpl implements Session { return loaded; } - #enqueue(job: () => Promise): Promise { - const run = this.#tail.then(job); - this.#tail = run.then( - () => undefined, - () => undefined, + /** + * Run `job` on the line, after the jobs before it, and deliver its outcome in line order: once the jobs before it + * have theirs, and once its commit, if it admitted one, is published. With `exempt`, a job that admitted no commit and + * waited at no gate is delivered as soon as it has its outcome: it is equivalent to having run before every + * unpublished commit. A job + * whose outcome comes after the one a failure is gets `SessionFailed` instead. At most `MAX_OUTSTANDING` jobs wait + * for their delivery; beyond it the line waits. + */ + #enqueue(work: (index: number) => Promise>, exempt = false): Promise { + const job: Job = { index: this.#nextIndex++, hit: false }; + const ran = this.#tail.then(async () => { + while (this.#outstanding >= MAX_OUTSTANDING) { + this.#room ??= Promise.withResolvers(); + await this.#room.promise; + } + this.#outstanding++; + return this.#onLine(job, work); + }); + this.#tail = ran.then(ignore, ignore); + const outcome = ran.then( + (result) => { + if (result.after === undefined) { + this.#ended(job); + return result.value; + } + return result.after.then( + () => { + this.#ended(job); + return result.value; + }, + (error: unknown) => { + this.#ended(job); + throw error; + }, + ); + }, + (error: unknown) => { + this.#ended(job); + throw error; + }, ); - return run; + // Delivered later, after the outcomes before it: handled now, so an early rejection is not reported unhandled. + outcome.catch(ignore); + const fence = this.#fence; + this.#fence = fence.then(() => outcome).then(ignore, ignore); + const index = job.index; + const deliver = (fenced: boolean) => + (fenced ? fence.then(() => outcome) : outcome).then( + (value) => { + this.#delivered(index, fenced); + if (this.#poisoned(index)) throw new SessionFailed(this.#failure!.error); + return value; + }, + (error: unknown) => { + this.#delivered(index, fenced); + throw this.#poisoned(index) ? new SessionFailed(this.#failure!.error) : error; + }, + ); + // Only a job without writes is exempt: one with writes waits for its own publication and the outcomes before it. + return exempt + ? ran.then( + (result) => deliver(job.hit || result.after !== undefined), + () => deliver(job.hit), + ) + : deliver(true); + } + + /** Whether job `index` comes after the failure's boundary without an outcome of its own: it gets `SessionFailed`. */ + #poisoned(index: number): boolean { + return this.#failAt !== undefined && index > this.#failAt && !this.#ownOutcomes.has(index); + } + + /** The job's outcome settled: run what waited for it (`onJobEnd()`). */ + #ended(job: Job): void { + const ends = job.ends; + if (ends === undefined) return; + job.ends = undefined; + for (const end of ends) end(); + } + + /** Run a job of the Session's own upkeep on the line: no outcome to deliver, so no fence and no `SessionFailed`. */ + #maintain(work: (index: number) => Promise): Promise { + const job: Job = { index: this.#nextIndex++, hit: false }; + const ran = this.#tail.then(() => this.#onLine(job, work)); + this.#tail = ran.then(ignore, ignore); + return ran; + } + + /** Hold the line for job `index` until its line part, through admission, is done. Gates are its touches' own. */ + async #onLine(job: Job, work: (index: number) => Promise): Promise { + this.#current = job; + try { + return await work(job.index); + } finally { + this.#current = undefined; + } + } + + /** A `*` barrier: once every part admitted so far is published, or never will be. */ + async #drained(): Promise { + const drained = this.#writer.drained(); + if (drained !== undefined) await drained; + } + + /** + * Run one commit's publication inside the Storage's own scope (`Storage.publishing`), such as a store host's batch; + * directly without one. `undefined` when it ran at once, else settles once it ran. A Storage that throws instead of + * running it is reported, and the publication runs directly: the commit is stored, so it must be published. + */ + #publishing(publish: () => void, context: Context): Promise | undefined { + const publishing = this.#storage.publishing; + if (publishing === undefined) { + publish(); + return undefined; + } + const done = Promise.withResolvers(); + let ran = false; + const once = (): void => { + if (ran) return; + ran = true; + try { + publish(); + } finally { + done.resolve(); + } + }; + try { + publishing(once, context); + } catch (error) { + this.report(error, LISTENER, context); + once(); + } + return done.promise; + } + + /** Job `index` has its outcome: let the line go on past `MAX_OUTSTANDING`. */ + #delivered(index: number, fenced: boolean): void { + if (fenced) this.#lastDelivered = index; + this.#outstanding--; + const room = this.#room; + if (room !== undefined && this.#outstanding < MAX_OUTSTANDING) { + this.#room = undefined; + room.resolve(); + } } /** Internal: throw `SessionFailed` when failed, else an error when closing. */ @@ -875,17 +1755,151 @@ export function contained(callback: () => unknown, onError: (error: unknown) => } } +/** A synchronous observer as Durable attaches it: `Observer` (`advance` gets a `WatchCall`) or `CommitObserver`. */ +type SyncObserver = { advance(call: A, context: Context): void; end(end: WatchEnd): void }; + +/** A guarded observer (`guardObserver()`) with its observation's detach. */ +export type GuardedObserver = SyncObserver & { detach(): void }; + /** - * Operations an observer applies for one committed change. An observer hydrated under another definition version holds - * a differently shaped value, so it receives the new value as a root replacement instead of operations for that shape. + * `observer`, a host's, behind the rule for observers that run at a commit's publication: one that throws, or whose + * `advance` returns a promise that rejects, is reported through `report`, detached, and ended `listener_error`, so the + * throw never reaches `adopt`; nothing reaches it after its detach or end, and from then on the guard references + * neither it nor its source. `detach` is the source's; the returned `detach()` is the observation's. */ -function observedOperations( - observed: { version: number }, - change: Extract, -): readonly Op[] { - if (change.version === observed.version) return change.ops; - observed.version = change.version!; - return [["r", change.value]]; +export function guardObserver( + observer: SyncObserver, + detach: () => void, + report: (error: unknown, context?: Context) => void, +): GuardedObserver { + type Attached = { + readonly observer: SyncObserver; + readonly detach: () => void; + readonly report: (error: unknown, context?: Context) => void; + }; + let attached: Attached | undefined = { observer, detach, report }; + /** Detach from the source once, and hand back what it was attached with. */ + const release = (): Attached | undefined => { + const released = attached; + attached = undefined; + released?.detach(); + return released; + }; + /** Report a throw of `current`'s `advance`, then detach and end it, unless that happened meanwhile. */ + const fault = (current: Attached, error: unknown, context: Context): void => { + current.report(error, context); + if (release() !== undefined) { + const end: WatchEnd = { kind: "listener_error", error: toError(error) }; + contained(() => current.observer.end(end), current.report); + } + }; + return { + advance: (call, context) => { + const current = attached; + if (current === undefined) return; + try { + const result: unknown = current.observer.advance(call, context); + // Any thenable, as `contained()` takes it: a rejection is the observer's throw, after the fact. + if (typeof (result as { then?: unknown } | null)?.then === "function") { + Promise.resolve(result).catch((error: unknown) => fault(current, error, context)); + } + } catch (error) { + fault(current, error, context); + } + }, + end: (end) => { + const ending = release(); + if (ending !== undefined) contained(() => ending.observer.end(end), ending.report); + }, + detach: () => { + release(); + }, + }; +} + +/** A listener's throw as the `Error` a `listener_error` end carries, as Chord's watch makes it. */ +function toError(error: unknown): Error { + if (error instanceof Error) return error; + try { + return new Error(String(error)); + } catch { + return new Error(Object.prototype.toString.call(error)); + } +} + +/** The end of an observer whose source closed. */ +export const ENDED: WatchEnd = { kind: "ended" }; + +/** + * How a Session's close ends what follows it (decision "Slice 8 S1"), made once per close: an app's watch `ended`; a + * synchronous observer (an adapter following the Session, which may open it again) `failed` with "session closed", so + * it follows the next Session; `ended` stays for an observed owner that ended (a task document's task). A failed + * Session ends both `failed` with its error. + */ +export function closeEnds(failure: { readonly error: unknown } | undefined): { + readonly watch: WatchEnd; + readonly observer: WatchEnd; +} { + if (failure !== undefined) { + const failed: WatchEnd = { kind: "failed", error: failure.error }; + return { watch: failed, observer: failed }; + } + return { watch: ENDED, observer: { kind: "failed", error: new Error("session closed") } }; +} + +/** The end `ends` give `follower`: a watch's, or an observer's. */ +export function closeEndOf(follower: unknown, ends: ReturnType): WatchEnd { + return follower instanceof Watch ? ends.watch : ends.observer; +} + +/** + * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached + * observation keeps nothing but `value` and the spent guard alive. + */ +export function observation(value: T, guarded: { detach(): void }): Observation { + return { value, detach: () => guarded.detach() }; +} + +/** A document watch over its source's detach and report. */ +function createWatch( + value: Readonly, + detach: () => void, + report: (error: unknown) => void, +): Watch> { + return new Watch>(value, detach, report); +} + +/** Undo a watch attached while its acquisition was aborted. */ +function abortWatch(watch: Watch>): void { + watch.abort(); +} + +/** Undo an observation attached while its attachment was cancelled. */ +function detachObservation(attached: Observation>): void { + attached.detach(); +} + +/** A task document's watch delivers the retirement that ends its task, then ends. */ +function endWatch(watch: Watch>): void { + watch.endAfterDelivery(); +} + +/** A task document's observer ends right after the retirement that ends its task. */ +function endGuarded(guarded: Observer>): void { + guarded.end(ENDED); +} + +/** Whether `record` is an incarnation at `address`. */ +function atAddress(record: DocumentRecord, address: DocumentAddress): boolean { + if (record.kind !== address.kind || record.key !== address.key) return false; + switch (record.scope.kind) { + case "session": + return address.scope.kind === "session"; + case "conversation": + return address.scope.kind === "conversation" && address.scope.conversationId === record.scope.conversationId; + case "task": + return address.scope.kind === "task" && address.scope.taskId === record.scope.taskId; + } } function cancellationError(signal: AbortSignal): Error { @@ -917,9 +1931,29 @@ class GuardedStorage implements Storage { this.#latch = latch; } - /** Never exempt, whatever it throws: once admitted, whether a failed batch committed is unknown. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { - return this.#call(undefined, () => this.#storage.commit(writes, adopt, context)); + /** The backend's own scope for a publication (`Storage.publishing`), when it has one; not a call to guard. */ + get publishing(): Storage["publishing"] { + return scopeOf(this.#storage); + } + + /** + * Guarded like every call, but its outcome is the Session writer's to judge: it publishes the batches received first, + * then fails the Session for a refusal or a rejection, and for a Session that failed meanwhile. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + this.#assertHealthy(); + if (this.#closing) throw new Error("Session is closed"); + const call = (async () => this.#storage.commit(batches, receive, context))(); + this.#underway.add(call); + try { + return await call; + } finally { + this.#underway.delete(call); + } } mintId>(): Promise { diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index e7b783408..a25197a74 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -1,6 +1,6 @@ import { type Context, copyJson, type Draft, type JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type Change, type Op, type Prepared, type Tracker, track } from "@earendil-works/chord/delta"; +import { type Change, type Op, type PendingOps, type Prepared, type Tracker, track } from "@earendil-works/chord/delta"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; import { type AnyDocDefinition, @@ -24,6 +24,7 @@ import type { Cursor, DocumentAddress, DocumentCommitChange, + DocumentContent, DocumentCopySource, DocumentCreate, DocumentId, @@ -56,6 +57,7 @@ import type { } from "../types.ts"; import { ROOT_CONVERSATION_ID } from "../types.ts"; import { prepareForkDocumentCopies } from "./forks.ts"; +import { documentKey, incarnationKey, scopeKey, tableKeys } from "./keys.ts"; type AnyTaskRecord = TaskRecord; @@ -63,8 +65,8 @@ type AnyTaskRecord = TaskRecord; type SubmissionChange = SubmissionSettlement | { readonly status: "placed"; readonly entry: EntryId }; /** - * Complete record after applying one change. Placement turns a queued input `placed` and a queued write `done`; only a - * placed input can be answered. A settled record stays. + * Complete record after applying one change. Placement turns a queued input `placed` and a queued write `done`; a + * continue is never queued, so never placed. Only a placed input or continue can be answered. A settled record stays. */ function applySubmissionChange(current: SubmissionRecord, change: SubmissionChange): SubmissionRecord { if (current.status === "done" || current.status === "unanswered") return current; @@ -74,7 +76,7 @@ function applySubmissionChange(current: SubmissionRecord, change: SubmissionChan return { ...current, status, entry: change.entry } as SubmissionRecord; } if (change.status === "done" && current.status !== "placed") { - throw new Error(`Submission ${current.id} is not a placed input`); + throw new Error(`Submission ${current.id} is not a placed input or continue`); } // Queued and placed records carry no answer, reason, or detail; an unanswered input keeps its entry. return { ...current, ...change } as SubmissionRecord; @@ -97,9 +99,31 @@ export type LoadedDocument = { readonly tracker: Tracker; }; +/** + * The changes of a cached incarnation that deferred commits adopted in memory and the Session has not stored yet + * (`CommitPersistence`), folded. The Session owns it; a commit that stores them takes them. + */ +export type HeldDocument = { + readonly loaded: LoadedDocument; + readonly pending: PendingOps; + /** Set when a held commit selected a base (`checkpointWhen`): their store writes the whole value. */ + base: boolean; +}; + +/** What adoption publishes: to the Session's own observers, and the stored changes to `observeCommits()`. */ +export type AdoptedDocuments = { + /** This commit's own changes, as memory adopted them. */ + readonly observed: readonly DocumentCommitChange[]; + /** What Storage got: `observed`, unless the commit also stored held changes, which precede its own. */ + readonly stored: readonly DocumentCommitChange[]; +}; + /** Session services used by a transaction while it holds the mutation line. */ export interface TransactionHost { + /** Reads wait while their key has an unpublished writer (`GatedStorage`). */ readonly storage: Storage; + /** Wait while `key` has an unpublished writer; `undefined` when it has none. */ + touch(key: string): Promise | undefined; /** Wall clock for task lifecycle times. */ now(): number; /** Return the cached current incarnation without loading. */ @@ -117,6 +141,17 @@ export interface TransactionHost { evict(addressId: string, recordId: DocumentId): void; /** Stage writes that belong to every newly created or forked conversation, in its creating transaction. */ conversationCreated(tx: Transaction, record: ConversationRecord): Promise; + /** The newest entry ID visible from a conversation, known from the Session's commits. */ + latestEntryId(conversationId: ConversationId, context: Context): Promise; + /** + * The current documents of a live task when the Session knows all of them (it created the task), else `undefined`. + */ + taskDocuments(taskId: TaskId): Iterable | undefined; + /** The newest visible head marker of a conversation, known from the Session's commits. */ + latestHeadMarker( + conversationId: ConversationId, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; } /** Committed and candidate state for one task touched by this transaction. */ @@ -188,6 +223,11 @@ type DocumentPlan = { }; /** Resolved before Storage admission so adoption performs no reads. */ conversationId?: ConversationId; + /** + * Set when this commit also stores held changes of the incarnation (`takeHeld()`): the operations its stored change + * publishes, held first, and whether it has a change of its own to show the Session's observers. + */ + held?: { readonly ops: readonly Op[]; readonly observed: boolean }; }; /** @@ -229,6 +269,24 @@ export class Transaction implements Tx { this.#scope = scope; } + /** Internal: the context of the commit this transaction belongs to. */ + get context(): Context { + return this.#context; + } + + /** Internal: whether a table write was staged, after which table reads throw `ReadAfterWrite`. */ + get hasTableWrite(): boolean { + return this.#hasTableWrite; + } + + /** + * Internal: whether `tx` is a transaction of a commit of the Session behind `host`. A brand check, so a proxy or + * another wrapper of a transaction is not one. + */ + static ownedBy(tx: Tx, host: TransactionHost): tx is Transaction { + return #host in tx && tx.#host === host; + } + // ─── Table reads ──────────────────────────────────────────────────────── conversation(id: ConversationId): Promise { @@ -261,9 +319,11 @@ export class Transaction implements Tx { } latestHeadMarker(conversationId: ConversationId) { - return this.#read("latestHeadMarker", () => - this.#host.storage.findLatestHeadMarker(conversationId, undefined, this.#context), - ); + return this.#read("latestHeadMarker", () => this.#host.latestHeadMarker(conversationId, this.#context)); + } + + latestEntryId(conversationId: ConversationId): Promise { + return this.#read("latestEntryId", () => this.#host.latestEntryId(conversationId, this.#context)); } scanTasks(query: TaskQuery, limit: number, cursor?: Cursor) { @@ -505,6 +565,13 @@ export class Transaction implements Tx { return records; } + /** Internal: IDs of the tasks this transaction creates. */ + createdTaskIds(): TaskId[] { + const ids: TaskId[] = []; + for (const [id, task] of this.#tasksById) if (task.write?.kind === "create") ids.push(id); + return ids; + } + /** Internal: conversations this transaction created or forked so far. */ stagedConversations(): ConversationRecord[] { const records: ConversationRecord[] = []; @@ -674,6 +741,8 @@ export class Transaction implements Tx { } async #findRetirement(entry: DocumentEntry): Promise { + const gate = this.#host.touch(documentKey(entry.addressId)); + if (gate !== undefined) await gate; const record = this.#host.cached(entry.addressId)?.record ?? (await this.#host.storage.findDocument(entry.address, "current", this.#context)); @@ -724,9 +793,119 @@ export class Transaction implements Tx { this.#abortChanges(); } - /** Adopt every prepared change by pointer swap after Storage success and describe the publication. */ - adopt(seq: Seq): DocumentCommitChange[] { + /** + * Internal: what a deferred commit holds in memory instead of storing (`CommitPersistence`), each changed incarnation + * with its operations and whether they selected a base; `undefined` when the commit must be stored. It may be held + * only when every write changes an existing `latest`, `initial`-fork conversation document at its stored version. + */ + holdable(): { readonly loaded: LoadedDocument; readonly ops: readonly Op[]; readonly base: boolean }[] | undefined { + if (this.#writes.some((write) => write.type !== "document.change")) return undefined; + const held = []; + for (const plan of this.#plans) { + const { content, change } = plan; + if (content === undefined && !plan.retire) continue; + if (plan.retire || change?.loaded === undefined || content?.type !== "document.change") return undefined; + const { loaded, prepared, version } = change; + const record = loaded.record; + if (record.scope.kind !== "conversation" || record.history !== "latest" || record.fork !== "initial") { + return undefined; + } + // A migration stores a base at once. + if (loaded.storedVersion !== version) return undefined; + held.push({ loaded, ops: prepared.ops, base: content.content.kind === "base" }); + } + return held; + } + + /** + * Internal: adopt a held commit (`holdable()`) in memory only. Trackers advance by pointer swap; stored versions and + * delta counts stay, as nothing was stored. Returns its publication for the Session's own observers. + */ + adoptHeld(): DocumentCommitChange[] { const publications: DocumentCommitChange[] = []; + for (const plan of this.#plans) { + // Every plan of a held commit changes or reads a loaded incarnation (`holdable()`). + const { tracker, prepared, version, loaded } = plan.change!; + if (prepared.ops.length === 0) { + prepared.abort(); + continue; + } + tracker.adopt(prepared); + const record = loaded!.record; + const conversationId = plan.conversationId; + publications.push({ + type: "document", + kind: "change", + record, + conversationId, + version, + value: prepared.value, + ops: prepared.ops, + }); + } + return publications; + } + + /** + * Internal: store the held changes of every incarnation this commit reads or writes in front of its own, so a stored + * document never misses an operation its later ones build on. Called right before Storage admission, after every + * fallible step, because it empties the holds: a failure earlier keeps them. Returns the holds it took. + */ + takeHeld(held: (id: DocumentId) => H | undefined): H[] { + const taken: H[] = []; + for (const plan of this.#plans) { + // A new incarnation holds nothing yet. + if (!("createdAt" in plan.record)) continue; + const hold = held(plan.record.id); + if (hold === undefined) continue; + taken.push(hold); + const own = plan.content?.type === "document.change" ? plan.content : undefined; + const change = plan.change; + const value = change?.prepared.value ?? hold.loaded.tracker.value; + const stored = storeHeld(hold, change?.prepared.ops ?? EMPTY_OPERATIONS, value, hold.loaded.valueVersion); + // A base this commit selected or must write covers what was held. + const content = own?.content.kind === "base" ? own.content : stored.content; + const write = { type: "document.change", id: plan.record.id, content } as const; + if (own !== undefined) this.#writes[this.#writes.indexOf(own)] = write; + else this.#insertBeforeRetirement(write); + if (!plan.retire) plan.held = { ops: stored.ops, observed: own !== undefined }; + plan.content = write; + // Held documents are conversation documents; one this commit only read has no owner resolved yet. + if (plan.record.scope.kind === "conversation") plan.conversationId = plan.record.scope.conversationId; + } + return taken; + } + + /** + * Internal: the keys this commit writes, once settled (and after `takeHeld()`), for the Session's unpublished write + * set: its table records, and the address and incarnation of every document it creates, changes, or retires, with + * the scope of every incarnation it creates or retires. + */ + writeKeys(): string[] { + const keys: string[] = []; + tableKeys(this.#writes, keys); + for (const plan of this.#plans) { + if (plan.content === undefined && !plan.retire) continue; + keys.push(documentKey(plan.addressId), incarnationKey(plan.record.id)); + if (plan.retire || !("createdAt" in plan.record)) keys.push(scopeKey(plan.record.scope)); + } + return keys; + } + + /** Stage a content write for an incarnation, before its retirement when it retires. */ + #insertBeforeRetirement(write: Extract): void { + const at = this.#writes.findIndex( + (candidate) => candidate.type === "document.retire" && candidate.id === write.id, + ); + if (at < 0) this.#writes.push(write); + else this.#writes.splice(at, 0, write); + } + + /** Adopt every prepared change by pointer swap after Storage success and describe the publication. */ + adopt(seq: Seq): AdoptedDocuments { + const publications: DocumentCommitChange[] = []; + // Only when the commit stored held changes: then what Storage got differs from what memory adopted. + let stored: DocumentCommitChange[] | undefined; for (const plan of this.#plans) { const committed = "createdAt" in plan.record; let record: DocumentRecord = committed ? (plan.record as DocumentRecord) : { ...plan.record, createdAt: seq }; @@ -754,31 +933,25 @@ export class Transaction implements Tx { }); } } - const conversationId = plan.conversationId; - if (plan.retire) { - if (committed) this.#host.evict(plan.addressId, record.id); - const retired = { version: undefined, value: null, ops: EMPTY_OPERATIONS }; - publications.push({ type: "document", record, conversationId, ...retired }); - } else if (plan.content?.type === "document.copy") { - publications.push({ - type: "document.copy", - record, - conversationId: conversationId!, - source: plan.content.source, - }); - } else if (change !== undefined && publishes(plan)) { - const ops = change.loaded === undefined ? EMPTY_OPERATIONS : change.prepared.ops; - publications.push({ - type: "document", - record, - conversationId, - version: change.version, - value: change.prepared.value, - ops, - }); + if (plan.retire && committed) this.#host.evict(plan.addressId, record.id); + const publication = describe(plan, record); + if (publication === undefined) continue; + const held = plan.held; + if (held === undefined) { + publications.push(publication); + stored?.push(publication); + continue; } + stored ??= [...publications]; + if (held.observed) publications.push(publication); + // Only a document change of a loaded incarnation carries held operations; a retirement is published alone. + stored.push( + publication.type === "document" && publication.kind === "change" + ? { ...publication, ops: held.ops } + : publication, + ); } - return publications; + return { observed: publications, stored: stored ?? publications }; } async #assemble(): Promise { @@ -813,8 +986,21 @@ export class Transaction implements Tx { plan.retire = true; retiring.add(plan.record.id); } + const retire = (record: DocumentRecord): void => { + if (retiring.has(record.id)) return; + plans.push({ addressId: addressId(record), record, retire: true }); + retiring.add(record.id); + }; for (const taskId of terminalTaskIds) { if (this.#tasksById.get(taskId)?.write?.kind === "create") continue; + // The Session knows the documents of the tasks it created; any other task's are scanned. + const gate = this.#host.touch(scopeKey({ kind: "task", taskId })); + if (gate !== undefined) await gate; + const known = this.#host.taskDocuments(taskId); + if (known !== undefined) { + for (const record of known) retire(record); + continue; + } let cursor: Cursor | undefined; do { const page = await storage.scanDocuments( @@ -823,11 +1009,7 @@ export class Transaction implements Tx { cursor, this.#context, ); - for (const record of page.items) { - if (retiring.has(record.id)) continue; - plans.push({ addressId: addressId(record), record, retire: true }); - retiring.add(record.id); - } + for (const record of page.items) retire(record); cursor = page.next; } while (cursor !== undefined); } @@ -950,7 +1132,7 @@ export class Transaction implements Tx { #read(method: string, read: () => Promise): Promise { try { this.#assertOpen(); - if (this.#hasTableWrite) throw new ReadAfterWrite(method); + if (this.#hasTableWrite) throw new ReadAfterWrite(`Tx.${method}()`); return this.#track(read()); } catch (error) { return Promise.reject(error); @@ -1062,6 +1244,45 @@ function planDocument(document: DocumentEntry): DocumentPlan | undefined { } } +/** The publication of one adopted plan, if it publishes (`publishes()`), with `record` as committed. */ +function describe(plan: DocumentPlan, record: DocumentRecord): DocumentCommitChange | undefined { + const conversationId = plan.conversationId; + if (plan.retire) return { type: "document", kind: "retire", record, conversationId }; + if (plan.content?.type === "document.copy") { + return { type: "document.copy", record, conversationId: conversationId!, source: plan.content.source }; + } + const change = plan.change; + if (change === undefined || !publishes(plan)) return undefined; + const ops = change.loaded === undefined ? EMPTY_OPERATIONS : change.prepared.ops; + return { + type: "document", + kind: "change", + record, + conversationId, + version: change.version, + value: change.prepared.value, + ops, + }; +} + +/** + * Take a hold's changes after `ops`, a commit's own on top of them, as one stored content for an incarnation now at + * `value`, and the operations a reader of the stored state applies to reach it: the folded ones, or a root replacement + * after the queue overflowed, which stores a base, as does a hold whose commits selected one. + */ +export function storeHeld( + hold: HeldDocument, + ops: readonly Op[], + value: JsonObject, + version: number, +): { readonly content: DocumentContent; readonly ops: readonly Op[] } { + hold.pending.push(ops); + const taken = hold.pending.take(); + if (taken.kind === "replace") return { content: { version, kind: "base", value }, ops: [["r", value]] }; + if (hold.base) return { content: { version, kind: "base", value }, ops: taken.ops }; + return { content: { version, kind: "delta", ops: taken.ops }, ops: taken.ops }; +} + /** * Whether adoption publishes the plan: every creation, copy, and retirement, and a loaded incarnation that writes * content, which includes a migration-only base so observers of the older shape receive the new value. diff --git a/packages/durable/src/session/writer.ts b/packages/durable/src/session/writer.ts new file mode 100644 index 000000000..18fc73d3e --- /dev/null +++ b/packages/durable/src/session/writer.ts @@ -0,0 +1,361 @@ +// The Session's writer: commits the mutation line admitted, stored in batches off the line and published in line order. +import type { Context } from "@earendil-works/chord"; +import { withoutAbortSignal } from "@earendil-works/chord/context"; +import { SessionFailed } from "../errors.ts"; +import type { CommitResult, Seq, StorageWrite } from "../types.ts"; + +/** A part's own failure at publication: memory could not adopt it, or one of the Session's own observers threw. */ +export type PublishFailure = { readonly kind: "adoption" | "listener"; readonly error: unknown }; + +/** One logical commit the line admitted: stored (`writes`, never empty), or held in memory only (no `writes`). */ +export type PartSpec = { + /** Line index of the job that admitted it: a failure it causes makes only later jobs' outcomes `SessionFailed`. */ + readonly index: number; + readonly writes: readonly StorageWrite[] | undefined; + /** Keys it writes: a read of one waits for its publication. */ + readonly keys: readonly string[]; + readonly context: Context; + /** Publish it in memory, in line order, once it is durable (with its seq) or, held, once the parts before it are. */ + readonly publish: (seq: Seq | undefined) => PublishFailure | undefined; + /** It will never be published: abort what it prepared. */ + readonly discard: () => void; +}; + +/** An admitted part and its outcome. */ +export type Part = PartSpec & { + /** + * Resolves once it is published; rejects once it never will be, and also after publication when its storage call + * rejected or the Session failed meanwhile. Always handled, so a part nobody awaits rejects silently. + */ + readonly outcome: Promise; +}; + +type Admitted = Part & { + readonly settle: { resolve(): void; reject(error: unknown): void }; + /** Set once its outcome is settled. */ + done: boolean; + /** Opens once it is published or never will be; never rejects. */ + readonly gate: PromiseWithResolvers; +}; + +/** How a part that will not resolve settles: with an error of its own, which may be any value, or `SessionFailed`. */ +type FailedOutcome = { readonly kind: "own"; readonly error: unknown } | { readonly kind: "failed" }; + +const SESSION_FAILED: FailedOutcome = { kind: "failed" }; + +/** What the writer needs of its Session. */ +export type WriterHost = { + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise; + /** Apply a stored batch to the Session's read caches, right before it is published. */ + stored(writes: readonly StorageWrite[]): void; + /** + * Run one part's publication inside the Storage's own scope (`Storage.publishing`); `undefined` when it ran at once, + * else settles once it ran. + */ + publishing(publish: () => void, context: Context): Promise | undefined; + /** The Session's failure, if it failed. */ + failure(): { readonly error: unknown } | undefined; + /** + * Fail the Session; `at` is the line index of the first job whose outcome the failure is, and `own` the jobs after it + * whose outcome is still their own error (the other parts of a call that rejected), not `SessionFailed`. + */ + fail(error: unknown, at: number, own: readonly number[]): void; + /** Report an own observer's throw that cannot fail the Session, as it fails with a storage error. */ + report(error: unknown, context: Context): void; +}; + +/** + * The writer of one Session. The line admits each commit as a part once its callback and settlement succeeded; the + * writer sends every part admitted while no call is in flight as one `commit()` call, off the line, and publishes + * the parts of a call one by one, in line order, with a macrotask between two, so the continuations of the waits one + * resolves start before the next is published. Held commits are parts without writes, published when the parts before + * them are. + * + * Each part registers the keys it writes; until it is published, it is the newest unpublished writer of each, and a + * job that touches one waits for its gate. A key leaves when its newest writer is published, so the map holds one entry + * per key with an unpublished writer. + */ +export class Writer { + readonly #host: WriterHost; + /** Admitted parts not yet sent, in line order. */ + #queue: Admitted[] = []; + #flight = false; + /** Per key, the newest unpublished part that writes it. */ + readonly #unpublished = new Map(); + /** The newest admitted part that is not yet published, whose gate opens after every earlier one's. */ + #newest: Admitted | undefined; + + constructor(host: WriterHost) { + this.#host = host; + } + + /** Admit a part: register its keys, queue it, and send it at once unless a call is in flight. */ + admit(spec: PartSpec): Part { + const outcome = Promise.withResolvers(); + outcome.promise.catch(() => {}); + const part: Admitted = { + ...spec, + outcome: outcome.promise, + settle: { + resolve: () => { + part.done = true; + outcome.resolve(); + }, + reject: (error) => { + part.done = true; + outcome.reject(error); + }, + }, + done: false, + gate: Promise.withResolvers(), + }; + for (const key of spec.keys) this.#unpublished.set(key, part); + this.#newest = part; + this.#queue.push(part); + this.#pump(); + return part; + } + + /** The gate of the newest unpublished writer of `key`, or `undefined` when it has none. */ + gate(key: string): Promise | undefined { + return this.#unpublished.get(key)?.gate.promise; + } + + /** Opens once every part admitted so far is published or never will be. */ + drained(): Promise | undefined { + return this.#newest?.gate.promise; + } + + /** Keys with an unpublished writer, for tests that bound memory. */ + get unpublished(): number { + return this.#unpublished.size; + } + + #pump(): void { + if (this.#flight || this.#queue.length === 0) return; + this.#flight = true; + const parts = this.#queue; + this.#queue = []; + void this.#flush(parts); + } + + /** + * Send and publish `parts`, then what was admitted meanwhile. Never rejects: a throw the writer did not expect fails + * the Session and settles the parts (`#abandon()`), so no outcome or gate is stranded. + */ + async #flush(parts: readonly Admitted[]): Promise { + try { + await this.#send(parts); + } catch (error) { + this.#abandon(parts, error); + } + this.#flight = false; + this.#pump(); + } + + /** Store `parts` in one call, then publish them in order; settles every one of them. */ + async #send(parts: readonly Admitted[]): Promise { + const stored = parts.filter((part) => part.writes !== undefined); + const seqs: (Seq | undefined)[] = []; + let result: CommitResult | undefined; + let rejected: { readonly error: unknown } | undefined; + const failed = this.#host.failure(); + if (failed !== undefined) { + for (const part of parts) this.#unpublish(part, new SessionFailed(failed.error)); + return; + } + if (stored.length > 0) { + try { + result = await this.#host.commit( + stored.map((part) => part.writes!), + (index, seq) => { + seqs[index] = seq; + }, + withoutAbortSignal(stored[0]!.context), + ); + } catch (error) { + rejected = { error }; + } + } + // A call that settles once another failure failed the Session reports `SessionFailed` with that first error, + // whatever it returned (spec §10.1), and publishes nothing. + const failedMeanwhile = this.#host.failure(); + if (failedMeanwhile !== undefined) { + for (const part of parts) this.#unpublish(part, new SessionFailed(failedMeanwhile.error)); + return; + } + await this.#publish(parts, seqs, result, rejected); + } + + /** + * Publish the received parts and held ones in order, a macrotask apart, and settle every part (group commit §3.4). A + * refusal, a rejection, or a part whose adoption or own observer throws fails the Session once the parts before it + * are published and the host observers got the last one; the parts after it are not published, and reject with + * `SessionFailed`. What the failure decides (the outcomes, and the gates of the parts it concerns) is settled only + * once the Session failed, so no job waiting at such a gate, and no later outcome, goes on as if it had not. + */ + async #publish( + parts: readonly Admitted[], + seqs: readonly (Seq | undefined)[], + result: CommitResult | undefined, + rejected: { readonly error: unknown } | undefined, + ): Promise { + let stop: { readonly error: unknown; readonly at: number; readonly external?: true } | undefined; + /** + * Settled once the Session failed: each part, whether it was published, and its outcome: an error of its own (any + * value, `undefined` included), or `SessionFailed`. + */ + const failing: { readonly part: Admitted; readonly outcome: FailedOutcome; readonly published: boolean }[] = []; + let storedIndex = 0; + let published = 0; + for (const part of parts) { + const index = part.writes === undefined ? undefined : storedIndex++; + if (stop !== undefined) { + // A part after a rejected call's last receipt has the call's error; any other part after a stop was not sent, + // nor any part once something else failed the Session. + const outcome: FailedOutcome = + rejected !== undefined && index !== undefined && stop.external !== true + ? { kind: "own", error: rejected.error } + : SESSION_FAILED; + failing.push({ part, outcome, published: false }); + continue; + } + const seq = index === undefined ? undefined : seqs[index]; + if (index !== undefined && seq === undefined) { + const error = + rejected !== undefined + ? rejected.error + : index === result?.stored && "refused" in result + ? result.refused + : new Error("Storage committed without adopting the batch"); + failing.push({ part, outcome: { kind: "own", error }, published: false }); + stop = { error, at: part.index }; + continue; + } + // A macrotask between two parts: what the first resolves runs before the second is published. + if (published++ > 0) await hop(); + let failure: PublishFailure | undefined; + // Failed by something else meanwhile, during the hop or before a storage's scope runs the publication: nothing + // more of the call is published. Checked right before it would be, after every wait. + let external: { readonly error: unknown } | undefined; + const publication = this.#host.publishing(() => { + external = this.#host.failure(); + if (external !== undefined) return; + // Its keys leave first, so what its publication schedules sees it published; its waiters wake after it. + this.#unregister(part); + // A throw anywhere in it, the read caches included, leaves memory behind what is stored: like an adoption's. + try { + if (part.writes !== undefined) this.#host.stored(part.writes); + failure = part.publish(seq); + } catch (error) { + failure = { kind: "adoption", error }; + } + }, part.context); + if (publication !== undefined) await publication; + if (external !== undefined) { + stop = { error: external.error, at: part.index, external: true }; + failing.push({ part, outcome: SESSION_FAILED, published: false }); + continue; + } + if (failure?.kind === "adoption") { + failing.push({ part, outcome: { kind: "own", error: failure.error }, published: true }); + stop = { error: failure.error, at: part.index }; + continue; + } + // A part published while its call rejected is durable but rejects (spec §10.1); an own observer's throw is then + // reported, as the Session fails with the storage error. + if (rejected !== undefined) { + if (failure !== undefined) this.#host.report(failure.error, part.context); + failing.push({ part, outcome: { kind: "own", error: rejected.error }, published: true }); + continue; + } + // An own observer's throw: the commit stands; the Session fails after it. + if (failure !== undefined) stop = { error: failure.error, at: part.index }; + const failedMeanwhile = this.#host.failure(); + if (failedMeanwhile !== undefined && failure === undefined) { + part.settle.reject(new SessionFailed(failedMeanwhile.error)); + } else part.settle.resolve(); + this.#open(part); + } + if (rejected !== undefined || stop !== undefined) { + // After the host dispatch of the last part published, which is one microtask away. + await Promise.resolve(); + // Every part of the call that rejected keeps the storage error; the first of them is the failure's boundary. + const own = failing.flatMap(({ part, outcome }) => (outcome.kind === "own" ? [part.index] : [])); + if (rejected !== undefined) this.#host.fail(rejected.error, own[0]!, own); + else this.#host.fail(stop!.error, stop!.at, own); + } + for (const { part, outcome, published: stored } of failing) { + const error = outcome.kind === "own" ? outcome.error : this.#failed(stop!.error); + if (stored) { + part.settle.reject(error); + this.#open(part); + } else this.#unpublish(part, error); + } + // Admitted while this call ran, after the failure: never sent. + const failure = this.#host.failure(); + if (failure !== undefined) { + const queued = this.#queue; + this.#queue = []; + for (const part of queued) this.#unpublish(part, new SessionFailed(failure.error)); + } + } + + /** + * A throw the writer itself did not expect while sending or publishing `parts` (a bug): fail the Session at the first + * of them and settle every one not settled yet, so no outcome, gate, or close waits on it forever. + */ + #abandon(parts: readonly Admitted[], error: unknown): void { + this.#host.fail(error, parts[0]!.index, []); + for (const part of parts) { + if (!part.done) this.#unpublish(part, this.#failed(error)); + this.#unregister(part); + this.#open(part); + } + } + + /** `SessionFailed` with the Session's failure, or with `error` until the Session records it. */ + #failed(error: unknown): SessionFailed { + const failure = this.#host.failure(); + return new SessionFailed(failure !== undefined ? failure.error : error); + } + + /** Settle a part that will never be published: discard it, reject it with `error`, and open its gate. */ + #unpublish(part: Admitted, error: unknown): void { + part.discard(); + part.settle.reject(error); + this.#unregister(part); + this.#open(part); + } + + /** Drop the keys a part is still the newest writer of: it is published, or never will be. */ + #unregister(part: Admitted): void { + for (const key of part.keys) if (this.#unpublished.get(key) === part) this.#unpublished.delete(key); + } + + /** Open a part's gate: the jobs waiting for it go on. The newest part's opens `drained()` too. */ + #open(part: Admitted): void { + part.gate.resolve(); + if (this.#newest === part) this.#newest = undefined; + } +} + +/** + * Yield to the event loop once: `setImmediate` where there is one, otherwise a `MessageChannel` message (Workers, + * browsers). Microtasks queued before it, such as wait continuations, run first. + */ +function hop(): Promise { + if (typeof setImmediate === "function") return new Promise((resolve) => setImmediate(resolve)); + const channel = new MessageChannel(); + return new Promise((resolve) => { + channel.port1.onmessage = () => { + channel.port1.close(); + resolve(); + }; + channel.port2.postMessage(undefined); + }); +} diff --git a/packages/durable/src/storage/indexeddb.ts b/packages/durable/src/storage/indexeddb.ts index c2aaefcf7..8c6c4706e 100644 --- a/packages/durable/src/storage/indexeddb.ts +++ b/packages/durable/src/storage/indexeddb.ts @@ -2,9 +2,10 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { apply, type Op } from "@earendil-works/chord/delta"; -import { LogInterruptedError, StorageRequestError } from "../errors.ts"; +import { StorageRequestError } from "../errors.ts"; import { idFromNumber, seqFromNumber } from "../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -21,11 +22,10 @@ import type { EntryRecord, Id, JsonObject, - LogRecord, Page, - ReplayableStorage, ScanOrder, Seq, + Storage, StorageWrite, StoredDocument, SubmissionId, @@ -35,18 +35,6 @@ import type { TaskQuery, TaskRecord, } from "../types.ts"; -import { - assertLogRecord, - creationWrite, - groupLog, - isCurrentOnly, - LOG_START, - revisionWrite, - type SequencedWrite, - type StoredRevision, - sortBySeq, - streamOf, -} from "./log.ts"; import { nextCursor, type ScanStart, scanStart } from "./scan.ts"; const DATABASE_VERSION = 1; @@ -83,13 +71,6 @@ const INDEXES = { documentsByScope: "documents_by_scope", documentsByScopeKind: "documents_by_scope_kind", documentRevisionsByKind: "document_revisions_by_kind", - conversationsBySeq: "conversations_by_seq", - tasksBySeq: "tasks_by_seq", - submissionsBySeq: "submissions_by_seq", - entriesBySeq: "entries_by_seq", - documentsByCreation: "documents_by_creation", - documentsByRetirement: "documents_by_retirement", - documentRevisionsBySeq: "document_revisions_by_seq", } as const; type StoredTask = TaskRecord; @@ -110,7 +91,6 @@ type ConversationRow = { readonly id: ConversationId; readonly ownerConversationId?: ConversationId; readonly ownerTaskId?: TaskId; - readonly seq: Seq; readonly record: ConversationRecord; }; type EntryRow = { @@ -127,7 +107,6 @@ type TaskRow = { readonly status: StoredTask["state"]["status"]; readonly abortRequested: 0 | 1; readonly background: 0 | 1; - readonly seq: Seq; readonly record: StoredTask; }; type SubmissionRow = { @@ -135,7 +114,6 @@ type SubmissionRow = { readonly conversationId: ConversationId; readonly requestId?: string; readonly status: SubmissionRecord["status"]; - readonly seq: Seq; readonly record: SubmissionRecord; }; type DocumentRow = { @@ -161,12 +139,33 @@ type IndexedDbStorageOptions = { readonly indexedDB?: IDBFactory; }; +/** A failure IndexedDB reported without an error of its own: the database's, like every `DOMException` it throws. */ +class IndexedDbFailure extends Error {} + +/** + * Whether `error`, or an error it was caused by (a copy's rejection wraps its cause), is IndexedDB's own: a + * `DOMException`, or a failure it reported without one. Any other error of a batch's checks is one of Durable's. + */ +function fromIndexedDb(error: unknown): boolean { + const seen = new Set(); + for (let at: unknown = error; at !== null && typeof at === "object" && !seen.has(at); ) { + if (at instanceof DOMException || at instanceof IndexedDbFailure) return true; + seen.add(at); + at = (at as { cause?: unknown }).cause; + } + return false; +} + const requestResult = (request: IDBRequest): Promise => new Promise((resolve, reject) => { request.addEventListener("success", () => resolve(request.result), { once: true }); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB request failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB request failed")), + { + once: true, + }, + ); }); const transactionResult = (transaction: IDBTransaction): Promise => @@ -174,12 +173,12 @@ const transactionResult = (transaction: IDBTransaction): Promise => transaction.addEventListener("complete", () => resolve(), { once: true }); transaction.addEventListener( "abort", - () => reject(transaction.error ?? new Error("IndexedDB transaction was aborted")), + () => reject(transaction.error ?? new IndexedDbFailure("IndexedDB transaction was aborted")), { once: true }, ); transaction.addEventListener( "error", - () => reject(transaction.error ?? new Error("IndexedDB transaction failed")), + () => reject(transaction.error ?? new IndexedDbFailure("IndexedDB transaction failed")), { once: true }, ); }); @@ -199,9 +198,13 @@ const cursorValues = ( new Promise((resolve, reject) => { const values: T[] = []; const request = source.openCursor(range, direction); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB cursor failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB cursor failed")), + { + once: true, + }, + ); request.addEventListener("success", () => { const cursor = request.result; if (cursor === null) { @@ -226,9 +229,13 @@ const firstCursorValue = ( ): Promise => new Promise((resolve, reject) => { const request = source.openCursor(range, direction); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB cursor failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB cursor failed")), + { + once: true, + }, + ); request.addEventListener("success", () => { const cursor = request.result; if (cursor === null) { @@ -336,6 +343,9 @@ const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => { return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); }; +const isCurrentOnly = (record: DocumentRecord): boolean => + record.scope.kind !== "conversation" || record.history === "latest"; + const writeId = (write: StorageWrite): Id | undefined => { switch (write.type) { case "conversation": @@ -360,12 +370,10 @@ function createSchema(database: IDBDatabase): void { const conversations = database.createObjectStore(STORES.conversations, { keyPath: "id" }); conversations.createIndex(INDEXES.conversationsByOwnerConversation, ["ownerConversationId", "id"]); conversations.createIndex(INDEXES.conversationsByOwnerTask, ["ownerTaskId", "id"]); - conversations.createIndex(INDEXES.conversationsBySeq, ["seq", "id"]); const entries = database.createObjectStore(STORES.entries, { keyPath: "id" }); entries.createIndex(INDEXES.entriesByConversation, ["conversationId", "id"]); entries.createIndex(INDEXES.entryHeadsByConversation, ["headConversationId", "id"]); - entries.createIndex(INDEXES.entriesBySeq, ["commitSeq", "id"]); const tasks = database.createObjectStore(STORES.tasks, { keyPath: "id" }); tasks.createIndex(INDEXES.tasksByStatus, ["status", "id"]); @@ -373,13 +381,11 @@ function createSchema(database: IDBDatabase): void { tasks.createIndex(INDEXES.tasksByKind, ["kind", "id"]); tasks.createIndex(INDEXES.tasksByAbortRequested, ["abortRequested", "id"]); tasks.createIndex(INDEXES.tasksByBackground, ["background", "id"]); - tasks.createIndex(INDEXES.tasksBySeq, ["seq", "id"]); const submissions = database.createObjectStore(STORES.submissions, { keyPath: "id" }); submissions.createIndex(INDEXES.submissionsByRequest, ["conversationId", "requestId"]); submissions.createIndex(INDEXES.submissionsByConversation, ["conversationId", "id"]); submissions.createIndex(INDEXES.submissionsByStatus, ["status", "id"]); - submissions.createIndex(INDEXES.submissionsBySeq, ["seq", "id"]); const documents = database.createObjectStore(STORES.documents, { keyPath: "id" }); documents.createIndex(INDEXES.documentsByAddress, [ @@ -393,17 +399,13 @@ function createSchema(database: IDBDatabase): void { ]); documents.createIndex(INDEXES.documentsByScope, ["scopeKind", "ownerId", "id"]); documents.createIndex(INDEXES.documentsByScopeKind, ["scopeKind", "ownerId", "kind", "id"]); - documents.createIndex(INDEXES.documentsByCreation, ["createdAt", "id"]); - // Live documents have no `retiredAt`, so only retired ones are in this index. - documents.createIndex(INDEXES.documentsByRetirement, ["retiredAt", "id"]); const revisions = database.createObjectStore(STORES.documentRevisions, { keyPath: ["documentId", "seq"] }); revisions.createIndex(INDEXES.documentRevisionsByKind, ["documentId", "kind", "seq"]); - revisions.createIndex(INDEXES.documentRevisionsBySeq, ["seq", "documentId"]); } /** Browser IndexedDB implementation of the durable storage contract. One owner must serialize access to each database. */ -export class IndexedDbStorage implements ReplayableStorage { +export class IndexedDbStorage implements Storage { private readonly database: IDBDatabase; private closed = false; private closing: Promise | undefined; @@ -445,115 +447,54 @@ export class IndexedDbStorage implements ReplayableStorage { await requestResult(factory.deleteDatabase(name)); } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { - return this.commitAt(writes, undefined, adopt); - } - - async head(_context: Context): Promise { + /** + * Commit `batches` in one IndexedDB transaction, each with its own seq. Each batch does every read and check, copy + * resolution included, against the state the earlier ones left before it issues its first write; the first one + * refused ends the call, and the transaction commits the ones before it. An error of IndexedDB itself aborts the + * transaction: the call rejects with nothing stored. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { this.assertOpen(); - return this.transaction(STORES.meta, "readonly", async (transaction) => { - const metadata = await requestResult( - transaction.objectStore(STORES.meta).get(META_KEY) as IDBRequest, - ); - if (metadata === undefined) throw new Error("Durable IndexedDB metadata is missing"); - return metadata.nextSeq === 1 ? undefined : seqFromNumber(metadata.nextSeq - 1); - }); - } - - async replay(record: LogRecord, _context: Context): Promise { - assertLogRecord(record); - // No Session holds a replayed storage: nothing adopts. - await this.commitAt(record.writes, record.seq, () => {}); - } - - /** Reads the changes after `after` through the sequence indexes in one transaction, then groups them in memory. */ - async *log(after: Seq | undefined, context: Context): AsyncGenerator { - this.assertOpen(); - const since = after ?? LOG_START; - const read = await this.transaction(ALL_STORES, "readonly", async (transaction) => { - // Index keys start with the sequence, so `[since + 1]` sorts before every key of a later sequence. - const range = since === LOG_START ? null : keyRanges().lowerBound([since + 1]); - const after = (store: string, index: string) => - requestResult(transaction.objectStore(store).index(index).getAll(range) as IDBRequest); - const metadata = await requestResult( - transaction.objectStore(STORES.meta).get(META_KEY) as IDBRequest, - ); - if (metadata === undefined) throw new Error("Durable IndexedDB metadata is missing"); - return { - head: metadata.nextSeq === 1 ? undefined : seqFromNumber(metadata.nextSeq - 1), - conversations: await after(STORES.conversations, INDEXES.conversationsBySeq), - tasks: await after(STORES.tasks, INDEXES.tasksBySeq), - submissions: await after(STORES.submissions, INDEXES.submissionsBySeq), - entries: await after(STORES.entries, INDEXES.entriesBySeq), - created: await after(STORES.documents, INDEXES.documentsByCreation), - retired: await after(STORES.documents, INDEXES.documentsByRetirement), - revisions: await after(STORES.documentRevisions, INDEXES.documentRevisionsBySeq), - }; - }); - const writes: SequencedWrite[] = [ - ...read.conversations.map( - (row): SequencedWrite => ({ seq: row.seq, write: { type: "conversation", value: row.record } }), - ), - ...read.tasks.map((row): SequencedWrite => ({ seq: row.seq, write: { type: "task", value: row.record } })), - ...read.submissions.map( - (row): SequencedWrite => ({ seq: row.seq, write: { type: "submission", value: row.record } }), - ), - ...read.entries.map( - (row): SequencedWrite => ({ seq: row.commitSeq, write: { type: "entry", value: row.record } }), - ), - ...read.retired.map( - (row): SequencedWrite => ({ seq: row.retiredAt!, write: { type: "document.retire", id: row.id } }), - ), - ]; - // A document created after `since` has its creation revision among those read: it is at least as late. - const created = new Map(read.created.map((row) => [row.id, row.record])); - const atCreation = new Map(); - for (const row of read.revisions) { - const revision: StoredRevision = - row.kind === "base" - ? { seq: row.seq, kind: "base", version: row.version, value: row.content as JsonObject } - : { seq: row.seq, kind: "delta", version: row.version, ops: row.content as readonly Op[] }; - if (created.get(row.documentId)?.createdAt === row.seq) atCreation.set(row.documentId, revision); - else writes.push(revisionWrite(row.documentId, revision)); - } - for (const record of created.values()) writes.push(creationWrite(record, atCreation.get(record.id))); - for await (const record of groupLog(read.head, since, [streamOf(sortBySeq(writes))])) { - if ((await this.head(context)) !== read.head) throw new LogInterruptedError(); - yield record; - } - } - - /** Commit at `seq`, or at the next sequence when it is undefined; `adopt` as for `commit()`. */ - private async commitAt( - writes: readonly StorageWrite[], - at: Seq | undefined, - adopt: (committed: Seq) => void, - ): Promise { - this.assertOpen(); - const documentActions = this.prepareDocumentActions(writes); - const seq = await this.transaction(ALL_STORES, "readwrite", async (transaction) => { + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + await this.transaction(ALL_STORES, "readwrite", async (transaction) => { const metaStore = transaction.objectStore(STORES.meta); const metadata = await requestResult(metaStore.get(META_KEY) as IDBRequest); if (metadata === undefined) throw new Error("Durable IndexedDB metadata is missing"); - if (at !== undefined && (!Number.isSafeInteger(at) || at < metadata.nextSeq)) { - throw new Error(`Commit sequence ${at} does not strictly increase`); - } - const seq = at ?? seqFromNumber(metadata.nextSeq); - await this.checkGlobalIds(transaction, writes); - await this.checkDocumentActions(transaction, documentActions); - for (const write of writes) await this.applyTableWrite(transaction, write, seq); - await this.applyDocumentActions(transaction, documentActions, seq); + let next = metadata.nextSeq; let nextId = metadata.nextId; - for (const write of writes) { - const id = writeId(write); - if (id !== undefined) nextId = Math.max(nextId, id + 1); + for (const writes of batches) { + let documentActions: Map; + try { + documentActions = this.prepareDocumentActions(writes); + await this.checkGlobalIds(transaction, writes); + await this.checkDocumentActions(transaction, documentActions); + await this.resolveCopies(transaction, documentActions); + } catch (error) { + if (fromIndexedDb(error)) throw error; + refused = { error }; + break; + } + const seq = seqFromNumber(next++); + for (const write of writes) await this.applyTableWrite(transaction, write, seq); + await this.applyDocumentActions(transaction, documentActions, seq); + for (const write of writes) { + const id = writeId(write); + if (id !== undefined) nextId = Math.max(nextId, id + 1); + } + seqs.push(seq); + } + if (seqs.length > 0) { + await requestResult(metaStore.put({ key: META_KEY, nextId, nextSeq: next } satisfies MetaRow)); } - await requestResult(metaStore.put({ key: META_KEY, nextId, nextSeq: seq + 1 } satisfies MetaRow)); - return seq; }); - // The transaction completed: the batch is durable. - adopt(seq); - return seq; + // The transaction completed: the stored batches are durable. + for (const [index, seq] of seqs.entries()) receive(index, seq); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } async mintId>(): Promise { @@ -1188,7 +1129,6 @@ export class IndexedDbStorage implements ReplayableStorage { ownerConversationId: write.value.owner.conversationId, ownerTaskId: write.value.owner.taskId, }), - seq, record: write.value, } satisfies ConversationRow), ); @@ -1215,7 +1155,6 @@ export class IndexedDbStorage implements ReplayableStorage { status: write.value.state.status, abortRequested: write.value.abortRequested ? 1 : 0, background: write.value.background ? 1 : 0, - seq, record: write.value, } satisfies TaskRow), ); @@ -1228,7 +1167,6 @@ export class IndexedDbStorage implements ReplayableStorage { conversationId: write.value.conversationId, ...(write.value.requestId === undefined ? {} : { requestId: write.value.requestId }), status: write.value.status, - seq, record: write.value, } satisfies SubmissionRow), ); @@ -1241,6 +1179,31 @@ export class IndexedDbStorage implements ReplayableStorage { } } + /** Resolve each copy to a base of its source's state before this batch, which may not change the source. */ + private async resolveCopies(transaction: IDBTransaction, actions: Map): Promise { + for (const [id, action] of actions) { + if (action.copy === undefined) continue; + try { + const stored = await this.materializeDocument(transaction, action.copy.id, action.copy.at); + if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); + } + action.content = { kind: "base", version: stored.version, value: stored.value }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + private async applyDocumentActions( transaction: IDBTransaction, actions: ReadonlyMap, @@ -1249,27 +1212,8 @@ export class IndexedDbStorage implements ReplayableStorage { const documents = transaction.objectStore(STORES.documents); const revisions = transaction.objectStore(STORES.documentRevisions); for (const [id, action] of actions) { - let content = action.content; - if (action.copy !== undefined) { - try { - const stored = await this.materializeDocument(transaction, action.copy.id, action.copy.at); - if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); - const create = action.create!; - if ( - stored.record.scope.kind !== "conversation" || - create.scope.kind !== "conversation" || - stored.record.kind !== create.kind || - stored.record.key !== create.key || - stored.record.history !== create.history || - stored.record.fork !== create.fork - ) { - throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); - } - content = { kind: "base", version: stored.version, value: stored.value }; - } catch (error) { - throw new Error(`Document copy ${id} was rejected`, { cause: error }); - } - } + // Copies are resolved to their content by `resolveCopies()`. + const content = action.content; let record: DocumentRecord; if (action.create !== undefined) { record = { diff --git a/packages/durable/src/storage/jsonl/storage.ts b/packages/durable/src/storage/jsonl/storage.ts index 14994ca6b..6e8bcbf0e 100644 --- a/packages/durable/src/storage/jsonl/storage.ts +++ b/packages/durable/src/storage/jsonl/storage.ts @@ -3,6 +3,7 @@ import type { FileError, FileSystem } from "../../env/index.ts"; import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, Cursor, @@ -15,8 +16,6 @@ import type { EntryId, EntryQuery, Id, - LogRecord, - ReplayableStorage, Seq, Storage, StorageWrite, @@ -26,8 +25,7 @@ import type { TaskQuery, TaskRecord, } from "../../types.ts"; -import { assertLogRecord, isCurrentOnly, RECLAIMED_BASE } from "../log.ts"; -import { MemoryStorage, type PreparedMemoryCommit } from "../memory.ts"; +import { MemoryStorage } from "../memory.ts"; const FORMAT_VERSION = 1; const MAIN_FILE = "main.jsonl"; @@ -107,6 +105,9 @@ const isSidecarFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d const isReclaimFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl\.reclaim$/.test(name); +const isCurrentOnly = (record: DocumentCreate): boolean => + record.scope.kind !== "conversation" || record.history === "latest"; + const sidecarKey = (file: string, seq: Seq, ordinal: number): string => JSON.stringify([file, seq, ordinal]); const jsonLine = (value: MainMarker | SidecarRecord): string => `${JSON.stringify(value)}\n`; @@ -239,7 +240,7 @@ const parseSidecarRecord = (text: string, file: string, line: number): SidecarRe }; /** Portable JSONL implementation of the storage contract. */ -export class JsonlStorage implements ReplayableStorage { +export class JsonlStorage implements Storage { private readonly fs: FileSystem; private readonly directory: string; private readonly mainPath: string; @@ -249,6 +250,8 @@ export class JsonlStorage implements ReplayableStorage { private readonly liveTaskSidecars = new Set(); private closed = false; private poisonError: JsonlStoragePoisonedError | undefined; + /** Settles once the batch being appended is; its batches are in memory already (`commit()`). */ + private writing: Promise | undefined; private constructor(fs: FileSystem, directory: string, mainPath: string, options: JsonlStorageOptions) { this.fs = fs; @@ -275,37 +278,74 @@ export class JsonlStorage implements ReplayableStorage { return storage; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { + /** + * Commit `batches` with one append per affected file: every batch's sidecar records first, then their markers in one + * write of `main.jsonl`. Each batch is checked against the state the earlier ones left, so memory applies each one + * as it passes, and reads wait until the appends are done; the first batch refused ends the batch. A torn append of + * the markers recovers a shorter complete prefix; any failure once a batch applied poisons the storage. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { this.assertUsable(); - return this.persist(this.memory.prepareCommit(writes), adopt, context); + const seqs: Seq[] = []; + const encoded: EncodedCommit[] = []; + const reclamations = new Map(); + let refused: { readonly error: unknown } | undefined; + const writing = Promise.withResolvers(); + this.writing = writing.promise; + try { + for (const writes of batches) { + let prepared: ReturnType; + try { + prepared = this.memory.prepareCommit(writes); + } catch (error) { + refused = { error }; + break; + } + const commit = this.encodeCommit(prepared.seq, prepared.writes); + // A sidecar an earlier batch replaces keeps what the later ones append, unless they replace it again. + for (const [file, content] of commit.sidecars) { + const replacement = reclamations.get(file); + if (replacement !== undefined) reclamations.set(file, replacement + content); + } + for (const [file, content] of this.planReclamations(prepared.writes, commit)) + reclamations.set(file, content); + seqs.push(prepared.apply()); + this.adoptSidecarState(prepared.writes); + encoded.push(commit); + } + if (encoded.length > 0) await this.append(encoded, context); + } catch (error) { + // Memory runs ahead of the file: once a batch applied, any failure poisons the storage before the reads + // waiting on the append go on, so none of them returns what is not stored. + if (seqs.length === 0) throw error; + throw this.poison(error instanceof Error ? error : new Error("JSONL commit failed", { cause: error })); + } finally { + this.writing = undefined; + writing.resolve(); + } + // The markers are appended: the batches are durable. Reclaiming sidecars comes after the receipts. + for (const [index, seq] of seqs.entries()) receive(index, seq); + await this.reclaimSidecars(reclamations, context); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } - async head(context: Context): Promise { - return this.store.head(context); - } - - async replay(record: LogRecord, context: Context): Promise { - this.assertUsable(); - assertLogRecord(record); - // No Session holds a replayed storage: nothing adopts. - await this.persist(this.memory.prepareCommit(record.writes, record.seq), () => {}, context); - } - - log(after: Seq | undefined, context: Context): AsyncIterable { - return this.store.log(after, context); - } - - private async persist(prepared: PreparedMemoryCommit, adopt: (seq: Seq) => void, context: Context): Promise { - const encoded = this.encodeCommit(prepared.seq, prepared.writes); - const reclamations = this.planReclamations(prepared.writes, encoded); + /** Append the sidecar records of `commits`, flushed with `fsync`, then their markers in one write of `main.jsonl`. */ + private async append(commits: readonly EncodedCommit[], context: Context): Promise { + const contents = new Map(); + for (const commit of commits) { + for (const [file, content] of commit.sidecars) contents.set(file, (contents.get(file) ?? "") + content); + } const sidecars = await Promise.all( - [...encoded.sidecars].map(async ([file, content]) => ({ + [...contents].map(async ([file, content]) => ({ file, content, path: await this.resolveFile(file, context), })), ); - for (const sidecar of sidecars) { const result = await this.fs.appendFile(sidecar.path, sidecar.content, context); if (!result.ok) throw this.poison(errorFromFile(`append to ${sidecar.file}`, result.error)); @@ -316,14 +356,9 @@ export class JsonlStorage implements ReplayableStorage { if (!result.ok) throw this.poison(errorFromFile(`flush of ${sidecar.file}`, result.error)); } } - const marker = await this.fs.appendFile(this.mainPath, encoded.marker, context); + const markers = commits.length === 1 ? commits[0]!.marker : commits.map((commit) => commit.marker).join(""); + const marker = await this.fs.appendFile(this.mainPath, markers, context); if (!marker.ok) throw this.poison(errorFromFile(`append to ${MAIN_FILE}`, marker.error)); - const seq = prepared.apply(); - this.adoptSidecarState(prepared.writes); - // The marker is appended: the batch is durable. Reclaiming sidecars comes after adoption. - adopt(seq); - await this.reclaimSidecars(reclamations, context); - return seq; } async mintId>(): Promise { @@ -331,20 +366,20 @@ export class JsonlStorage implements ReplayableStorage { } async conversation(id: ConversationId, context: Context) { - return this.store.conversation(id, context); + return (await this.readable()).conversation(id, context); } async scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanConversations(query, limit, cursor, context); + return (await this.readable()).scanConversations(query, limit, cursor, context); } entry(id: EntryId, context: Context): ReturnType; entry(conversationId: ConversationId, id: EntryId, context: Context): ReturnType; async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) { if (context === undefined) - return this.store.entry(idFromNumber(idOrConversationId), idOrContext as Context); + return (await this.readable()).entry(idFromNumber(idOrConversationId), idOrContext as Context); if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID"); - return this.store.entry( + return (await this.readable()).entry( idFromNumber(idOrConversationId), idFromNumber(idOrContext), context, @@ -356,43 +391,43 @@ export class JsonlStorage implements ReplayableStorage { atOrBeforeEntryId: EntryId | undefined, context: Context, ) { - return this.store.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); + return (await this.readable()).findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); } async scanEntries(query: EntryQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanEntries(query, limit, cursor, context); + return (await this.readable()).scanEntries(query, limit, cursor, context); } async task(id: TaskId, context: Context) { - return this.store.task(id, context); + return (await this.readable()).task(id, context); } async scanTasks(query: TaskQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanTasks(query, limit, cursor, context); + return (await this.readable()).scanTasks(query, limit, cursor, context); } async submission(id: SubmissionId, context: Context) { - return this.store.submission(id, context); + return (await this.readable()).submission(id, context); } async scanSubmissions(query: SubmissionQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanSubmissions(query, limit, cursor, context); + return (await this.readable()).scanSubmissions(query, limit, cursor, context); } async submissionByRequest(conversationId: ConversationId, requestId: string, context: Context) { - return this.store.submissionByRequest(conversationId, requestId, context); + return (await this.readable()).submissionByRequest(conversationId, requestId, context); } async findDocument(address: DocumentAddress, at: DocumentPoint, context: Context) { - return this.store.findDocument(address, at, context); + return (await this.readable()).findDocument(address, at, context); } async document(id: DocumentId, at: DocumentPoint, context: Context) { - return this.store.document(id, at, context); + return (await this.readable()).document(id, at, context); } async scanDocuments(query: DocumentQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanDocuments(query, limit, cursor, context); + return (await this.readable()).scanDocuments(query, limit, cursor, context); } async close(context: Context): Promise { @@ -715,7 +750,7 @@ export class JsonlStorage implements ReplayableStorage { writes.push({ type: "document.create", record: operation.record, - content: reclaimed || content === undefined ? RECLAIMED_BASE : content, + content: reclaimed || content === undefined ? { kind: "base", version: 1, value: {} } : content, }); } else if (!reclaimed && content !== undefined) { writes.push({ type: "document.change", id, content }); @@ -854,6 +889,12 @@ export class JsonlStorage implements ReplayableStorage { return this.memory; } + /** The state to read once no batch is being appended: memory runs ahead of the file while one is. */ + private async readable(): Promise { + while (this.writing !== undefined) await this.writing; + return this.store; + } + private poison(cause: Error): JsonlStoragePoisonedError { this.poisonError ??= new JsonlStoragePoisonedError(cause); return this.poisonError; diff --git a/packages/durable/src/storage/log.ts b/packages/durable/src/storage/log.ts deleted file mode 100644 index f62ee7037..000000000 --- a/packages/durable/src/storage/log.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { seqFromNumber } from "../ids.ts"; -import type { - DocumentContent, - DocumentCreate, - DocumentId, - DocumentRecord, - LogRecord, - LogWrite, - Seq, - StorageWrite, -} from "../types.ts"; - -/** Whether a document keeps only its newest base: session, task, and `history: "latest"` conversation documents. */ -export const isCurrentOnly = (record: DocumentCreate | DocumentRecord): boolean => - record.scope.kind !== "conversation" || record.history === "latest"; - -/** Reject a log record holding a fork copy: a log carries copies as the creations they became. */ -export function assertLogRecord(record: LogRecord): void { - if (record.writes.some((write) => (write as StorageWrite).type === "document.copy")) { - throw new Error("A log record cannot contain a document copy"); - } -} - -/** - * One write of a log at the sequence that stamped it. Sequence 0 marks a record stored before stamps existed; it - * belongs to the first record of a complete log. - */ -export type SequencedWrite = { readonly seq: Seq; readonly write: LogWrite }; - -/** A stored revision of a document with the sequence that wrote it. */ -export type StoredRevision = DocumentContent & { readonly seq: Seq }; - -/** The sequence before every commit: `log(undefined)` reads what follows it, which is everything. */ -export const LOG_START = seqFromNumber(0); - -/** - * Content of a document creation whose own base was reclaimed: a current-only document keeps only its newest base, - * and nothing reads its content before that base. Logs, compaction, and JSONL recovery write it. - */ -export const RECLAIMED_BASE: Extract = Object.freeze({ - kind: "base", - version: 1, - value: Object.freeze({}), -}); - -/** The creation record a document was committed with. */ -export function documentCreate(record: DocumentRecord): DocumentCreate { - const { createdAt: _createdAt, retiredAt: _retiredAt, ...create } = record; - return create as DocumentCreate; -} - -/** - * The creation of a document at `createdAt`, carrying the base stored with it, or a placeholder when that base was - * reclaimed. `atCreation` is the stored revision at `createdAt`, if any. - */ -export function creationWrite(record: DocumentRecord, atCreation: StoredRevision | undefined): SequencedWrite { - if (atCreation !== undefined && atCreation.kind !== "base") { - throw new Error(`Document ${record.id} has a delta at its creation`); - } - const content: Extract = - atCreation === undefined - ? RECLAIMED_BASE - : { kind: "base", version: atCreation.version, value: atCreation.value }; - return { seq: record.createdAt, write: { type: "document.create", record: documentCreate(record), content } }; -} - -/** A stored revision after the creation, as the change that wrote it. */ -export function revisionWrite(id: DocumentId, revision: StoredRevision): SequencedWrite { - const content: DocumentContent = - revision.kind === "base" - ? { kind: "base", version: revision.version, value: revision.value } - : { kind: "delta", version: revision.version, ops: revision.ops }; - return { seq: revision.seq, write: { type: "document.change", id, content } }; -} - -/** - * The writes of one document after `after`, unordered: its creation when that came later, every retained revision - * after `after` except the one stored with the creation, and its retirement when that came later. - */ -export function documentWrites( - record: DocumentRecord, - revisions: readonly StoredRevision[], - after: Seq, -): SequencedWrite[] { - const writes: SequencedWrite[] = []; - if (record.createdAt > after) { - writes.push( - creationWrite( - record, - revisions.find((revision) => revision.seq === record.createdAt), - ), - ); - } - for (const revision of revisions) { - if (revision.seq > after && revision.seq !== record.createdAt) writes.push(revisionWrite(record.id, revision)); - } - if (record.retiredAt !== undefined && record.retiredAt > after) { - writes.push({ seq: record.retiredAt, write: { type: "document.retire", id: record.id } }); - } - return writes; -} - -/** Writes sorted by sequence, keeping the given order within one sequence. */ -export function sortBySeq(writes: SequencedWrite[]): SequencedWrite[] { - return writes.sort((a, b) => a.seq - b.seq); -} - -/** - * Group sequence-ordered streams of the writes after `after` into log records, ending with a record at `head`, empty - * when nothing else is there, so a log read after `head` follows on. Writes at sequence 0 join the first record. - * - * Nothing follows a head at or before `after`. An `after` past the head means the caller holds commits this storage - * does not have. - */ -export async function* groupLog( - head: Seq | undefined, - after: Seq, - streams: readonly AsyncIterable[], -): AsyncGenerator { - if ((head ?? LOG_START) < after) throw new Error(`Log requested after ${after}, past the head ${head ?? "(none)"}`); - if (head === undefined || head === after) return; - const cursors = streams.map((stream) => stream[Symbol.asyncIterator]()); - const current: (SequencedWrite | undefined)[] = []; - for (const cursor of cursors) current.push(await advance(cursor)); - - let first: LogWrite[] | undefined = []; - let last: Seq | undefined; - while (true) { - let seq: Seq | undefined; - for (const item of current) if (item !== undefined && (seq === undefined || item.seq < seq)) seq = item.seq; - if (seq === undefined) break; - if (seq > head) throw new Error(`Logged write at ${seq} is past the head ${head}`); - const writes: LogWrite[] = first ?? []; - for (let index = 0; index < cursors.length; index++) { - while (current[index]?.seq === seq) { - writes.push(current[index]!.write); - current[index] = await advance(cursors[index]!); - } - } - if (seq === LOG_START) continue; - first = undefined; - yield { seq, writes }; - last = seq; - } - if (last !== head) yield { seq: head, writes: first ?? [] }; -} - -async function advance(cursor: AsyncIterator): Promise { - const next = await cursor.next(); - if (next.done) return undefined; - return next.value; -} - -/** Iterate an array as a stream. */ -export async function* streamOf(items: Iterable): AsyncGenerator { - yield* items; -} diff --git a/packages/durable/src/storage/memory.ts b/packages/durable/src/storage/memory.ts index 63780c706..defcc0f56 100644 --- a/packages/durable/src/storage/memory.ts +++ b/packages/durable/src/storage/memory.ts @@ -1,8 +1,9 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta"; -import { LogInterruptedError, StorageRequestError } from "../errors.ts"; +import { StorageRequestError } from "../errors.ts"; import { idFromNumber, seqFromNumber } from "../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -19,12 +20,10 @@ import type { EntryRecord, Id, JsonObject, - LogRecord, - LogWrite, Page, - ReplayableStorage, ScanOrder, Seq, + Storage, StorageWrite, StoredDocument, SubmissionId, @@ -34,16 +33,6 @@ import type { TaskQuery, TaskRecord, } from "../types.ts"; -import { - assertLogRecord, - documentWrites, - groupLog, - isCurrentOnly, - LOG_START, - type SequencedWrite, - sortBySeq, - streamOf, -} from "./log.ts"; import { nextCursor, type ScanStart, scanStart } from "./scan.ts"; type StoredTask = TaskRecord; @@ -91,8 +80,6 @@ type State = { entryIds: Map; headEntryIds: Map; entryCommitSeqs: Map; - /** Sequence of the last commit that wrote each conversation, task, and submission. */ - recordSeqs: Map, Seq>; tasks: Map; taskIds: TaskId[]; taskIdsByStatus: Record; @@ -208,6 +195,9 @@ const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => { return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); }; +const isCurrentOnly = (record: DocumentRecord): boolean => + record.scope.kind !== "conversation" || record.history === "latest"; + const tableContaining = (state: State, id: Id): TableName | undefined => state.recordTypes.get(id); const page = }>( @@ -245,7 +235,7 @@ export interface PreparedMemoryCommit { * Reads and retained writes are cloned intentionally to match the ownership boundary * of serialization-backed stores. This is backend conformance, not validation. */ -export class MemoryStorage implements ReplayableStorage { +export class MemoryStorage implements Storage { private readonly state: State = { recordTypes: new Map(), conversations: new Map(), @@ -256,7 +246,6 @@ export class MemoryStorage implements ReplayableStorage { entryIds: new Map(), headEntryIds: new Map(), entryCommitSeqs: new Map(), - recordSeqs: new Map(), tasks: new Map(), taskIds: [], taskIdsByStatus: { pending: [], running: [], waiting: [], sleeping: [], completing: [], terminal: [] }, @@ -272,10 +261,35 @@ export class MemoryStorage implements ReplayableStorage { private nextSeq = 1; private closed = false; - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { - const seq = this.prepareCommit(writes).apply(); - adopt(seq); - return seq; + /** + * Apply `batches` one by one, each checked against the state the earlier ones left (`prepareCommit()`, which only + * checks and detaches, so a batch it refuses changed nothing); stops at the first one refused. Memory is durable at + * once; the receipts follow the last batch. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { + this.assertOpen(); + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + try { + for (const writes of batches) { + let prepared: PreparedMemoryCommit; + try { + prepared = this.prepareCommit(writes); + } catch (error) { + refused = { error }; + break; + } + seqs.push(prepared.apply()); + } + } finally { + // Also when a batch's apply throws: the batches applied before it are stored, and their receipts stand. + for (const [index, seq] of seqs.entries()) receive(index, seq); + } + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } /** Validate and detach one commit without changing observable state. */ @@ -350,7 +364,6 @@ export class MemoryStorage implements ReplayableStorage { switch (write.type) { case "conversation": { this.state.recordTypes.set(write.value.id, "conversation"); - this.state.recordSeqs.set(write.value.id, seq); this.state.conversations.set(write.value.id, write.value); insertSorted(this.state.conversationIds, write.value.id); const owner = write.value.owner; @@ -384,7 +397,6 @@ export class MemoryStorage implements ReplayableStorage { } case "task": { this.state.recordTypes.set(write.value.id, "task"); - this.state.recordSeqs.set(write.value.id, seq); const previous = this.state.tasks.get(write.value.id); if (previous === undefined) { insertSorted(this.state.taskIds, write.value.id); @@ -399,7 +411,6 @@ export class MemoryStorage implements ReplayableStorage { } case "submission": { this.state.recordTypes.set(write.value.id, "submission"); - this.state.recordSeqs.set(write.value.id, seq); const previous = this.state.submissions.get(write.value.id); if (previous === undefined) { insertSorted(this.state.submissionIds, write.value.id); @@ -447,53 +458,6 @@ export class MemoryStorage implements ReplayableStorage { return idFromNumber(this.nextId++); } - async head(_context: Context): Promise { - this.assertOpen(); - return this.headSeq(); - } - - async replay(record: LogRecord, _context: Context): Promise { - assertLogRecord(record); - this.prepareCommit(record.writes, record.seq).apply(); - } - - /** Builds the log from a consistent view when iteration starts; a later commit makes it reject. */ - async *log(after: Seq | undefined, _context: Context): AsyncGenerator { - this.assertOpen(); - const head = this.headSeq(); - const since = after ?? LOG_START; - const writes: SequencedWrite[] = []; - const stamped = (id: Id, write: LogWrite): void => { - const seq = this.state.recordSeqs.get(id)!; - if (seq > since) writes.push({ seq, write }); - }; - for (const id of this.state.conversationIds) { - stamped(id, { type: "conversation", value: clone(this.state.conversations.get(id)!) }); - } - for (const id of this.state.taskIds) stamped(id, { type: "task", value: clone(this.state.tasks.get(id)!) }); - for (const id of this.state.submissionIds) { - stamped(id, { type: "submission", value: clone(this.state.submissions.get(id)!) }); - } - for (const [id, value] of this.state.entries) { - const seq = this.state.entryCommitSeqs.get(id)!; - if (seq > since) writes.push({ seq, write: { type: "entry", value: clone(value) } }); - } - for (const stored of this.state.documents.values()) { - if (stored.record.createdAt <= since && (stored.record.retiredAt ?? since) <= since) { - if (stored.revisions.every((revision) => revision.seq <= since)) continue; - } - writes.push(...documentWrites(clone(stored.record), clone(stored.revisions), since)); - } - for await (const record of groupLog(head, since, [streamOf(sortBySeq(writes))])) { - if (this.headSeq() !== head) throw new LogInterruptedError(); - yield record; - } - } - - private headSeq(): Seq | undefined { - return this.nextSeq === 1 ? undefined : seqFromNumber(this.nextSeq - 1); - } - async conversation(id: ConversationId, _context: Context): Promise { this.assertOpen(); const value = this.state.conversations.get(id); diff --git a/packages/durable/src/storage/sqlite/migrations.ts b/packages/durable/src/storage/sqlite/migrations.ts index ffc8f73a8..a5ee244ae 100644 --- a/packages/durable/src/storage/sqlite/migrations.ts +++ b/packages/durable/src/storage/sqlite/migrations.ts @@ -110,26 +110,9 @@ export function createSqliteMigrations(schema: SqliteSchema): readonly SqliteMig `CREATE INDEX ${indexes.tasksByAbortRequested} ON ${tables.tasks} (abort_requested, id)`, `CREATE INDEX ${indexes.tasksByBackground} ON ${tables.tasks} (background, id)`, ]; - // Version 3 stamps conversations (their creation), tasks and submissions (their last write) with a commit sequence, - // as entries, documents and revisions already are, and orders every stamp by an index, so a log of the changes - // after a sequence is read in ranges instead of whole tables. Rows stored before carry stamp 0: they precede any - // sequence a log can be read after. - const sequenceStamps: readonly string[] = [ - `ALTER TABLE ${tables.conversations} ADD COLUMN seq INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE ${tables.tasks} ADD COLUMN seq INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE ${tables.submissions} ADD COLUMN seq INTEGER NOT NULL DEFAULT 0`, - `CREATE INDEX ${indexes.conversationsBySeq} ON ${tables.conversations} (seq, id)`, - `CREATE INDEX ${indexes.tasksBySeq} ON ${tables.tasks} (seq, id)`, - `CREATE INDEX ${indexes.submissionsBySeq} ON ${tables.submissions} (seq, id)`, - `CREATE INDEX ${indexes.entriesBySeq} ON ${tables.entries} (commit_seq, id)`, - `CREATE INDEX ${indexes.documentsByCreation} ON ${tables.documents} (created_at, id)`, - `CREATE INDEX ${indexes.documentsByRetirement} ON ${tables.documents} (retired_at, id) WHERE retired_at IS NOT NULL`, - `CREATE INDEX ${indexes.documentRevisionsBySeq} ON ${tables.documentRevisions} (seq, document_id)`, - ]; return [ { version: 1, statements: initialSchema }, { version: 2, statements: sleepingStatus }, - { version: 3, statements: sequenceStamps }, ]; } diff --git a/packages/durable/src/storage/sqlite/schema.ts b/packages/durable/src/storage/sqlite/schema.ts index df75ef319..b807ba529 100644 --- a/packages/durable/src/storage/sqlite/schema.ts +++ b/packages/durable/src/storage/sqlite/schema.ts @@ -35,13 +35,6 @@ const INDEX_NAMES = { documentsByScope: "documents_by_scope", documentsByScopeKind: "documents_by_scope_kind", documentRevisionsByKind: "document_revisions_by_kind", - conversationsBySeq: "conversations_by_seq", - tasksBySeq: "tasks_by_seq", - submissionsBySeq: "submissions_by_seq", - entriesBySeq: "entries_by_seq", - documentsByCreation: "documents_by_creation", - documentsByRetirement: "documents_by_retirement", - documentRevisionsBySeq: "document_revisions_by_seq", } as const; type SqliteIdentifiers> = { readonly [K in keyof T]: string }; diff --git a/packages/durable/src/storage/sqlite/storage.ts b/packages/durable/src/storage/sqlite/storage.ts index 71fd4fc22..1e55d6c72 100644 --- a/packages/durable/src/storage/sqlite/storage.ts +++ b/packages/durable/src/storage/sqlite/storage.ts @@ -1,9 +1,10 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { apply, type Op } from "@earendil-works/chord/delta"; import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "@earendil-works/chord/sqlite"; -import { LogInterruptedError, StorageRequestError } from "../../errors.ts"; +import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -20,12 +21,10 @@ import type { EntryRecord, Id, JsonObject, - LogRecord, - LogWrite, Page, - ReplayableStorage, ScanOrder, Seq, + Storage, StorageWrite, StoredDocument, SubmissionId, @@ -35,16 +34,6 @@ import type { TaskQuery, TaskRecord, } from "../../types.ts"; -import { - assertLogRecord, - creationWrite, - groupLog, - isCurrentOnly, - LOG_START, - revisionWrite, - type SequencedWrite, - type StoredRevision, -} from "../log.ts"; import { nextCursor, type ScanStart, scanStart } from "../scan.ts"; import { applySqliteMigrationsForSchema } from "./migrations.ts"; import { createSqliteSchema, type SqliteSchema } from "./schema.ts"; @@ -68,19 +57,6 @@ type DocumentAction = { content?: DocumentContent; retire: boolean; }; -/** Columns of a log scan: rows are read in pages ordered by `(k1, k2)`. */ -type KeysetRow = { readonly k1: number; readonly k2: number }; -type LogRevisionRow = KeysetRow & { - readonly kind: DocumentContent["kind"]; - readonly version: number; - readonly content: string; -}; -type LogCreationRow = KeysetRow & { - readonly record: string; - readonly kind: DocumentContent["kind"] | null; - readonly version: number | null; - readonly content: string | null; -}; type ScopeColumns = { readonly scopeKind: DocumentRecord["scope"]["kind"]; readonly ownerId: number; @@ -94,15 +70,6 @@ export type SqliteStorageOptions = { readonly prefix?: string; }; -const LOG_PAGE_SIZE = 256; - -const revisionFromRow = (row: RevisionRow): StoredRevision => { - const seq = seqFromNumber(row.seq); - return row.kind === "base" - ? { seq, kind: "base", version: row.version, value: parseJson(row.content) } - : { seq, kind: "delta", version: row.version, ops: parseJson(row.content) }; -}; - const parseJson = (value: string): T => JSON.parse(value) as T; const encodeJson = (value: unknown): string => JSON.stringify(value) as string; // Some SQLite bindings replace lone UTF-16 surrogates. JSON encoding keeps indexed identities lossless. @@ -163,6 +130,9 @@ const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => { return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); }; +const isCurrentOnly = (record: DocumentRecord): boolean => + record.scope.kind !== "conversation" || record.history === "latest"; + const writeId = (write: StorageWrite): Id | undefined => { switch (write.type) { case "conversation": @@ -179,8 +149,33 @@ const writeId = (write: StorageWrite): Id | undefined => { } }; +/** Whether a database call of the checks failed, whatever it threw or rejected with. */ +type CheckCalls = { failed: boolean }; + +/** + * `executor`, setting `calls.failed` when one of its calls throws, synchronously (a synchronous driver) or by + * rejecting, with any value: then a failing check is the database's, not one of Durable's own. + */ +function watchCalls(executor: SqliteExecutor, calls: CheckCalls): SqliteExecutor { + const watched = async (call: () => Promise): Promise => { + try { + return await call(); + } catch (error) { + calls.failed = true; + throw error; + } + }; + return { + exec: (sql) => watched(() => executor.exec(sql)), + run: (sql, ...params) => watched(() => executor.run(sql, ...params)), + runCounted: (sql, ...params) => watched(() => executor.runCounted(sql, ...params)), + get: (sql, ...params) => watched(() => executor.get(sql, ...params)), + all: (sql, ...params) => watched(() => executor.all(sql, ...params)), + }; +} + /** Portable SQLite implementation of the Pico storage contract. */ -export class SqliteStorage implements ReplayableStorage { +export class SqliteStorage implements Storage { private readonly db: SqliteDatabase; private readonly schema: SqliteSchema; private nextId: number; @@ -215,176 +210,89 @@ export class SqliteStorage implements ReplayableStorage { } } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { - return this.commitAt(writes, undefined, adopt); - } - - async head(_context: Context): Promise { - this.assertOpen(); - return this.readHead(); - } - - async replay(record: LogRecord, _context: Context): Promise { - assertLogRecord(record); - // No Session holds a replayed storage: nothing adopts. - await this.commitAt(record.writes, record.seq, () => {}); - } - /** - * Streams in pages through the sequence indexes, so neither a whole session nor the changes since `after` are held - * in memory. + * Commit `batches` in one SQL transaction, each with its own seq. Each batch is checked before it writes anything, + * against the state the earlier ones left, and the first one refused ends the batch: the transaction commits the ones + * before it. No savepoints, which Durable Objects do not allow: the checks run before the writes instead. */ - async *log(after: Seq | undefined, _context: Context): AsyncGenerator { + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { this.assertOpen(); - const head = await this.readHead(); - const since = after ?? LOG_START; - const unchanged = async (): Promise => { - this.assertOpen(); - if ((await this.readHead()) !== head) throw new LogInterruptedError(); - }; - const { tables } = this.schema; - const stamped = (table: string, write: (record: string) => T) => - this.mapRows( - this.keyset( - `SELECT seq AS k1, id AS k2, record FROM ${table} WHERE (seq, id) > (?, ?) ORDER BY seq, id LIMIT ?`, - since, - unchanged, - ), - (row): SequencedWrite => ({ seq: seqFromNumber(row.k1), write: write(row.record) }), - ); - const conversations = stamped(tables.conversations, (record) => ({ - type: "conversation", - value: parseJson(record), - })); - const tasks = stamped(tables.tasks, (record) => ({ type: "task", value: parseJson(record) })); - const submissions = stamped(tables.submissions, (record) => ({ - type: "submission", - value: parseJson(record), - })); - const entries = this.mapRows( - this.keyset( - `SELECT commit_seq AS k1, id AS k2, record FROM ${tables.entries} - WHERE (commit_seq, id) > (?, ?) ORDER BY commit_seq, id LIMIT ?`, - since, - unchanged, - ), - (row): SequencedWrite => ({ - seq: seqFromNumber(row.k1), - write: { type: "entry", value: parseJson(row.record) }, - }), - ); - const creations = this.mapRows( - this.keyset( - `SELECT d.created_at AS k1, d.id AS k2, d.record, r.kind, r.version, r.content - FROM ${tables.documents} d - LEFT JOIN ${tables.documentRevisions} r ON r.document_id = d.id AND r.seq = d.created_at - WHERE (d.created_at, d.id) > (?, ?) ORDER BY d.created_at, d.id LIMIT ?`, - since, - unchanged, - ), - (row) => - creationWrite( - parseJson(row.record), - row.kind === null - ? undefined - : revisionFromRow({ seq: row.k1, kind: row.kind, version: row.version!, content: row.content! }), - ), - ); - const revisions = this.mapRows( - this.keyset( - `SELECT r.seq AS k1, r.document_id AS k2, r.kind, r.version, r.content - FROM ${tables.documentRevisions} r JOIN ${tables.documents} d ON d.id = r.document_id - WHERE (r.seq, r.document_id) > (?, ?) AND r.seq <> d.created_at - ORDER BY r.seq, r.document_id LIMIT ?`, - since, - unchanged, - ), - (row) => revisionWrite(idFromNumber(row.k2), revisionFromRow({ ...row, seq: row.k1 })), - ); - const retirements = this.mapRows( - this.keyset( - `SELECT retired_at AS k1, id AS k2 FROM ${tables.documents} - WHERE retired_at IS NOT NULL AND (retired_at, id) > (?, ?) ORDER BY retired_at, id LIMIT ?`, - since, - unchanged, - ), - (row): SequencedWrite => ({ - seq: seqFromNumber(row.k1), - write: { type: "document.retire", id: idFromNumber(row.k2) }, - }), - ); - const streams = [conversations, tasks, submissions, entries, creations, revisions, retirements]; - for await (const record of groupLog(head, since, streams)) { - await unchanged(); - yield record; - } + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + let candidateNextId = this.nextId; + await this.db.transaction(async (transaction) => { + const metadata = await this.readMetadata(transaction); + let next = metadata.next_seq; + // Whether a database call of a batch's checks failed: then the failure is an unknown outcome, not a refusal. + const calls: CheckCalls = { failed: false }; + const checked = watchCalls(transaction, calls); + for (const writes of batches) { + let documentActions: Map; + // Only Durable's own checks refuse a batch, before it wrote anything: the ones before it stand. A failure of + // the database rejects the call, and rolls back the batches before it with the transaction. + calls.failed = false; + try { + documentActions = this.prepareDocumentActions(writes); + await this.checkBatch(checked, writes, documentActions); + } catch (error) { + if (calls.failed) throw error; + refused = { error }; + break; + } + const seq = seqFromNumber(next++); + await this.applyBatch(transaction, writes, documentActions, seq); + seqs.push(seq); + candidateNextId = Math.max(candidateNextId, this.candidateNextId(writes)); + } + if (seqs.length > 0) { + await this.writeMetadata(transaction, Math.max(Number(metadata.next_id), candidateNextId), next); + } + }); + this.nextId = Math.max(this.nextId, candidateNextId); + // `COMMIT` returned: the stored batches are durable. + for (const [index, seq] of seqs.entries()) receive(index, seq); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } - /** - * Rows of `sql` with a first key after `after`, in `(k1, k2)` order, a page at a time; `sql` binds the last key pair - * and the page size. - */ - private async *keyset( - sql: string, - after: Seq, - unchanged: () => Promise, - ): AsyncGenerator { - // Rows at `after` itself are excluded: every row ID is below the largest safe integer. - let last: KeysetRow = after === LOG_START ? { k1: -1, k2: -1 } : { k1: after, k2: Number.MAX_SAFE_INTEGER }; - while (true) { - const rows = await this.db.all(sql, last.k1, last.k2, LOG_PAGE_SIZE); - await unchanged(); - yield* rows; - if (rows.length < LOG_PAGE_SIZE) return; - last = rows.at(-1)!; - } - } - - private async *mapRows(rows: AsyncIterable, map: (row: R) => T): AsyncGenerator { - for await (const row of rows) yield map(row); - } - - private async readHead(): Promise { - const metadata = await this.db.get( + private async readMetadata(executor: SqliteExecutor): Promise { + const metadata = await executor.get( `SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`, ); if (metadata === undefined) throw new Error("Durable SQLite metadata is missing"); - return metadata.next_seq === 1 ? undefined : seqFromNumber(metadata.next_seq - 1); + return metadata; } - /** Commit at `seq`, or at the next sequence when it is undefined; `adopt` as for `commit()`. */ - private async commitAt( + private async writeMetadata(executor: SqliteExecutor, nextId: number, nextSeq: number): Promise { + await executor.run( + `UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`, + String(nextId), + nextSeq, + ); + } + + /** Durable's own checks of one batch, copy resolution included, before it writes anything. */ + private async checkBatch( + executor: SqliteExecutor, writes: readonly StorageWrite[], - seq: Seq | undefined, - adopt: (committed: Seq) => void, - ): Promise { - this.assertOpen(); - const documentActions = this.prepareDocumentActions(writes); - const candidateNextId = this.candidateNextId(writes); - const committed = await this.db.transaction(async (transaction) => { - const metadata = await transaction.get( - `SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`, - ); - if (metadata === undefined) throw new Error("Durable SQLite metadata is missing"); - if (seq !== undefined && (!Number.isSafeInteger(seq) || seq < metadata.next_seq)) { - throw new Error(`Commit sequence ${seq} does not strictly increase`); - } - const committedSeq = seq ?? seqFromNumber(metadata.next_seq); - await this.checkGlobalIds(transaction, writes); - await this.checkDocumentActions(transaction, documentActions); - for (const write of writes) await this.applyTableWrite(transaction, write, committedSeq); - await this.applyDocumentActions(transaction, documentActions, committedSeq); - await transaction.run( - `UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`, - String(Math.max(Number(metadata.next_id), candidateNextId)), - committedSeq + 1, - ); - return committedSeq; - }); - this.nextId = Math.max(this.nextId, candidateNextId); - // `COMMIT` returned: the batch is durable. - adopt(committed); - return committed; + documentActions: Map, + ): Promise { + await this.checkGlobalIds(executor, writes); + await this.checkDocumentActions(executor, documentActions); + await this.resolveCopies(executor, documentActions); + } + + private async applyBatch( + executor: SqliteExecutor, + writes: readonly StorageWrite[], + documentActions: ReadonlyMap, + seq: Seq, + ): Promise { + for (const write of writes) await this.applyTableWrite(executor, write, seq); + await this.applyDocumentActions(executor, documentActions, seq); } async mintId>(): Promise { @@ -975,11 +883,10 @@ export class SqliteStorage implements ReplayableStorage { case "conversation": await this.claimId(executor, write.value.id, "conversation"); await executor.run( - `INSERT INTO ${this.schema.tables.conversations} (id, owner_conversation_id, owner_task_id, seq, record) VALUES (?, ?, ?, ?, ?)`, + `INSERT INTO ${this.schema.tables.conversations} (id, owner_conversation_id, owner_task_id, record) VALUES (?, ?, ?, ?)`, write.value.id, write.value.owner?.conversationId ?? null, write.value.owner?.taskId ?? null, - seq, encodeJson(write.value), ); break; @@ -997,32 +904,30 @@ export class SqliteStorage implements ReplayableStorage { case "task": await this.claimId(executor, write.value.id, "task"); await executor.run( - `INSERT INTO ${this.schema.tables.tasks} (id, conversation_id, kind, status, abort_requested, background, seq, record) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `INSERT INTO ${this.schema.tables.tasks} (id, conversation_id, kind, status, abort_requested, background, record) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id, kind = excluded.kind, status = excluded.status, abort_requested = excluded.abort_requested, - background = excluded.background, seq = excluded.seq, record = excluded.record`, + background = excluded.background, record = excluded.record`, write.value.id, write.value.conversationId, encodeIndexedString(write.value.kind), write.value.state.status, write.value.abortRequested ? 1 : 0, write.value.background ? 1 : 0, - seq, encodeJson(write.value), ); break; case "submission": await this.claimId(executor, write.value.id, "submission"); await executor.run( - `INSERT INTO ${this.schema.tables.submissions} (id, conversation_id, request_id, status, seq, record) VALUES (?, ?, ?, ?, ?, ?) + `INSERT INTO ${this.schema.tables.submissions} (id, conversation_id, request_id, status, record) VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id, - request_id = excluded.request_id, status = excluded.status, seq = excluded.seq, record = excluded.record`, + request_id = excluded.request_id, status = excluded.status, record = excluded.record`, write.value.id, write.value.conversationId, write.value.requestId === undefined ? null : encodeIndexedString(write.value.requestId), write.value.status, - seq, encodeJson(write.value), ); break; @@ -1042,33 +947,39 @@ export class SqliteStorage implements ReplayableStorage { ); } + /** Resolve each copy to a base of its source's state before this batch, which may not change the source. */ + private async resolveCopies(executor: SqliteExecutor, actions: Map): Promise { + for (const [id, action] of actions) { + if (action.copy === undefined) continue; + try { + const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at); + if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); + } + action.content = { kind: "base", version: stored.version, value: stored.value }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + private async applyDocumentActions( executor: SqliteExecutor, actions: ReadonlyMap, seq: Seq, ): Promise { for (const [id, action] of actions) { - let content = action.content; - if (action.copy !== undefined) { - try { - const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at); - if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); - const create = action.create!; - if ( - stored.record.scope.kind !== "conversation" || - create.scope.kind !== "conversation" || - stored.record.kind !== create.kind || - stored.record.key !== create.key || - stored.record.history !== create.history || - stored.record.fork !== create.fork - ) { - throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); - } - content = { kind: "base", version: stored.version, value: stored.value }; - } catch (error) { - throw new Error(`Document copy ${id} was rejected`, { cause: error }); - } - } + // Copies are resolved to their content by `resolveCopies()`. + const content = action.content; let record: DocumentRecord; if (action.create !== undefined) { record = { diff --git a/packages/durable/src/storage/store/commit.ts b/packages/durable/src/storage/store/commit.ts new file mode 100644 index 000000000..7f89e7130 --- /dev/null +++ b/packages/durable/src/storage/store/commit.ts @@ -0,0 +1,345 @@ +// One Durable commit as one commit of the store's series (slice 6b §2.3): staged before the queue (rows encoded, +// contents serialized, explicit ids raising the store's allocator), then inside the series' transaction what +// `SqliteStorage` does inside its own, with the step's reader: Durable's checks of ids and documents and the copies +// resolved from their sources (its messages, a refusal), then its one composite write: the table puts, and the +// documents in two passes, existing ones first (their content, then their retirement, which frees the addresses this +// commit's creations take), then the creations and copies. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type EncodedRow, + type DocumentContent as EngineContent, + type DocumentRecord as EngineRecord, + findDocument, + insertDocument, + type OwnerTable, + readDocument, + retireRecord, + type SeriesCommit, + type StagedWrite, + type Step, + type StoreWrite, + writeContent, +} from "@earendil-works/chord/next"; +import { StorageRequestError } from "../../errors.ts"; +import type { + DocumentContent, + DocumentCopySource, + DocumentCreate, + DocumentId, + Id, + JsonObject, + StorageWrite, +} from "../../types.ts"; +import { addressOf, durableRecord, engineRecord, readAt } from "./documents.ts"; +import { conversationRow, entryRow, submissionRow, taskRow } from "./tables.ts"; + +/** Durable's record types, as `record_ids` names them. */ +export type RecordType = "conversation" | "entry" | "task" | "submission" | "document"; + +/** The storage's tables, as a commit writes them. */ +export type CommitTables = { + readonly recordIds: OwnerTable; + readonly conversations: OwnerTable; + readonly entries: OwnerTable; + readonly entryHeads: OwnerTable; + readonly tasks: OwnerTable; + readonly submissions: OwnerTable; +}; + +/** A table write of the commit: its record's id and type, its table and its row encoded at staging. */ +type RowWrite = { + readonly id: number; + readonly type: Exclude; + readonly table: OwnerTable; + readonly row: EncodedRow; +}; + +/** A base as the commit stores it: its version and value, and the value serialized. */ +type StagedBase = { readonly version: number; readonly value: JsonObject; readonly text: string }; + +/** What one commit does with one document, as `SqliteStorage`'s action map has it. */ +type DocumentAction = { + create?: DocumentCreate; + copy?: DocumentCopySource; + /** A creation's base: the write's own, or a copy's once resolved. */ + base?: StagedBase; + content?: { readonly content: DocumentContent; readonly text: string }; + retire: boolean; +}; + +/** A Durable commit staged for the store's series: its checks, then itself as its one write. */ +export class DurableCommit implements SeriesCommit, StagedWrite { + readonly order = "write"; + readonly writes: readonly StagedWrite[]; + readonly #owner: string; + readonly #tables: CommitTables; + readonly #context: Context; + /** Every id a write claims, in write order (documents created or copied included), for the id checks. */ + readonly #claims: { readonly id: number; readonly type: RecordType }[] = []; + readonly #rows: RowWrite[] = []; + readonly #actions = new Map(); + /** What `check` found for `apply`, in the same step: the stored types of the claimed ids, the changed documents. */ + #checked: + | { readonly claimed: Map; readonly existing: Map } + | undefined; + + /** + * Stages `writes` (rule 8: rows and contents encoded here, before the queue); `raise` makes room above each explicit + * id. A malformed batch (two content commands for one document, a second retirement) throws Durable's message. + */ + constructor( + writes: readonly StorageWrite[], + owner: string, + tables: CommitTables, + raise: (id: number) => void, + context: Context, + ) { + this.#owner = owner; + this.#tables = tables; + this.#context = context; + for (const write of writes) this.#stage(write, raise); + this.writes = [this]; + } + + /** + * Durable's checks against the state the series' earlier commits left, writing nothing: the ids, the documents and + * the copies; a throw refuses the commit. + */ + async check(step: Step): Promise { + const claimed = await this.#checkIds(step); + const existing = await this.#checkDocuments(step); + await this.#resolveCopies(step); + this.#checked = { claimed, existing }; + } + + /** + * Resolves the copies (S11): each source read at its point and checked against the copy's record, entered as a + * creation carrying the source's value as its base. + */ + async #resolveCopies(step: Step): Promise { + for (const [id, action] of this.#actions) { + const source = action.copy; + if (source === undefined) continue; + if (this.#actions.has(source.id)) throw new Error(`Document copy ${id} source is changed in the copy batch`); + try { + const read = await readAt(step, source.id, source.at, this.#owner); + if (read.kind === "current_only") { + throw new StorageRequestError(`Document ${read.id} does not retain historical content`); + } + const stored = read.document; + if (stored === undefined) throw new Error(`Fork source document ${source.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${source.id} does not match the copied record`); + } + action.base = { version: stored.version, value: stored.value, text: JSON.stringify(stored.value) }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + + /** The commit's writes, after its `check` in the same step. */ + async apply(step: Step): Promise { + const recorded: StoreWrite[] = []; + const { claimed, existing } = this.#checked!; + for (const write of this.#rows) { + if (claimed.get(write.id) === undefined) recorded.push(await this.#claim(step, write.id, write.type)); + claimed.set(write.id, write.type); + const stamp = write.table === this.#tables.entries ? "commit_seq" : undefined; + recorded.push(await write.table.put(step, write.row, stamp)); + } + // Existing documents first: a retirement frees its address before a creation of this commit takes it. + for (const [id, record] of existing) recorded.push(...(await this.#change(step, record, this.#actions.get(id)!))); + for (const [id, action] of this.#actions) { + if (action.create === undefined) continue; + recorded.push(await this.#claim(step, id, "document")); + recorded.push(await this.#create(step, action.create, action.base!, action.retire)); + } + return recorded; + } + + /** Stages one write: a table row encoded, a document command into its action. */ + #stage(write: StorageWrite, raise: (id: number) => void): void { + const tables = this.#tables; + switch (write.type) { + case "conversation": + this.#row(write.value.id, write.type, tables.conversations, conversationRow(write.value), raise); + return; + case "entry": { + const { id, conversationId, head } = write.value; + this.#row(id, write.type, tables.entries, entryRow(write.value), raise); + // Its claim is the entry's: `apply` claims an id once. + if (head !== undefined) { + const row = tables.entryHeads.encode({ id, conversation_id: conversationId }); + this.#rows.push({ id, type: write.type, table: tables.entryHeads, row }); + } + return; + } + case "task": + this.#row(write.value.id, write.type, tables.tasks, taskRow(write.value), raise); + return; + case "submission": + this.#row(write.value.id, write.type, tables.submissions, submissionRow(write.value), raise); + return; + case "document.create": + case "document.copy": { + const { id } = write.record; + const action = this.#action(id); + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${id} has more than one content command`); + } + action.create = write.record; + if (write.type === "document.copy") action.copy = write.source; + else action.base = { ...write.content, text: JSON.stringify(write.content.value) }; + this.#claims.push({ id, type: "document" }); + raise(id); + return; + } + case "document.change": { + const action = this.#action(write.id); + // A creation's base is its content command (`SqliteStorage` counts it so too), in either order. + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${write.id} has more than one content command`); + } + const { content } = write; + action.content = { content, text: JSON.stringify(content.kind === "base" ? content.value : content.ops) }; + return; + } + case "document.retire": { + const action = this.#action(write.id); + if (action.retire) throw new Error(`Document ${write.id} is retired more than once`); + action.retire = true; + return; + } + } + } + + #row(id: Id, type: RowWrite["type"], table: OwnerTable, row: object, raise: (id: number) => void): void { + this.#claims.push({ id, type }); + this.#rows.push({ id, type, table, row: table.encode(row) }); + raise(id); + } + + #action(id: DocumentId): DocumentAction { + let action = this.#actions.get(id); + if (action === undefined) { + action = { retire: false }; + this.#actions.set(id, action); + } + return action; + } + + /** + * Durable's global id checks (`SqliteStorage.checkGlobalIds`): a conversation, entry or document id claimed by + * anything, or written twice; a task or submission id claimed by another type. Returns the stored types. + */ + async #checkIds(step: Step): Promise> { + const stored = new Map(); + const claimed = new Map(); + for (const { id, type } of this.#claims) { + let existing = stored.get(id); + if (!stored.has(id)) { + const row = await this.#tables.recordIds.get(step, [id], this.#context); + existing = row?.record_type as RecordType | undefined; + stored.set(id, existing); + } + const earlier = claimed.get(id); + if (type === "conversation" || type === "entry" || type === "document") { + if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`); + } else { + if (existing !== undefined && existing !== type) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined && earlier !== type) throw new Error(`ID ${id} is written as two record types`); + } + claimed.set(id, type); + } + const types = new Map(); + for (const [id, type] of stored) if (type !== undefined) types.set(id, type); + return types; + } + + /** + * Durable's document checks (`SqliteStorage.checkDocumentActions`): a change or retirement of an unknown or retired + * document, a delta on another version, more than one live incarnation per address after the commit. Returns the + * existing records the commit changes or retires, in action order. + */ + async #checkDocuments(step: Step): Promise> { + const existing = new Map(); + for (const [id, action] of this.#actions) { + if (action.create !== undefined) continue; + const stored = await readDocument(step, id); + const record = stored === undefined ? undefined : durableRecord(stored, this.#owner); + if (record === undefined) throw new Error(`Unknown document: ${id}`); + if (record.retiredAt !== undefined) throw new Error(`Document ${id} is retired`); + const content = action.content?.content; + if (content?.kind === "delta" && content.version !== stored!.version) { + throw new Error(`Document ${id} version transition requires a base`); + } + existing.set(id, stored!); + } + /** Per address a live creation takes: its live incarnations after the commit so far. */ + const live = new Map(); + for (const action of this.#actions.values()) { + if (action.create === undefined || action.retire) continue; + const address = addressOf(action.create, this.#owner); + const key = JSON.stringify([address.namespace, address.kind, address.key]); + let count = live.get(key); + if (count === undefined) { + // The incarnation live there now counts unless this commit retires it. + const current = await findDocument(step, address); + count = current === undefined || this.#actions.get(current.id as DocumentId)?.retire === true ? 0 : 1; + } + if (++count > 1) throw new Error("Document address already has a current incarnation"); + live.set(key, count); + } + return existing; + } + + /** Puts `id`'s `record_ids` row: its record type. */ + #claim(step: Step, id: number, type: RecordType): Promise { + const table = this.#tables.recordIds; + return table.put(step, table.encode({ id, record_type: type })); + } + + /** An existing document's content, then its retirement, as their writes. */ + async #change(step: Step, record: EngineRecord, action: DocumentAction): Promise { + const writes: StoreWrite[] = []; + let { revision } = record; + if (action.content !== undefined) { + const { content, text } = action.content; + const stored = engineContent(content); + await writeContent(step, record.id, record.history, revision, stored, text); + writes.push({ kind: "document.change", id: record.id, revision, content: stored }); + revision++; + } + if (action.retire) { + await retireRecord(step, record); + writes.push({ kind: "document.retire", id: record.id, revision }); + } + return writes; + } + + /** A creation or a resolved copy (retired at once for a create-plus-retire, an empty lifetime), as its write. */ + async #create(step: Step, create: DocumentCreate, base: StagedBase, retired: boolean): Promise { + const record = engineRecord(create, this.#owner, base.version, step.seq, retired); + await insertDocument(step, record, base.text); + const value = base.value as JsonValue; + return { kind: "document.create", record, base: { version: base.version, fingerprint: "", value } }; + } +} + +/** A Durable content as the engine stores it: its documents have no schema fingerprint. */ +function engineContent(content: DocumentContent): EngineContent { + return content.kind === "base" + ? { kind: "base", version: content.version, fingerprint: "", value: content.value } + : { kind: "delta", version: content.version, ops: content.ops }; +} diff --git a/packages/durable/src/storage/store/documents.ts b/packages/durable/src/storage/store/documents.ts new file mode 100644 index 000000000..d81f94d2d --- /dev/null +++ b/packages/durable/src/storage/store/documents.ts @@ -0,0 +1,143 @@ +// Durable's documents in the store's documents engine (slice 6b §1.5, R1): a Durable address as an engine address +// under the storage's owner name (its scope as the namespace, its kind and key JSON-encoded as Durable's indexed +// strings), a Durable record as an engine record and back (the Durable record in `meta`), and a document read at a +// point. + +import { + type DocumentEntry, + type DocumentHistory, + type DocumentAddress as EngineAddress, + type DocumentPoint as EnginePoint, + type DocumentRecord as EngineRecord, + loadContent, + readDocumentEntry, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { seqFromNumber } from "../../ids.ts"; +import type { + DocumentAddress, + DocumentCreate, + DocumentId, + DocumentPoint, + DocumentRecord, + JsonObject, + StoredDocument, +} from "../../types.ts"; +import { encodeIndexed } from "./tables.ts"; + +/** The namespaces a storage claims under its owner: Durable's scopes (Chord's namespaces never contain a `:`). */ +export const SCOPE_NAMESPACES = ["session", "conversation:", "task:"] as const; + +/** The engine namespace of a Durable scope: `session`, `conversation:`, `task:`. */ +export function namespaceOf(scope: DocumentRecord["scope"]): string { + switch (scope.kind) { + case "session": + return "session"; + case "conversation": + return `conversation:${scope.conversationId}`; + case "task": + return `task:${scope.taskId}`; + } +} + +/** The engine address of a Durable address under `owner`. */ +export function addressOf(address: DocumentAddress | DocumentCreate, owner: string): EngineAddress { + const at = { owner, namespace: namespaceOf(address.scope), kind: encodeIndexed(address.kind) }; + return address.key === undefined ? at : { ...at, key: encodeIndexed(address.key) }; +} + +/** What a record of `scope` keeps: session and task documents and latest conversation ones their latest state. */ +export function historyOf(record: DocumentCreate | DocumentRecord): DocumentHistory { + return record.scope.kind === "conversation" && record.history === "rewindable" ? "rewindable" : "latest"; +} + +/** + * The engine record of a Durable creation at `seq` (retired at once for an empty lifetime): the Durable record itself, + * its creation seq included, in `meta`, so a read of a live record parses one JSON text from one column, as + * `SqliteStorage`'s does (a retirement later sets only the engine's `retired_at`). + */ +export function engineRecord( + create: DocumentCreate, + owner: string, + version: number, + seq: number, + retired: boolean, +): EngineRecord { + return { + id: create.id, + ...addressOf(create, owner), + history: historyOf(create), + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: JSON.stringify({ ...create, createdAt: seq }), + revision: 1, + version, + fingerprint: "", + }; +} + +/** + * The Durable record of an engine record of this storage's owner and scopes, `undefined` for anyone else's (a Chord + * document, another owner's): its id is unknown to this storage. + */ +export function durableRecord(record: EngineRecord, owner: string): DocumentRecord | undefined { + const { meta } = record; + if (meta === undefined || !ownedHere(record, owner)) return undefined; + return recordOfEntry({ meta, retired_at: record.retiredAt ?? null }); +} + +/** The Durable record of an engine entry read at one of this storage's addresses or scopes (the record in `meta`). */ +export function recordOfEntry(entry: DocumentEntry): DocumentRecord { + // Written by `engineRecord`, parsed fresh: a retirement since is set on it in place. + const record = JSON.parse(entry.meta!) as { -readonly [K in keyof DocumentRecord]: DocumentRecord[K] }; + if (entry.retired_at !== undefined && entry.retired_at !== null) record.retiredAt = seqFromNumber(entry.retired_at); + return record as DocumentRecord; +} + +/** Whether a record is this storage's: `owner`'s, in one of Durable's scopes (not a Chord document of the owner). */ +function ownedHere(record: { readonly owner: string; readonly namespace: string }, owner: string): boolean { + const { namespace } = record; + if (record.owner !== owner) return false; + return namespace === "session" || namespace.startsWith("conversation:") || namespace.startsWith("task:"); +} + +/** Whether a document is alive at `at` (the half-open lifetime). */ +export function aliveAt(record: DocumentRecord, at: DocumentPoint): boolean { + if (at === "current") return record.retiredAt === undefined; + return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); +} + +/** + * A document read at a point, as a tagged result: request errors are told after the guarded read (slice 6b §1.4), + * never thrown inside it. + */ +export type DocumentRead = + | { readonly kind: "found"; readonly document: StoredDocument | undefined } + | { readonly kind: "current_only"; readonly id: DocumentId }; + +/** + * Document `id` of `owner` at `at`: absent (unknown, someone else's, not alive at `at`), a record whose history does + * not reach back to `at`, or its value from the base at or below `at` and the deltas up to it. + */ +export async function readAt( + executor: Pick, + id: DocumentId, + at: DocumentPoint, + owner: string, +): Promise { + const entry = await readDocumentEntry(executor, id); + if (entry === undefined || entry.meta === null || !ownedHere(entry, owner)) { + return { kind: "found", document: undefined }; + } + const record = recordOfEntry(entry); + if (at !== "current" && entry.history !== "rewindable") return { kind: "current_only", id }; + if (!aliveAt(record, at)) return { kind: "found", document: undefined }; + const point: EnginePoint = at; + const loaded = await loadContent(executor, id, point); + // The engine stores what Durable's commits gave it: JSON objects. + const value = loaded.value as JsonObject; + return { + kind: "found", + document: { record, version: loaded.version, value, deltasSinceBase: loaded.deltasSinceBase }, + }; +} diff --git a/packages/durable/src/storage/store/index.ts b/packages/durable/src/storage/store/index.ts new file mode 100644 index 000000000..0ddf86cdc --- /dev/null +++ b/packages/durable/src/storage/store/index.ts @@ -0,0 +1 @@ +export { StoreStorage } from "./storage.ts"; diff --git a/packages/durable/src/storage/store/queries.ts b/packages/durable/src/storage/store/queries.ts new file mode 100644 index 000000000..8491270b7 --- /dev/null +++ b/packages/durable/src/storage/store/queries.ts @@ -0,0 +1,137 @@ +// StoreStorage's reads as the store's queries (slice 6b D2, §2.4), compiled once from the store's query vocabulary +// into SQL over the declared tables: no SQL is rendered per call, and a read selects only the record cell (by id, an +// entry's commit seq too; each column returned costs) and parses one JSON text per record, as `SqliteStorage`'s do. + +import type { Context } from "@earendil-works/chord"; +import type { Order, OwnerTable, WhereNode } from "@earendil-works/chord/next"; +import type { SqliteExecutor, SqliteValue } from "@earendil-works/chord/sqlite"; +import type { CommitTables } from "./commit.ts"; + +/** + * A stored row as a query selects it: the record cell, its version (selected only once the column has versions past + * 1), and an entry's commit seq. + */ +type RecordRow = { readonly record: string; readonly record$v?: number; readonly commit_seq?: number }; + +/** One condition of a read: a column, how it compares, its value; a call site keeps one order of them. */ +export type Condition = readonly [string, "eq" | "gt" | "lt" | "gte" | "lte", SqliteValue]; + +const ASCENDING: Order = { columns: ["id"], direction: "asc" }; +const DESCENDING: Order = { columns: ["id"], direction: "desc" }; + +/** The columns of a read of ids only. */ +const ID = ["id"]; + +/** A placeholder value: a query's shape binds its values at each call. */ +const ANY = 0; + +/** + * One table's compiled reads: by id (with the `byIdExtra` columns), and by shape (conditions in binding order, an + * order; the record only), each shape compiled at its first use. + */ +export class TableQueries { + readonly #table: OwnerTable; + /** The record column's read of a cell at a version past 1; `undefined` while the column is at version 1. */ + readonly #versioned: ((text: string, version: number, context: Context) => unknown) | undefined; + readonly #byId: string; + readonly #shapes = new Map(); + readonly #columns: readonly string[]; + + constructor(table: OwnerTable, byIdExtra: readonly string[] = []) { + this.#table = table; + const versioned = table.jsonVersion("record") > 1; + this.#versioned = versioned ? table.reader("record") : undefined; + this.#columns = versioned ? ["record", "record$v"] : ["record"]; + this.#byId = table.prepareGet([...this.#columns, ...byIdExtra]); + } + + /** The row with `id`, `undefined` when absent. */ + byId(executor: Pick, id: number): Promise { + return executor.get(this.#byId, id); + } + + /** Up to `limit` rows whose `conditions` hold, by id in `direction`. */ + select( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + limit: number, + ): Promise { + const sql = this.#shape(conditions, direction, "many"); + return executor.all(sql, ...conditions.map(([, , value]) => value), limit); + } + + /** The first row whose `conditions` hold, by id in `direction`; `undefined` when none does. */ + first( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + ): Promise { + const sql = this.#shape(conditions, direction, "first"); + return executor.get(sql, ...conditions.map(([, , value]) => value)); + } + + /** A row's record, read as its column's (version checked, migrated on access). */ + record(row: RecordRow, context: Context): T { + const versioned = this.#versioned; + // The cell holds a record of this table the storage wrote; at version 1 it is that JSON (`OwnerTable.reader`). + return (versioned === undefined ? JSON.parse(row.record) : versioned(row.record, row.record$v!, context)) as T; + } + + /** Appends the records of `rows` to `into`, in order, each read as `record` reads it (one check per call). */ + records(rows: readonly RecordRow[], into: T[], context: Context): void { + const versioned = this.#versioned; + if (versioned === undefined) { + for (const row of rows) into.push(JSON.parse(row.record) as T); + return; + } + for (const row of rows) into.push(versioned(row.record, row.record$v!, context) as T); + } + + /** The compiled read of a shape: its conditions' columns and comparisons, the direction, one row or many. */ + #shape(conditions: readonly Condition[], direction: "ascending" | "descending", rows: "first" | "many"): string { + let key = `${rows} ${direction}`; + for (const [column, condition] of conditions) key += ` ${column} ${condition}`; + let sql = this.#shapes.get(key); + if (sql !== undefined) return sql; + // One comparison per condition: the vocabulary's node of that name, its value a placeholder. + const nodes = conditions.map(([column, condition]) => ({ [condition]: [column, ANY] }) as unknown as WhereNode); + const where: WhereNode | undefined = nodes.length === 0 ? undefined : { and: nodes }; + const order = direction === "ascending" ? ASCENDING : DESCENDING; + sql = + rows === "first" + ? this.#table.prepareFirst(this.#columns, where, order) + : this.#table.prepare(this.#columns, where, order); + this.#shapes.set(key, sql); + return sql; + } +} + +/** The compiled reads of the storage's tables. */ +export type StorageQueries = { + readonly conversations: TableQueries; + readonly entries: TableQueries; + /** The id of a conversation's newest entry with a head marker: any, or at or before an id (bound second). */ + readonly latestHead: { readonly any: string; readonly atOrBefore: string }; + readonly tasks: TableQueries; + readonly submissions: TableQueries; +}; + +/** The reads of `tables`. */ +export function storageQueries(tables: CommitTables): StorageQueries { + const heads = tables.entryHeads; + return { + conversations: new TableQueries(tables.conversations), + entries: new TableQueries(tables.entries, ["commit_seq"]), + latestHead: { + any: heads.prepareFirst(ID, { eq: ["conversation_id", ANY] }, DESCENDING), + atOrBefore: heads.prepareFirst( + ID, + { and: [{ eq: ["conversation_id", ANY] }, { lte: ["id", ANY] }] }, + DESCENDING, + ), + }, + tasks: new TableQueries(tables.tasks), + submissions: new TableQueries(tables.submissions), + }; +} diff --git a/packages/durable/src/storage/store/storage.ts b/packages/durable/src/storage/store/storage.ts new file mode 100644 index 000000000..bfbfb894c --- /dev/null +++ b/packages/durable/src/storage/store/storage.ts @@ -0,0 +1,575 @@ +// Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): opening under an owner name (claims, tables, the +// allocator), commits as one composite write of the store's commit step, and the reads, each one guarded read of the +// store (several queries at one state in a read transaction), its request errors in a tagged result told after it. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type ChordStore, + type Claimant, + findDocumentEntry, + listDocumentEntries, + type SeriesCommit, + type Store, + StoreOwnerTables, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { StorageRequestError } from "../../errors.ts"; +import { idFromNumber, seqFromNumber } from "../../ids.ts"; +import type { + CommitResult, + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + ScanOrder, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../../types.ts"; +import { nextCursor, type ScanStart, scanStart } from "../scan.ts"; +import { type CommitTables, DurableCommit } from "./commit.ts"; +import { addressOf, type DocumentRead, namespaceOf, readAt, recordOfEntry, SCOPE_NAMESPACES } from "./documents.ts"; +import { type Condition, type StorageQueries, storageQueries, type TableQueries } from "./queries.ts"; +import { Conversations, Entries, EntryHeads, encodeIndexed, RecordIds, Submissions, Tasks } from "./tables.ts"; + +type StoredTask = TaskRecord; +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** + * A guarded read's result: its data, or its request error, which is thrown after the guarded read (slice 6b §1.4), + * never inside it. + */ +type Read = { readonly kind: "found"; readonly value: T } | { readonly kind: "refused"; readonly message: string }; + +/** The id condition of a table scan from `start`. */ +function scanFrom(start: ScanStart): Condition { + return start.order === "ascending" + ? ["id", "gt", start.after ?? -1] + : ["id", "lt", start.after ?? Number.MAX_SAFE_INTEGER]; +} + +function page }>( + values: readonly T[], + limit: number, + order: ScanOrder, +): Page { + const items = values.slice(0, limit); + if (values.length <= limit) return { items }; + return { items, next: nextCursor(items.at(-1)!.id, order) }; +} + +function cursorId(cursor: Cursor | undefined): number | undefined { + const after = cursor?.after; + if (after === undefined) return undefined; + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); + return after; +} + +/** + * Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): its records as declared tables under an owner name, its + * documents in the store's documents engine, its ids from the store's one allocator, each commit call as one series + * of the store's commit queue (one backend transaction, one seq per batch), published in the host's batch. Several may share a store under distinct owner names + * (claims); the app owns the store, and `close` leaves it open. + */ +export class StoreStorage implements Storage { + readonly #store: ChordStore; + readonly #owner: string; + readonly #claimant: Claimant; + readonly #tables: CommitTables; + readonly #queries: StorageQueries; + #closed = false; + #closing: Promise | undefined; + /** Calls underway, which `close` waits for before it releases the claims (`#drained` once they reach 0). */ + #underway = 0; + #drained: (() => void) | undefined; + + private constructor(store: ChordStore, owner: string, claimant: Claimant, tables: CommitTables) { + this.#store = store; + this.#owner = owner; + this.#claimant = claimant; + this.#tables = tables; + this.#queries = storageQueries(tables); + } + + /** + * Opens Durable's storage in `store` under `owner` (a lowercase `[a-z0-9@/._~-]` name): claims its tables and its + * document namespaces (a second claim is refused `invalid/args`), creates or migrates its tables, and raises the + * store's allocator above every id it holds (one index read). A failure releases what it claimed. + */ + static async open(store: Store, owner: string, context: Context): Promise { + // Chord's own: `openStore` or `importStore` made it. + const chord = store as ChordStore; + const claimant: Claimant = { label: `StoreStorage(${owner})` }; + try { + const declared = new StoreOwnerTables(chord, owner, claimant); + const tables: CommitTables = { + recordIds: declared.declare(RecordIds), + conversations: declared.declare(Conversations), + entries: declared.declare(Entries), + entryHeads: declared.declare(EntryHeads), + tasks: declared.declare(Tasks), + submissions: declared.declare(Submissions), + }; + for (const namespace of SCOPE_NAMESPACES) chord.claims.documents(owner, namespace, claimant); + await declared.prepare(context); + const highest = await chord.read( + (executor) => + executor.get<{ readonly id: number | null }>( + `SELECT max("id") AS id FROM "${tables.recordIds.sql.table}"`, + ), + context, + ); + if (highest?.id != null) chord.ids.raise(highest.id); + return new StoreStorage(chord, owner, claimant, tables); + } catch (error) { + chord.claims.release(claimant); + throw error; + } + } + + /** + * Commit `batches` as one series of the store (`ChordStore.commitSeries`): one backend transaction, one seq per + * batch, each batch's checks (at staging, then against the state the earlier ones left) before its writes. + */ + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#call(() => { + const store = this.#store; + const raise = (id: number): void => store.ids.raise(id); + const commits: SeriesCommit[] = []; + for (const writes of batches) { + try { + commits.push(new DurableCommit(writes, this.#owner, this.#tables, raise, context)); + } catch (error) { + // Staging runs Durable's checks of the batch itself: a refusal in its turn, the batches before it stored. + commits.push({ check: () => Promise.reject(error), writes: [] }); + break; + } + } + return store.commitSeries(commits, (index, seq) => receive(index, seqFromNumber(seq)), context); + }); + } + + /** + * Run a commit's publication in the attached host's batch (`batch.within`: it joins a running batch, opens one, or + * runs next while one sends): Chord sources over Durable's views are marked in one batch per commit. Plainly without + * a host. + */ + publishing(publish: () => void, context: Context): void { + const batch = this.#store.batch; + if (batch === undefined) publish(); + else batch.within(publish, context); + } + + mintId>(): Promise { + return this.#call(async () => idFromNumber(this.#store.mintId())); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#read((executor) => this.#conversation(executor, id, context), context); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.ownerConversationId !== undefined) { + conditions.push(["owner_conversation_id", "eq", query.ownerConversationId]); + } + if (query.ownerTaskId !== undefined) conditions.push(["owner_task_id", "eq", query.ownerTaskId]); + return this.#scan(this.#queries.conversations, conditions, start, limit, context); + }); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + idOrConversationId: EntryId | ConversationId, + idOrContext: EntryId | Context, + maybeContext?: Context, + ): Promise { + if (maybeContext === undefined) { + const context = idOrContext as Context; + return this.#read((executor) => this.#entry(executor, idOrConversationId as EntryId, context), context); + } + const context = maybeContext; + const id = typeof idOrContext === "number" ? idFromNumber(idOrContext) : undefined; + return this.#reading(() => { + if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID"); + const conversationId = idOrConversationId as ConversationId; + return this.#store + .read((executor) => this.#visibleEntry(executor, conversationId, id, context), context) + .then(told); + }); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#reading(() => + this.#store + .read((executor) => this.#latestHeadMarker(executor, conversationId, atOrBeforeEntryId, context), context) + .then(told), + ); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const { order, after } = scanStart(query.order, cursor, "descending"); + const read = + order === "ascending" + ? (executor: SqliteExecutor) => this.#entriesAscending(executor, query, limit, after, context) + : (executor: SqliteExecutor) => this.#entriesDescending(executor, query, limit, after, context); + return this.#store.read(read, context).then(told); + }); + } + + task(id: TaskId, context: Context): Promise { + const tasks = this.#queries.tasks; + return this.#read(async (executor) => { + const row = await tasks.byId(executor, id); + return row === undefined ? undefined : tasks.record(row, context); + }, context); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.kind !== undefined) conditions.push(["kind", "eq", encodeIndexed(query.kind)]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + if (query.abortRequested !== undefined) + conditions.push(["abort_requested", "eq", query.abortRequested ? 1 : 0]); + if (query.background !== undefined) conditions.push(["background", "eq", query.background ? 1 : 0]); + return this.#scan(this.#queries.tasks, conditions, start, limit, context); + }); + } + + submission(id: SubmissionId, context: Context): Promise { + const submissions = this.#queries.submissions; + return this.#read(async (executor) => { + const row = await submissions.byId(executor, id); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + return this.#scan(this.#queries.submissions, conditions, start, limit, context); + }); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + const submissions = this.#queries.submissions; + const conditions: Condition[] = [ + ["conversation_id", "eq", conversationId], + ["request_id", "eq", encodeIndexed(requestId)], + ]; + return this.#read(async (executor) => { + const row = await submissions.first(executor, conditions, "ascending"); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#read(async (executor) => { + const entry = await findDocumentEntry(executor, addressOf(address, this.#owner), at); + return entry === undefined ? undefined : recordOfEntry(entry); + }, context); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + // The record and its revisions must come from one committed state (S4). + return this.#reading(() => + this.#store.readTransaction((executor) => readAt(executor, id, at, this.#owner), context).then(documentOf), + ); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const after = cursorId(cursor) ?? -1; + const scope = { + owner: this.#owner, + namespace: namespaceOf(query.scope), + ...(query.kind === undefined ? {} : { kind: encodeIndexed(query.kind) }), + }; + return this.#store + .read((executor) => listDocumentEntries(executor, scope, query.at, after, limit + 1), context) + .then((found) => page(found.map(recordOfEntry), limit, "ascending")); + }); + } + + /** + * Refuses new calls, waits for its commits underway, then releases its claims; the store stays open (the app's), + * and finishes the reads underway, which change nothing. + */ + close(_context: Context): Promise { + if (this.#closing === undefined) { + this.#closed = true; + this.#closing = (async () => { + if (this.#underway > 0) { + await new Promise((resolve) => { + this.#drained = resolve; + }); + } + this.#store.claims.release(this.#claimant); + })(); + } + return this.#closing; + } + + /** Runs one commit or mint of this storage: refused once it closed, counted until it settles (`close` waits). */ + async #call(run: () => Promise): Promise { + if (this.#closed) throw new Error("StoreStorage is closed"); + this.#underway++; + try { + return await run(); + } finally { + if (--this.#underway === 0) this.#drained?.(); + } + } + + /** + * A call that is one guarded read of the store: refused once closed; not waited for by `close` (a read changes + * nothing, and the store, which stays open, finishes it). + */ + #read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#reading(() => this.#store.read(read, context)); + } + + /** + * A read call: `read` prepares its guarded read (a request error thrown then, before it) and returns it. Refused + * once closed; one promise per call besides the guard's (no async wrapper). + */ + #reading(read: () => Promise): Promise { + try { + if (this.#closed) throw new Error("StoreStorage is closed"); + return read(); + } catch (error) { + return Promise.reject(error); + } + } + + async #conversation( + executor: SqliteExecutor, + id: ConversationId, + context: Context, + ): Promise { + const conversations = this.#queries.conversations; + const row = await conversations.byId(executor, id); + return row === undefined ? undefined : conversations.record(row, context); + } + + #entry(executor: SqliteExecutor, id: EntryId, context: Context): Promise { + const entries = this.#queries.entries; + return entries.byId(executor, id).then((row) => { + if (row === undefined) return undefined; + return { entry: entries.record(row, context), commitSeq: seqFromNumber(row.commit_seq!) }; + }); + } + + /** Entry `id` when visible through `conversationId`'s fork chain (`SqliteStorage.readEntry`). */ + async #visibleEntry( + executor: SqliteExecutor, + conversationId: ConversationId, + id: EntryId, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + const stored = await this.#entry(executor, id, context); + if (stored === undefined) return NOTHING_FOUND; + let upper = Number.POSITIVE_INFINITY; + while (conversation.id !== stored.entry.conversationId) { + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return stored.entry.id > upper ? NOTHING_FOUND : { kind: "found", value: stored }; + } + + async #latestHeadMarker( + executor: SqliteExecutor, + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + let upper: number | undefined = atOrBeforeEntryId; + while (true) { + const { latestHead } = this.#queries; + const head = + upper === undefined + ? await executor.get<{ readonly id: number }>(latestHead.any, conversation.id) + : await executor.get<{ readonly id: number }>(latestHead.atOrBefore, conversation.id, upper); + if (head !== undefined) { + // Written with its entry, in the same commit. + const row = (await this.#queries.entries.byId(executor, head.id))!; + const value = this.#queries.entries.record(row, context); + return { kind: "found", value }; + } + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + } + + /** Newest first: each conversation of the fork chain up to its fork point (`SqliteStorage.readEntries`). */ + async #entriesDescending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1); + const values: EntryRecord[] = []; + while (true) { + const range = this.#range(conversation.id, query.minEntryId, upper); + const rows = await this.#queries.entries.select(executor, range, "descending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit || conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return { kind: "found", value: page(values, limit, "descending") }; + } + + /** Oldest first: the fork chain's segments from the root forward, each up to its fork point. */ + async #entriesAscending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = []; + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + while (true) { + segments.push({ conversationId: conversation.id, upper }); + if (conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + let lower: number | undefined = query.minEntryId; + if (after !== undefined) lower = Math.max(lower ?? after + 1, after + 1); + const values: EntryRecord[] = []; + for (const segment of segments.reverse()) { + const range = this.#range(segment.conversationId, lower, segment.upper); + const rows = await this.#queries.entries.select(executor, range, "ascending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit) break; + } + return { kind: "found", value: page(values, limit, "ascending") }; + } + + /** One conversation's entries with ids from `lower` to `upper` (inclusive, each optional). */ + #range(conversationId: ConversationId, lower: number | undefined, upper: number | undefined): Condition[] { + const conditions: Condition[] = [["conversation_id", "eq", conversationId]]; + if (lower !== undefined) conditions.push(["id", "gte", lower]); + if (upper !== undefined) conditions.push(["id", "lte", upper]); + return conditions; + } + + /** A page of a table scan: up to `limit` records of `queries`' table whose `conditions` hold, from `start`. */ + #scan }>( + queries: TableQueries, + conditions: readonly Condition[], + start: ScanStart, + limit: number, + context: Context, + ): Promise> { + return this.#store + .read((executor) => queries.select(executor, conditions, start.order, limit + 1), context) + .then((rows) => { + const values: T[] = []; + queries.records(rows, values, context); + return page(values, limit, start.order); + }); + } +} + +/** A read that found nothing. */ +const NOTHING_FOUND: Read = { kind: "found", value: undefined }; + +function unknownConversation(id: ConversationId): Read { + return { kind: "refused", message: `Unknown conversation: ${id}` }; +} + +/** A read's data, its request error thrown now that the guarded read returned (slice 6b §1.4). */ +function told(read: Read): T { + if (read.kind === "refused") throw new StorageRequestError(read.message); + return read.value; +} + +/** A document read's result, a record that keeps no history at the point thrown as Durable's request error. */ +function documentOf(read: DocumentRead): StoredDocument | undefined { + if (read.kind === "current_only") + throw new StorageRequestError(`Document ${read.id} does not retain historical content`); + return read.document; +} diff --git a/packages/durable/src/storage/store/tables.ts b/packages/durable/src/storage/store/tables.ts new file mode 100644 index 000000000..49523292d --- /dev/null +++ b/packages/durable/src/storage/store/tables.ts @@ -0,0 +1,122 @@ +// Durable's records as declared tables of the store (slice 6b D2, §2.2): every row carries its record's indexed columns +// and the whole record as one `json(...)` cell. Indexed strings that must stay lossless (`kind`, `request_id`) hold +// Durable's JSON encoding; index columns are never `opt`, so absent values take sentinels no encoded value can be +// (`0` for an absent owner, which is never an id; `""` for an absent request id: an encoded string starts with `"`). + +import type { JsonValue } from "@earendil-works/chord"; +import { defineTable, int, json, str, unknown } from "@earendil-works/chord/next"; +import type { ConversationRecord, EntryRecord, SubmissionRecord, TaskRecord } from "../../types.ts"; + +/** A record as one `json(...)` cell. */ +const record = () => json(unknown(), { version: 1 }); + +/** Which record type owns an id: Durable's one namespace across its five record types. */ +export const RecordIds = defineTable("record_ids", { + version: 1, + columns: { id: int(), record_type: str() }, + key: ["id"], +}); + +export const Conversations = defineTable("conversations", { + version: 1, + columns: { id: int(), owner_conversation_id: int(), owner_task_id: int(), record: record() }, + key: ["id"], + indexes: { by_owner_conversation: ["owner_conversation_id"], by_owner_task: ["owner_task_id"] }, +}); + +export const Entries = defineTable("entries", { + version: 1, + columns: { id: int(), conversation_id: int(), commit_seq: int(), record: record() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +/** + * The entries with a head marker, by conversation: what `SqliteStorage`'s partial index on `head IS NOT NULL` is, + * as rows written only with such an entry, so a commit of an entry without one updates no second index. + */ +export const EntryHeads = defineTable("entry_heads", { + version: 1, + columns: { id: int(), conversation_id: int() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +export const Tasks = defineTable("tasks", { + version: 1, + columns: { + id: int(), + conversation_id: int(), + kind: str(), + status: str(), + abort_requested: int(), + background: int(), + record: record(), + }, + key: ["id"], + indexes: { + by_status: ["status"], + by_conversation: ["conversation_id"], + by_kind: ["kind"], + by_abort_requested: ["abort_requested"], + by_background: ["background"], + }, +}); + +export const Submissions = defineTable("submissions", { + version: 1, + columns: { id: int(), conversation_id: int(), request_id: str(), status: str(), record: record() }, + key: ["id"], + indexes: { + by_request: ["conversation_id", "request_id"], + by_conversation: ["conversation_id"], + by_status: ["status"], + }, +}); + +/** Durable's JSON encoding of an indexed string: lossless whatever the string (lone surrogates, NUL). */ +export const encodeIndexed = (value: string): string => JSON.stringify(value); + +/** The row of a conversation record. */ +export function conversationRow(value: ConversationRecord): object { + return { + id: value.id, + owner_conversation_id: value.owner?.conversationId ?? 0, + owner_task_id: value.owner?.taskId ?? 0, + record: value, + }; +} + +/** The row of an entry record; `commit_seq` is stamped by the step. */ +export function entryRow(value: EntryRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + commit_seq: 0, + record: value, + }; +} + +/** The row of a task record. */ +export function taskRow(value: TaskRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + kind: encodeIndexed(value.kind), + status: value.state.status, + abort_requested: value.abortRequested ? 1 : 0, + background: value.background ? 1 : 0, + record: value, + }; +} + +/** The row of a submission record. */ +export function submissionRow(value: SubmissionRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + request_id: value.requestId === undefined ? "" : encodeIndexed(value.requestId), + status: value.status, + record: value, + }; +} diff --git a/packages/durable/src/testing/commit-writes.ts b/packages/durable/src/testing/commit-writes.ts new file mode 100644 index 000000000..7d655294d --- /dev/null +++ b/packages/durable/src/testing/commit-writes.ts @@ -0,0 +1,18 @@ +// One batch through `Storage.commit()`, for tests and suites that seed or write a storage directly. + +import type { Context } from "@earendil-works/chord"; +import type { Seq, Storage, StorageWrite } from "../types.ts"; + +/** Commits `writes` as one batch and resolves with its seq; rejects with the refusal if the storage refused it. */ +export async function commitWrites(storage: Storage, writes: readonly StorageWrite[], context: Context): Promise { + let stored: Seq | undefined; + const result = await storage.commit( + [writes], + (_index, seq) => { + stored = seq; + }, + context, + ); + if (result.stored === 0) throw result.refused; + return stored!; +} diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index a2c7d43ba..596ea95ad 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -2,12 +2,12 @@ import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context"; import { type ExecutionEnv, + type FileChange, type FileInfo, getOrThrow, type ListFilesOptions, type Result, type ShellOutputInfo, - type WatchChange, type WatchTarget, } from "../env/index.ts"; import type { EnvConformanceCase, EnvConformanceOptions } from "./types.ts"; @@ -63,9 +63,9 @@ async function listed( } /** Whether a change reports `path`: an overflow, or a reported path at or above it. */ -function covers(change: WatchChange, path: string): boolean { - if ("overflow" in change) return true; - if (!("paths" in change)) return false; +function covers(change: FileChange, path: string): boolean { + if (change.kind === "overflow") return true; + if (change.kind !== "paths") return false; return change.paths.some( (reported) => path === reported || path.startsWith(`${reported}/`) || path.startsWith(`${reported}\\`), ); @@ -76,13 +76,13 @@ async function watching( env: ExecutionEnv, targets: readonly WatchTarget[], run: (helpers: { - readonly changes: WatchChange[]; + readonly changes: FileChange[]; /** Wait up to three seconds for a change, after the call, that reports `path`. */ expectChange(path: string, change: () => Promise): Promise; absolute(path: string): Promise; }) => Promise, ): Promise { - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch(targets, (change) => changes.push(change), context)); try { const absolute = async (path: string) => getOrThrow(await env.absolutePath(path, context)); @@ -103,7 +103,7 @@ async function watching( }, }); } finally { - await watcher.close(context); + await watcher.stop(context); } } @@ -346,7 +346,7 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }); const hidden = [await absolute("skills/node_modules"), await absolute("skills/.SKILL.md.tmp")]; for (const change of changes) { - if (!("paths" in change)) continue; + if (change.kind !== "paths") continue; for (const path of change.paths) { assert.ok( !hidden.some((excluded) => path === excluded || path.startsWith(excluded)), @@ -383,12 +383,12 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }); }), - watchCase("watch stops reporting once closed", async (env) => { - const changes: WatchChange[] = []; + watchCase("watch stops reporting once stopped", async (env) => { + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch([{ path: "file.txt" }], (change) => changes.push(change), context)); assert.ok(watcher.mode === "native" || watcher.mode === "polling", "watcher reports its mode"); - await watcher.close(context); - await watcher.close(context); + await watcher.stop(context); + await watcher.stop(context); getOrThrow(await env.writeFile("file.txt", "x", context)); await new Promise((resolve) => setTimeout(resolve, 300)); assert.deepEqual(changes, []); diff --git a/packages/durable/src/testing/index.ts b/packages/durable/src/testing/index.ts index fdaa421f5..394fd24eb 100644 --- a/packages/durable/src/testing/index.ts +++ b/packages/durable/src/testing/index.ts @@ -1,4 +1,5 @@ export { createExpectAssertions, type ExpectLike } from "./assertions.ts"; +export { commitWrites } from "./commit-writes.ts"; export { createEnvConformance } from "./env-conformance.ts"; export { registerEnvConformance, registerStorageConformance, type StorageConformanceRunner } from "./runner.ts"; export { diff --git a/packages/durable/src/testing/storage-benchmark.ts b/packages/durable/src/testing/storage-benchmark.ts index d8e0e3ad4..9acef51c9 100644 --- a/packages/durable/src/testing/storage-benchmark.ts +++ b/packages/durable/src/testing/storage-benchmark.ts @@ -13,9 +13,7 @@ import type { TaskRecord, } from "../types.ts"; import { ROOT_CONVERSATION_ID } from "../types.ts"; - -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; +import { commitWrites } from "./commit-writes.ts"; export type StorageBenchmarkScale = { readonly name: string; @@ -92,7 +90,7 @@ export async function seedStorageBenchmark( storage: Storage, scale: StorageBenchmarkScale = TIMING_SCALE, ): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); let firstEntryId: EntryId | undefined; for (let start = 0; start < scale.entryCount; start += BATCH_SIZE) { @@ -111,7 +109,7 @@ export async function seedStorageBenchmark( }, }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } for (let start = 0; start < scale.taskCount; start += BATCH_SIZE) { @@ -119,7 +117,7 @@ export async function seedStorageBenchmark( for (let index = start; index < Math.min(start + BATCH_SIZE, scale.taskCount); index++) { writes.push({ type: "task", value: task(await storage.mintId>(), index) }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } let exactDocumentId: DocumentId | undefined; @@ -134,7 +132,7 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { index, text: "x".repeat(128) } }, }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } const replayEntries = await Promise.all( @@ -143,7 +141,8 @@ export async function seedStorageBenchmark( return { tail, id }; }), ); - await storage.commit( + await commitWrites( + storage, replayEntries.map(({ tail, id }) => ({ type: "document.create" as const, record: { @@ -156,11 +155,11 @@ export async function seedStorageBenchmark( }, content: { kind: "base" as const, version: 1, value: { count: 0, text: "x".repeat(64) } }, })), - adopt, BACKGROUND_CONTEXT, ); for (let count = 1; count <= REPLAY_TAILS.at(-1)!; count++) { - await storage.commit( + await commitWrites( + storage, replayEntries .filter(({ tail }) => count <= tail) .map(({ id }) => ({ @@ -168,7 +167,6 @@ export async function seedStorageBenchmark( id, content: { kind: "delta" as const, version: 1, ops: [["s", ["count"], count] as const] }, })), - adopt, BACKGROUND_CONTEXT, ); } @@ -181,7 +179,8 @@ export async function seedStorageBenchmark( history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -189,12 +188,12 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, BACKGROUND_CONTEXT, ); let ancientAt: Seq | undefined; for (let count = 1; count <= HISTORY_SEGMENT_LENGTH; count++) { - ancientAt = await storage.commit( + ancientAt = await commitWrites( + storage, [ { type: "document.change", @@ -202,11 +201,11 @@ export async function seedStorageBenchmark( content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, BACKGROUND_CONTEXT, ); } - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -214,13 +213,13 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { count: HISTORY_SEGMENT_LENGTH } }, }, ], - adopt, BACKGROUND_CONTEXT, ); if (ancientAt === undefined) throw new Error("Benchmark history seed produced no commits"); let recentAt = ancientAt; for (let count = HISTORY_SEGMENT_LENGTH + 1; count <= HISTORY_SEGMENT_LENGTH * 2; count++) { - recentAt = await storage.commit( + recentAt = await commitWrites( + storage, [ { type: "document.change", @@ -228,7 +227,6 @@ export async function seedStorageBenchmark( content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, BACKGROUND_CONTEXT, ); } @@ -239,18 +237,19 @@ export async function seedStorageBenchmark( let deepestConversationId = ROOT_CONVERSATION_ID; for (let depth = 0; depth < FORK_DEPTH; depth++) { const conversationId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId, parent: { conversationId: parentConversationId, at: parentAt } }, }, ], - adopt, BACKGROUND_CONTEXT, ); const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId())); - await storage.commit( + await commitWrites( + storage, ids.map((id, index) => ({ type: "entry" as const, value: { @@ -260,7 +259,6 @@ export async function seedStorageBenchmark( data: { depth, index }, }, })), - adopt, BACKGROUND_CONTEXT, ); parentConversationId = conversationId; @@ -400,8 +398,9 @@ export type StorageWriteBenchmark = { /** Seed the common state expected by every write benchmark sample. */ export async function seedStorageWriteBenchmark(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); - await storage.commit( + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); + await commitWrites( + storage, await Promise.all( Array.from({ length: 100 }, async (_, index) => ({ type: "entry" as const, @@ -413,7 +412,6 @@ export async function seedStorageWriteBenchmark(storage: Storage): Promise }, })), ), - adopt, BACKGROUND_CONTEXT, ); } @@ -424,7 +422,8 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ expected: 1, async run(storage) { const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -436,7 +435,6 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ }, }, ], - adopt, BACKGROUND_CONTEXT, ); return 1; @@ -460,7 +458,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ }), ), ); - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); return writes.length; }, }, @@ -495,7 +493,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ content: { kind: "base", version: 1, value: { entryId, taskId } }, }, ]; - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); return writes.length; }, }, diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 58dcd599b..70f295454 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -22,16 +22,14 @@ import { type TaskId, type TaskRecord, } from "../types.ts"; +import { commitWrites } from "./commit-writes.ts"; import type { StorageConformanceAssertions, StorageConformanceCase, StorageConformanceOptions } from "./types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; type StoredTask = TaskRecord; async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); return ROOT_CONVERSATION_ID; } @@ -92,30 +90,27 @@ function assertionFacade(assertions: StorageConformanceAssertions) { } /** - * `storage` with every commit checked against the adoption contract: `adopt` runs exactly once, with the committed - * sequence, before the commit resolves (so before the next commit starts), and never for a commit that rejects. The - * cases fail commits only before they are durable. + * `storage` with every commit checked against the receipt contract: one `receive` per stored batch, in order, with + * increasing seqs, all before the call settles. */ -function checkAdoption(storage: Storage, assertions: StorageConformanceAssertions): Storage { - const commit: Storage["commit"] = async (writes, adopt, commitContext) => { - const adopted: Seq[] = []; +function checkReceipts(storage: Storage, assertions: StorageConformanceAssertions): Storage { + const commit: Storage["commit"] = async (batches, receive, commitContext) => { + const seqs: Seq[] = []; let settled = false; - const recording = (seq: Seq): void => { - assertions.ok(!settled, "Storage.commit() adopted after it settled"); - adopted.push(seq); - adopt(seq); + const recording = (index: number, seq: Seq): void => { + assertions.ok(!settled, "Storage.commit() received a batch after it settled"); + assertions.strictEqual(index, seqs.length); + if (seqs.length > 0) assertions.greaterThan(seq, seqs.at(-1)!); + seqs.push(seq); + receive(index, seq); }; - let seq: Seq; try { - seq = await storage.commit(writes, recording, commitContext); - } catch (error) { + const result = await storage.commit(batches, recording, commitContext); + assertions.strictEqual(seqs.length, result.stored); + return result; + } finally { settled = true; - assertions.deepEqual(adopted, []); - throw error; } - settled = true; - assertions.deepEqual(adopted, [seq]); - return seq; }; return new Proxy(storage, { get(target, property) { @@ -127,7 +122,7 @@ function checkAdoption(storage: Storage, assertions: StorageConformanceAssertion } function createCase(options: StorageConformanceOptions, name: string, test: ConformanceTest): StorageConformanceCase { - return { name, run: () => options.withStorage((storage) => test(checkAdoption(storage, options.assertions))) }; + return { name, run: () => options.withStorage((storage) => test(checkReceipts(storage, options.assertions))) }; } /** Creates runner-independent cases. `withStorage` must call and await its callback exactly once per case. */ @@ -139,50 +134,51 @@ export function createStorageConformance(options: StorageConformanceOptions): re await expect(createRoot(storage)).resolves.toBe(ROOT_CONVERSATION_ID); expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); await expect( - storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context), + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context), ).rejects.toThrow(`ID ${ROOT_CONVERSATION_ID} already belongs to conversation`); }), - createCase(options, "adopts a durable commit once before resolving, and never a failed one", async (storage) => { - const rootId = await createRoot(storage); - const events: string[] = []; - const entryId = await storage.mintId(); - let read: ReturnType | undefined; - const seq = await storage - .commit( - [{ type: "entry", value: entry(entryId, rootId) }], - (adopted) => { - events.push(`adopt ${adopted}`); - // The batch is durable when it is adopted: a read started now observes it. + createCase( + options, + "receives a durable commit once before resolving, and never a refused one", + async (storage) => { + const rootId = await createRoot(storage); + const events: string[] = []; + const entryId = await storage.mintId(); + let read: ReturnType | undefined; + let seq: Seq | undefined; + const result = await storage.commit( + [[{ type: "entry", value: entry(entryId, rootId) }]], + (_index, received) => { + seq = received; + events.push(`receive ${received}`); + // The batch is durable when it is received: a read started now observes it. read = storage.entry(entryId, context); }, context, - ) - .then((committed) => { - events.push(`resolve ${committed}`); - return committed; - }); - expect(events).toEqual([`adopt ${seq}`, `resolve ${seq}`]); - expect((await read)?.commitSeq).toBe(seq); + ); + events.push(`resolve ${result.stored}`); + expect(events).toEqual([`receive ${seq}`, "resolve 1"]); + expect((await read)?.commitSeq).toBe(seq); - events.length = 0; - await expect( - storage.commit( - [{ type: "conversation", value: { id: rootId } }], - (adopted) => events.push(`adopt ${adopted}`), + events.length = 0; + const refused = await storage.commit( + [[{ type: "conversation", value: { id: rootId } }]], + (_index, received) => events.push(`receive ${received}`), context, - ), - ).rejects.toThrow(`ID ${rootId} already belongs to conversation`); - expect(events).toEqual([]); + ); + expect(refused.stored).toBe(0); + expect(String(refused.refused)).toBe(`Error: ID ${rootId} already belongs to conversation`); + expect(events).toEqual([]); - const next = await storage.commit( - [{ type: "entry", value: entry(await storage.mintId(), rootId) }], - (adopted) => events.push(`adopt ${adopted}`), - context, - ); - expect(events).toEqual([`adopt ${next}`]); - expect(next).toBeGreaterThan(seq); - }), + const next = await commitWrites( + storage, + [{ type: "entry", value: entry(await storage.mintId(), rootId) }], + context, + ); + expect(next).toBeGreaterThan(seq!); + }, + ), createCase( options, @@ -201,13 +197,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re status: "placed", entry: entryId, }; - const initialSeq = await storage.commit( + const initialSeq = await commitWrites( + storage, [ { type: "entry", value: entry(entryId, rootId, "user", { data: { text: "hello" } }) }, { type: "task", value: task }, { type: "submission", value: input }, ], - adopt, context, ); @@ -225,14 +221,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re }; const doneInput: SubmissionRecord = { ...input, status: "done", answer: transientEntryId }; await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: runningTask }, { type: "submission", value: doneInput }, { type: "entry", value: entry(transientEntryId, rootId, "assistant") }, { type: "conversation", value: { id: rootId } }, ], - adopt, context, ), ).rejects.toThrow(`ID ${rootId} already belongs to conversation`); @@ -240,9 +236,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.task(taskId, context)).toEqual(task); expect(await storage.submission(submissionId, context)).toEqual(input); expect(await storage.entry(transientEntryId, context)).toBeUndefined(); - const afterRollbackSeq = await storage.commit( + const afterRollbackSeq = await commitWrites( + storage, [{ type: "entry", value: entry(await storage.mintId(), rootId, "after-rollback") }], - adopt, context, ); expect(afterRollbackSeq).toBeGreaterThan(initialSeq); @@ -270,13 +266,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re reason: "failed", detail, }; - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: storedEntry }, { type: "task", value: storedTask }, { type: "submission", value: storedInput }, ], - adopt, context, ); @@ -313,7 +309,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re const data = JSON.parse( '{"__proto__":{"polluted":false},"constructor":{"label":"stored"},"toString":"value"}', ) as Record; - await storage.commit([{ type: "entry", value: entry(entryId, rootId, "note", { data }) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(entryId, rootId, "note", { data }) }], context); (Reflect.get(data, "__proto__") as Record).polluted = true; (Reflect.get(data, "constructor") as Record).label = "mutated"; @@ -334,7 +330,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re createCase(options, "indexes entries committed out of ID order", async (storage) => { const rootId = await createRoot(storage); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(idFromNumber(30), rootId) }, { type: "entry", value: entry(idFromNumber(10), rootId) }, @@ -345,7 +342,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }), }, ], - adopt, context, ); @@ -360,20 +356,20 @@ export function createStorageConformance(options: StorageConformanceOptions): re const oldestId = await storage.mintId(); const middleId = await storage.mintId(); const newestId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(oldestId, rootId) }, { type: "entry", value: entry(middleId, rootId) }, { type: "entry", value: entry(newestId, rootId) }, ], - adopt, context, ); const first = await storage.scanEntries({ conversationId: rootId }, 2, undefined, context); expect(first.items.map(({ id }) => id)).toEqual([newestId, middleId]); const appendedId = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(appendedId, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(appendedId, rootId) }], context); const second = await storage.scanEntries({ conversationId: rootId }, 2, first.next, context); expect(second.items.map(({ id }) => id)).toEqual([oldestId]); expect(second.next).toBeUndefined(); @@ -383,12 +379,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootId = await createRoot(storage); const secondId = await storage.mintId(); const thirdId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: thirdId } }, { type: "conversation", value: { id: secondId } }, ], - adopt, context, ); @@ -409,7 +405,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const firstId = await storage.mintId(); const secondId = await storage.mintId(); const thirdId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: otherOwnerId } }, { @@ -425,7 +422,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re value: { id: thirdId, owner: { conversationId: otherOwnerId, taskId: firstTaskId } }, }, ], - adopt, context, ); @@ -457,7 +453,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootFirst = await storage.mintId(); const rootForkPoint = await storage.mintId(); const rootExcludedSameCommit = await storage.mintId(); - const rootEntriesSeq = await storage.commit( + const rootEntriesSeq = await commitWrites( + storage, [ { type: "entry", value: entry(rootFirst, rootId) }, { @@ -466,46 +463,46 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "entry", value: entry(rootExcludedSameCommit, rootId) }, ], - adopt, context, ); const childId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId, parent: { conversationId: rootId, at: rootForkPoint } }, }, ], - adopt, context, ); const childForkPoint = await storage.mintId(); const childExcluded = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(childForkPoint, childId, "note") }, { type: "entry", value: entry(childExcluded, childId) }, ], - adopt, context, ); const rootExcludedLater = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(rootExcludedLater, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(rootExcludedLater, rootId) }], context); const grandchildId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: grandchildId, parent: { conversationId: childId, at: childForkPoint } }, }, ], - adopt, context, ); const grandchildHead = await storage.mintId(); const grandchildTail = await storage.mintId(); - const grandchildEntriesSeq = await storage.commit( + const grandchildEntriesSeq = await commitWrites( + storage, [ { type: "entry", @@ -513,11 +510,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "entry", value: entry(grandchildTail, grandchildId) }, ], - adopt, context, ); const childExcludedLater = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(childExcludedLater, childId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(childExcludedLater, childId) }], context); const first = await storage.scanEntries({ conversationId: grandchildId }, 2, undefined, context); expect(first.items.map(({ id }) => id)).toEqual([grandchildTail, grandchildHead]); @@ -628,7 +624,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const conversationId = await storage.mintId(); const taskId = await storage.mintId>(); const submissionId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId } }, { type: "task", value: pendingTask(taskId, rootId) }, @@ -637,7 +634,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re value: { id: submissionId, conversationId: rootId, type: "input", status: "queued" }, }, ], - adopt, context, ); conversationIds.push(conversationId); @@ -680,13 +676,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re const first = { ...pendingTask(firstId, rootId), memos: { winner: "first" } } satisfies StoredTask; const second = { ...pendingTask(secondId, rootId), background: true } satisfies StoredTask; const third = { ...pendingTask(thirdId, rootId), abortRequested: true } satisfies StoredTask; - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: first }, { type: "task", value: second }, { type: "task", value: third }, ], - adopt, context, ); @@ -695,7 +691,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "running", checkpoint: { phase: "effect", attempt: 1 } }, abortRequested: true, }; - await storage.commit([{ type: "task", value: running }], adopt, context); + await commitWrites(storage, [{ type: "task", value: running }], context); expect(await storage.task(firstId, context)).toEqual(running); const terminal: StoredTask = { id: firstId, @@ -707,7 +703,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re background: false, abortRequested: true, }; - await storage.commit([{ type: "task", value: terminal }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminal }], context); expect(await storage.task(firstId, context)).toEqual(terminal); const pendingPage = await storage.scanTasks({ status: "pending" }, 1, undefined, context); @@ -743,7 +739,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "completing", outcome: { status: "failed", error: { message: "held" } } }, }; const writes = [owner, waiting, completing].map((value) => ({ type: "task", value }) as const); - await storage.commit(writes, adopt, context); + await commitWrites(storage, writes, context); expect(await storage.task(waitingId, context)).toEqual(waiting); expect(await storage.task(completingId, context)).toEqual(completing); const scan = async (status: StoredTask["state"]["status"]) => @@ -755,7 +751,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re ...completing, state: { status: "terminal", outcome: completing.state.outcome! }, }; - await storage.commit([{ type: "task", value: terminal }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminal }], context); expect(await scan("completing")).toEqual([]); expect(await scan("terminal")).toEqual([terminal]); }), @@ -768,13 +764,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "sleeping", checkpoint: { phase: "next" }, until: 61_000 }, memos: { kept: true }, }; - await storage.commit([{ type: "task", value: sleeping }], adopt, context); + await commitWrites(storage, [{ type: "task", value: sleeping }], context); expect(await storage.task(sleepingId, context)).toEqual(sleeping); const scan = async (status: StoredTask["state"]["status"]) => (await storage.scanTasks({ status }, 10, undefined, context)).items; expect(await scan("sleeping")).toEqual([sleeping]); const running: StoredTask = { ...sleeping, state: { status: "running", checkpoint: { phase: "next" } } }; - await storage.commit([{ type: "task", value: running }], adopt, context); + await commitWrites(storage, [{ type: "task", value: running }], context); expect(await scan("sleeping")).toEqual([]); expect(await scan("running")).toEqual([running]); }), @@ -785,7 +781,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re async (storage) => { const rootId = await createRoot(storage); const secondConversationId = await storage.mintId(); - await storage.commit([{ type: "conversation", value: { id: secondConversationId } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: secondConversationId } }], context); const firstId = await storage.mintId(); const secondId = await storage.mintId(); const otherConversationId = await storage.mintId(); @@ -810,13 +806,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re type: "input", status: "queued", }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: first }, { type: "submission", value: second }, { type: "submission", value: otherConversation }, ], - adopt, context, ); expect(await storage.submissionByRequest(rootId, "same", context)).toEqual(first); @@ -827,7 +823,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re status: "placed", entry: await storage.mintId(), }; - await storage.commit([{ type: "submission", value: placedSecond }], adopt, context); + await commitWrites(storage, [{ type: "submission", value: placedSecond }], context); expect(await storage.submission(secondId, context)).toEqual(placedSecond); expect(await storage.submissionByRequest(rootId, "other", context)).toEqual(placedSecond); @@ -874,12 +870,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re type: "write", status: "queued", }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: queuedDone }, { type: "submission", value: queuedFailed }, ], - adopt, context, ); @@ -894,12 +890,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re reason: "closed", detail: { retryable: false }, }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: done }, { type: "submission", value: unanswered }, ], - adopt, context, ); expect(await storage.submission(doneId, context)).toEqual(done); @@ -908,6 +904,54 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.submissionByRequest(rootId, "passive-failed", context)).toEqual(unanswered); }), + createCase(options, "stores continue submissions, which have no entry and are never queued", async (storage) => { + const rootId = await createRoot(storage); + const doneId = await storage.mintId(); + const failedId = await storage.mintId(); + const placedDone: SubmissionRecord = { + id: doneId, + conversationId: rootId, + requestId: "continue-done", + type: "continue", + status: "placed", + }; + const placedFailed: SubmissionRecord = { + id: failedId, + conversationId: rootId, + requestId: "continue-failed", + type: "continue", + status: "placed", + }; + await commitWrites( + storage, + [ + { type: "submission", value: placedDone }, + { type: "submission", value: placedFailed }, + ], + context, + ); + expect(await storage.submission(doneId, context)).toEqual(placedDone); + expect((await storage.scanSubmissions({ status: "placed" }, 10, undefined, context)).items).toEqual([ + placedDone, + placedFailed, + ]); + + const done: SubmissionRecord = { ...placedDone, status: "done", answer: await storage.mintId() }; + const unanswered: SubmissionRecord = { ...placedFailed, status: "unanswered", reason: "nothing_to_answer" }; + await commitWrites( + storage, + [ + { type: "submission", value: done }, + { type: "submission", value: unanswered }, + ], + context, + ); + expect(await storage.submission(doneId, context)).toEqual(done); + expect(await storage.submissionByRequest(rootId, "continue-done", context)).toEqual(done); + expect(await storage.submission(failedId, context)).toEqual(unanswered); + expect(await storage.submissionByRequest(rootId, "continue-failed", context)).toEqual(unanswered); + }), + createCase(options, "reconstructs rewindable documents and preserves half-open incarnations", async (storage) => { const rootId = await createRoot(storage); const firstId = await storage.mintId(); @@ -919,7 +963,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re fork: "asOf", } satisfies DocumentCreate; const initial: JsonObject = { items: ["a"], nested: { count: 1 } }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "document.create", @@ -927,7 +972,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: initial }, }, ], - adopt, context, ); const appended = ["b"]; @@ -935,9 +979,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re ["p", ["items"], 1, 0, appended], ["s", ["nested", "count"], 2], ]; - const changedAt = await storage.commit( + const changedAt = await commitWrites( + storage, [{ type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops } }], - adopt, context, ); @@ -957,7 +1001,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re nested: { count: 2 }, }); - const checkpointAt = await storage.commit( + const checkpointAt = await commitWrites( + storage, [ { type: "document.change", @@ -965,10 +1010,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 2, value: { items: ["checkpoint"], nested: { count: 3 } } }, }, ], - adopt, context, ); - const replacedAt = await storage.commit( + const replacedAt = await commitWrites( + storage, [ { type: "document.change", @@ -980,7 +1025,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); expect(await storage.document(firstId, changedAt, context)).toMatchObject({ @@ -999,7 +1043,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect((await storage.document(firstId, "current", context))?.deltasSinceBase).toBe(1); const secondId = await storage.mintId(); - const retiredAt = await storage.commit( + const retiredAt = await commitWrites( + storage, [ { type: "document.create", @@ -1013,7 +1058,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 2, ops: [["s", ["retiring"], true]] }, }, ], - adopt, context, ); const address = { kind: firstRecord.kind, scope: firstRecord.scope }; @@ -1050,9 +1094,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re revision: 0, rows: Array.from({ length: 512 }, (_, value) => ({ value, stable: `row-${value}` })), }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: initial } }], - adopt, context, ); const beforeReplacement = structuredClone(initial); @@ -1061,7 +1105,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const index = (revision * 17) % beforeReplacement.rows.length; beforeReplacement.rows[index]!.value = -revision; beforeReplacement.revision = revision; - beforeReplacementAt = await storage.commit( + beforeReplacementAt = await commitWrites( + storage, [ { type: "document.change", @@ -1076,7 +1121,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); } @@ -1086,7 +1130,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re rows: Array.from({ length: 512 }, (_, value) => ({ value: 10_000 + value, stable: `new-${value}` })), }; const replacementSnapshot = structuredClone(replacement); - const replacementAt = await storage.commit( + const replacementAt = await commitWrites( + storage, [ { type: "document.change", @@ -1094,7 +1139,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 1, ops: [["r", replacement]] }, }, ], - adopt, context, ); replacement.rows[0]!.value = -999; @@ -1104,7 +1148,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const index = (revision * 19) % current.rows.length; current.rows[index]!.value = -revision; current.revision = revision; - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1119,7 +1164,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); } @@ -1140,12 +1184,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootId = await createRoot(storage); const childId = await storage.mintId(); const secondChildId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId } }, { type: "conversation", value: { id: secondChildId } }, ], - adopt, context, ); const sourceId = await storage.mintId(); @@ -1156,7 +1200,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "document.create", @@ -1164,10 +1209,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 2, value: { count: 1, rows: [{ value: "base" }] } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1182,7 +1227,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); const historicalCopyId = await storage.mintId(); @@ -1195,7 +1239,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "rewindable", fork: "asOf", }); - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1214,7 +1259,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: retiredCopyId }, ], - adopt, context, ); expect(await storage.document(historicalCopyId, "current", context)).toMatchObject({ @@ -1227,7 +1271,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re }); expect(await storage.document(retiredCopyId, "current", context)).toBeUndefined(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1236,7 +1281,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: sourceId }, ], - adopt, context, ); expect((await storage.document(currentCopyId, "current", context))?.value).toEqual({ @@ -1253,7 +1297,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "latest", fork: "current", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1261,10 +1306,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 4, value: { retained: "copy" } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1276,10 +1321,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re source: { id: latestSourceId, at: "current" }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1288,7 +1333,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: latestSourceId }, ], - adopt, context, ); expect(await storage.document(latestCopyId, "current", context)).toMatchObject({ @@ -1299,7 +1343,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const conflictId = await storage.mintId(); let conflictError: unknown; try { - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1308,7 +1353,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: currentCopyId }, ], - adopt, context, ); } catch (error) { @@ -1325,7 +1369,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const mismatchId = await storage.mintId(); let mismatchError: unknown; try { - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1333,7 +1378,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re source: { id: currentCopyId, at: "current" }, }, ], - adopt, context, ); } catch (error) { @@ -1355,19 +1399,19 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "session.settings", scope: { kind: "session" }, } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } }], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] } }], - adopt, context, ); - const migratedAt = await storage.commit( + const migratedAt = await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 2, value: { count: 3 } } }], - adopt, context, ); expect(await storage.document(id, "current", context)).toMatchObject({ version: 2, value: { count: 3 } }); @@ -1376,14 +1420,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re ); await expect( - storage.commit( + commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 4]] } }], - adopt, context, ), ).rejects.toThrow("version transition requires a base"); expect((await storage.document(id, "current", context))?.value).toEqual({ count: 3 }); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(await storage.document(id, "current", context)).toBeUndefined(); }, ), @@ -1397,7 +1441,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const taskSingletonId = await storage.mintId(); const taskFamilyId = await storage.mintId(); const taskOtherKindId = await storage.mintId(); - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, rootId) }, { @@ -1461,7 +1506,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { owner: "other" } }, }, ], - adopt, context, ); @@ -1540,13 +1584,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "singleton", scope: { kind: "session" }, } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { value: 1 } } }], - adopt, context, ); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -1555,7 +1600,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops: [] } }, ], - adopt, context, ), ).rejects.toThrow("already has a current incarnation"); @@ -1563,7 +1607,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.document(secondId, "current", context)).toBeUndefined(); const emptyId = await storage.mintId(); - const emptyAt = await storage.commit( + const emptyAt = await commitWrites( + storage, [ { type: "document.create", @@ -1579,7 +1624,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: emptyId }, ], - adopt, context, ); expect(await storage.document(emptyId, "current", context)).toBeUndefined(); @@ -1598,6 +1642,30 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, ), + createCase( + options, + "refuses a creation and a change of one document in one commit, in either order", + async (storage) => { + await createRoot(storage); + for (const order of ["create first", "change first"] as const) { + const id = await storage.mintId(); + const create: StorageWrite = { + type: "document.create", + record: { id, kind: "twice", key: order, scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { x: 1 } }, + }; + const change: StorageWrite = { + type: "document.change", + id, + content: { kind: "delta", version: 1, ops: [["s", ["x"], 2]] }, + }; + const writes = order === "create first" ? [create, change] : [change, create]; + await expect(commitWrites(storage, writes, context)).rejects.toThrow("more than one content command"); + expect(await storage.document(id, "current", context)).toBeUndefined(); + } + }, + ), + createCase( options, "rolls back record tables and secondary indexes when a document command fails", @@ -1619,20 +1687,21 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "atomic", scope: { kind: "session" }, } satisfies DocumentCreate; - const baselineSeq = await storage.commit( + const baselineSeq = await commitWrites( + storage, [ { type: "task", value: task }, { type: "submission", value: submission }, { type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } }, ], - adopt, context, ); const entryId = await storage.mintId(); const conflictingDocumentId = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", @@ -1649,7 +1718,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).rejects.toThrow("already has a current incarnation"); @@ -1662,7 +1730,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect( (await storage.findDocument({ kind: record.kind, scope: record.scope }, "current", context))?.id, ).toBe(documentId); - const afterRollbackSeq = await storage.commit( + const afterRollbackSeq = await commitWrites( + storage, [ { type: "document.change", @@ -1670,7 +1739,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, }, ], - adopt, context, ); expect(afterRollbackSeq).toBeGreaterThan(baselineSeq); @@ -1689,7 +1757,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const secondKindDocumentId = await storage.mintId(); const firstKeyDocumentId = await storage.mintId(); const secondKeyDocumentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: { ...pendingTask(firstTaskId, rootId), kind: first } }, { type: "task", value: { ...pendingTask(secondTaskId, rootId), kind: second } }, @@ -1734,7 +1803,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { identity: "second key" } }, }, ], - adopt, context, ); @@ -1783,30 +1851,248 @@ export function createStorageConformance(options: StorageConformanceOptions): re createCase(options, "keeps one global record ID namespace and rejects exhausted ID minting", async (storage) => { const rootId = await createRoot(storage); const explicitEntryId = idFromNumber(100); - await storage.commit([{ type: "entry", value: entry(explicitEntryId, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(explicitEntryId, rootId) }], context); expect(await storage.mintId()).toBe(101); await expect( - storage.commit( + commitWrites( + storage, [{ type: "task", value: pendingTask(idFromNumber>(explicitEntryId), rootId) }], - adopt, context, ), ).rejects.toThrow(`ID ${explicitEntryId} already belongs to entry`); - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: entry(idFromNumber(Number.MAX_SAFE_INTEGER), rootId, "last-id") }], - adopt, context, ); await expect(storage.mintId()).rejects.toThrow("ID space is exhausted"); await expect(storage.mintId()).rejects.toThrow("ID space is exhausted"); }), + createCase( + options, + "commits batches in order, each with its seq, checked against the earlier ones", + async (storage) => { + const rootId = await createRoot(storage); + const firstId = await storage.mintId(); + const lastId = await storage.mintId(); + const documentId = await storage.mintId(); + const receipts: [number, Seq][] = []; + let read: ReturnType | undefined; + const result = await storage.commit( + [ + [ + { type: "entry", value: entry(firstId, rootId) }, + { + type: "document.create", + record: { + id: documentId, + kind: "batch.doc", + scope: { kind: "conversation", conversationId: rootId }, + history: "latest", + fork: "initial", + }, + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + // Changes the document the batch before created. + [ + { + type: "document.change", + id: documentId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [{ type: "entry", value: entry(lastId, rootId) }], + ], + (index, seq) => { + receipts.push([index, seq]); + // Receipts follow the last durable batch: a read started at the first observes it. + read ??= storage.entry(lastId, context); + }, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect(receipts.map(([index]) => index)).toEqual([0, 1, 2]); + const [[, first], [, second], [, last]] = receipts as [[number, Seq], [number, Seq], [number, Seq]]; + expect(second).toBeGreaterThan(first); + expect(last).toBeGreaterThan(second); + expect((await read)?.commitSeq).toBe(last); + expect((await storage.entry(firstId, context))?.commitSeq).toBe(first); + expect((await storage.document(documentId, "current", context))?.value).toEqual({ count: 2 }); + const next = await commitWrites( + storage, + [{ type: "entry", value: entry(await storage.mintId(), rootId) }], + context, + ); + expect(next).toBeGreaterThan(last); + }, + ), + + createCase(options, "stores the batches before the first one refused, and none after it", async (storage) => { + const rootId = await createRoot(storage); + const firstId = await storage.mintId(); + const afterId = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [{ type: "entry", value: entry(firstId, rootId) }], + // Refused against the state the first batch left: the ID is taken now. + [ + { type: "task", value: pendingTask(await storage.mintId>(), rootId) }, + { type: "entry", value: entry(firstId, rootId) }, + ], + [{ type: "entry", value: entry(afterId, rootId) }], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${firstId} already belongs to entry`); + expect(receipts).toEqual([0]); + expect(await storage.entry(firstId, context)).toBeDefined(); + expect(await storage.entry(afterId, context)).toBeUndefined(); + expect((await storage.scanTasks({}, 10, undefined, context)).items.map((task) => task.id)).toEqual([]); + + // A refused first batch stores nothing; the storage goes on. + const refused = await storage.commit( + [[{ type: "conversation", value: { id: rootId } }]], + (index) => receipts.push(index), + context, + ); + expect(refused.stored).toBe(0); + expect(String(refused.refused)).toBe(`Error: ID ${rootId} already belongs to conversation`); + expect(receipts).toEqual([0]); + await commitWrites(storage, [{ type: "entry", value: entry(afterId, rootId) }], context); + expect(await storage.entry(afterId, context)).toBeDefined(); + }), + + createCase( + options, + "refuses a later batch its own malformed actions fail, keeping the batches before it", + async (storage) => { + const rootId = await createRoot(storage); + const documentId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: { id: documentId, kind: "staged", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + context, + ); + const firstId = await storage.mintId(); + const secondId = await storage.mintId(); + const afterId = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [{ type: "entry", value: entry(firstId, rootId) }], + // Malformed in itself, whatever is stored: one document retired twice. + [ + { type: "entry", value: entry(secondId, rootId) }, + { type: "document.retire", id: documentId }, + { type: "document.retire", id: documentId }, + ], + [{ type: "entry", value: entry(afterId, rootId) }], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(result.refused).toBeDefined(); + expect(receipts).toEqual([0]); + expect(await storage.entry(firstId, context)).toBeDefined(); + expect(await storage.entry(secondId, context)).toBeUndefined(); + expect(await storage.entry(afterId, context)).toBeUndefined(); + expect((await storage.document(documentId, "current", context))?.value).toEqual({ n: 1 }); + }, + ), + + createCase(options, "resolves a batched copy against the state the earlier batches left", async (storage) => { + const rootId = await createRoot(storage); + const childId = await storage.mintId(); + const sourceId = await storage.mintId(); + const copyId = await storage.mintId(); + const record = (id: DocumentId, conversationId: ConversationId): DocumentCreate => ({ + id, + kind: "batch.copy", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }); + const seqs: Seq[] = []; + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: record(sourceId, rootId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + [ + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [ + { type: "conversation", value: { id: childId } }, + { + type: "document.copy", + record: record(copyId, childId), + source: { id: sourceId, at: "current" }, + }, + ], + ], + (_index, seq) => seqs.push(seq), + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copyId, "current", context))?.value).toEqual({ count: 2 }); + expect((await storage.document(sourceId, seqs[0]!, context))?.value).toEqual({ count: 1 }); + + // A copy whose source the same batch changes is refused before the batch writes anything. + const laterId = await storage.mintId(); + const laterCopyId = await storage.mintId(); + const refused = await storage.commit( + [ + [ + { type: "conversation", value: { id: laterId } }, + { + type: "document.copy", + record: record(laterCopyId, laterId), + source: { id: sourceId, at: "current" }, + }, + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, + }, + ], + ], + () => {}, + context, + ); + expect(refused.stored).toBe(0); + expect(refused.refused).toBeDefined(); + expect(await storage.conversation(laterId, context)).toBeUndefined(); + expect((await storage.document(sourceId, "current", context))?.value).toEqual({ count: 2 }); + }), + createCase(options, "rejects every operation after close", async (storage) => { await createRoot(storage); await storage.close(context); await expect(storage.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("closed"); - await expect(storage.commit([] satisfies StorageWrite[], adopt, context)).rejects.toThrow("closed"); + await expect( + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context), + ).rejects.toThrow("closed"); await expect(storage.mintId()).rejects.toThrow("closed"); }), ]; diff --git a/packages/durable/src/tool-search/tool-search.ts b/packages/durable/src/tool-search/tool-search.ts index ca600ae5e..5d79b007e 100644 --- a/packages/durable/src/tool-search/tool-search.ts +++ b/packages/durable/src/tool-search/tool-search.ts @@ -3,7 +3,7 @@ * and the matches are loaded with `control.addTools`, so the next request offers them. Loading writes the * conversation's `modelTools` (its `add`), so loaded tools stay offered on that branch, also after a compaction. */ -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { defineExtension, defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; import { rankDocuments } from "../search.ts"; @@ -14,12 +14,13 @@ export const DEFAULT_TOOL_SEARCH_LIMIT = 8; /** Fixed, so it does not change while deferred tools are registered, for example as MCP servers connect. */ const TOOL_SEARCH_DESCRIPTION = `# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (\`${TOOL_SEARCH_TOOL_NAME}\`) to search for the required tools. For MCP tool discovery, always use \`${TOOL_SEARCH_TOOL_NAME}\`.`; -const toolSearchSchema = Type.Object({ - query: Type.String({ description: "Search query for deferred tools." }), - limit: Type.Optional( - Type.Number({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` }), - ), -}); +const toolSearchSchema = obj( + { + query: str({ description: "Search query for deferred tools." }), + limit: opt(num({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` })), + }, + { open: true }, +); /** `tool_search` result details: the tools it loaded. */ export type ToolSearchToolDetails = { loaded: string[] }; @@ -45,9 +46,9 @@ function schemaText(schema: unknown, parts: string[]): void { } } -/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */ +/** Search text of a tool: its description and its parameters' names and descriptions; the ranker reads the name. */ function searchText(tool: ToolRegistration): string { - const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + const parts = [tool.description]; schemaText(tool.parameters, parts); return parts.filter((part) => part.trim() !== "").join(" "); } diff --git a/packages/durable/src/tools/bash.ts b/packages/durable/src/tools/bash.ts index 21d1e1d03..211338f16 100644 --- a/packages/durable/src/tools/bash.ts +++ b/packages/durable/src/tools/bash.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { type Static, Type } from "typebox"; +import { bool, num, obj, opt, type Static, str } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts"; @@ -7,23 +7,33 @@ import { requireEnv } from "./env.ts"; const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000; -const bashSchema = Type.Object({ - command: Type.String({ description: "Bash command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +// Open objects, as the tools accepted extra arguments when their schemas were TypeBox's. +const bashSchema = obj( + { + command: str({ description: "Bash command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); -const powershellSchema = Type.Object({ - command: Type.String({ description: "PowerShell command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +const powershellSchema = obj( + { + command: str({ description: "PowerShell command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); /** What a program that calls `bash` or `powershell` receives. */ -const shellStructuredOutputSchema = Type.Object({ - output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), - truncated: Type.Boolean({ description: "Whether earlier output was dropped" }), - fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })), - exitCode: Type.Number(), -}); +const shellStructuredOutputSchema = obj( + { + output: str({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), + truncated: bool({ description: "Whether earlier output was dropped" }), + fullOutputPath: opt(str({ description: "File with the complete output, when it was spilled" })), + exitCode: num(), + }, + { open: true }, +); export type BashToolInput = Static; export type PowerShellToolInput = Static; diff --git a/packages/durable/src/tools/edit.ts b/packages/durable/src/tools/edit.ts index 802150152..e5b9be73d 100644 --- a/packages/durable/src/tools/edit.ts +++ b/packages/durable/src/tools/edit.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { arr, obj, type Static, str } from "@earendil-works/pi-schema"; import type { FileError } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -16,21 +16,27 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const replaceEditSchema = Type.Object({ - oldText: Type.String({ - description: - "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", - }), - newText: Type.String({ description: "Replacement text for this targeted edit." }), -}); +const replaceEditSchema = obj( + { + oldText: str({ + description: + "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", + }), + newText: str({ description: "Replacement text for this targeted edit." }), + }, + { open: true }, +); -const editSchema = Type.Object({ - path: Type.String({ description: "Path to the file to edit (relative or absolute)" }), - edits: Type.Array(replaceEditSchema, { - description: - "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", - }), -}); +const editSchema = obj( + { + path: str({ description: "Path to the file to edit (relative or absolute)" }), + edits: arr(replaceEditSchema, { + description: + "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", + }), + }, + { open: true }, +); export type EditToolInput = Static; type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown }; diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index cede96ef3..de84a2b2f 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { characterEnd } from "@earendil-works/chord/output"; -import { type Static, Type } from "typebox"; +import { lit, num, obj, opt, type Static, str, union } from "@earendil-works/pi-schema"; import { type BinaryReader, type FileInfo, @@ -31,14 +31,26 @@ import { } from "./image-processor.ts"; import { resolveReadToolPath } from "./path-utils.ts"; -const readSchema = Type.Object({ - path: Type.String({ description: "Path to the file to read (relative or absolute)" }), - offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })), - limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })), -}); +const readSchema = obj( + { + path: str({ description: "Path to the file to read (relative or absolute)" }), + offset: opt(num({ description: "Line number to start reading from (1-indexed)" })), + limit: opt(num({ description: "Maximum number of lines to read" })), + }, + { open: true }, +); export type ReadToolInput = Static; +/** + * What programs such as code mode scripts get: the text of a text file, or the image block of an image, which code + * mode's `image()` shows. + */ +const readStructuredOutputSchema = union([ + str(), + obj({ type: lit("image"), data: str(), mimeType: str() }, { open: true }), +]); + export type ReadToolDetails = { /** How the shown text was cut; the text itself is the result content. */ truncation?: Omit; @@ -84,7 +96,7 @@ export type ReadToolOptions = { /** * Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content - * is only file text, or one image, which programs get as its `ImageContent`. + * is only file text, or one image. Programs get the text, or the image's `ImageContent`. */ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration { const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp"; @@ -92,6 +104,7 @@ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration< name: "read", description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, parameters: readSchema, + structuredOutputSchema: readStructuredOutputSchema, async execute(args, api, context) { const { path, offset, limit } = args; const env = requireEnv(api); @@ -187,7 +200,8 @@ async function readImage( const diagnostics: ToolDiagnostic[] = [ { severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") }, ]; - return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics }; + const block = { type: "image" as const, data: image.data, mimeType: image.mimeType }; + return { output: [block], structuredOutput: block, diagnostics }; } /** What the conversation's model takes: whether images, and within which limits. */ @@ -306,6 +320,7 @@ async function readText( return { output: outputText === "" ? [] : [{ type: "text" as const, text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/durable/src/tools/write.ts b/packages/durable/src/tools/write.ts index ee9f724cb..d8c804558 100644 --- a/packages/durable/src/tools/write.ts +++ b/packages/durable/src/tools/write.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { getOrThrow } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -6,10 +6,13 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const writeSchema = Type.Object({ - path: Type.String({ description: "Path to the file to write (relative or absolute)" }), - content: Type.String({ description: "Content to write to the file" }), -}); +const writeSchema = obj( + { + path: str({ description: "Path to the file to write (relative or absolute)" }), + content: str({ description: "Content to write to the file" }), + }, + { open: true }, +); export type WriteToolInput = Static; diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index ebf0ecfd2..027832568 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1,4 +1,4 @@ -import type { Context, Draft, JsonValue, WatchHandle } from "@earendil-works/chord"; +import type { Context, Draft, JsonValue, WatchCall, WatchEnd, WatchHandle } from "@earendil-works/chord"; import type { Op } from "@earendil-works/chord/delta"; import type { Message, Models } from "@earendil-works/pi-ai"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; @@ -142,19 +142,26 @@ export type ReportTask = { readonly conversationId: ConversationId; }; -/** A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call. */ -export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +/** + * A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call, and + * `telemetry` a telemetry adapter call made while a commit is published, which must not fail it. + */ +export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; /** - * What a report to `HarnessOptions.onReport` is about. `extension`, where present, names the extension whose code - * failed, when known; for a task's own code, the one that installed the definition the task ran. - * - `session_failed`: the error failed the Session, which closes: a Storage error, often a platform reset. + * What a report to `HarnessOptions.onReport` is about, by `kind` (`Report` adds the error). `extension`, where present, + * names the extension whose code failed, when known; for a task's own code, the one that installed the definition the + * task ran. + * - `storage_failed`: the error failed the Session, which closes: a Storage error, often a platform reset. * - `task_faulted`: the task ended `faulted`: its phase threw (`reason: "threw"`), or its abort handler was cut off - * `settings.restartLimit` times (`reason: "restart_limit"`). - * - `task_parked`: the restart limit parked the task until `Harness.retryTask()`. + * `settings.restartLimit` times (`reason: "restart_limit"`), as was a phase of a task of an extension's kind. + * - `task_interrupted`: crashes ended the task: it ended itself after being cut off `settings.restartLimit` times in a + * row (`reason: "restart_limit"`; a tool call, a compaction, a generation), or a generation's run ended because the + * host kept stopping while the model answered (`reason: "interrupted"`). `restarts` is the task's count when it ended. * - `migration_failed`: the task's stored state could not be migrated; it is blocked until the registry changes. * - `task_outdated`: a running task keeps running under its old definition, which was uninstalled (`missing_task`) - * or replaced by one that cannot take over its state (`incompatible_task`). + * or replaced by one that cannot take over its state: older than the task (`task_too_old`), or newer without a + * `migrate` (`migration_failed`). * - `task_error`: the task's own `runtime.report()`, or a failure in running it. * - `extension_error`: a hook, a section, a wrapper, or a tool's invalid structured output; with the task that ran it, * or, outside a task, the conversation whose agent was resolved. @@ -162,35 +169,44 @@ export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | * - `listener_error`: a commit, close, watch, view, or document listener threw. */ export type ReportInfo = - | { readonly type: "session_failed" } + | { readonly kind: "storage_failed" } | { - readonly type: "task_faulted"; + readonly kind: "task_faulted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "threw" | "restart_limit"; } - | { readonly type: "task_parked"; readonly task: ReportTask; readonly extension?: string } - | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "task_outdated"; + readonly kind: "task_interrupted"; readonly task: ReportTask; readonly extension?: string; - readonly reason: "missing_task" | "incompatible_task"; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; } - | { readonly type: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "extension_error"; + readonly kind: "task_outdated"; + readonly task: ReportTask; + readonly extension?: string; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; + } + | { readonly kind: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { + readonly kind: "extension_error"; readonly extension?: string; readonly task?: ReportTask; readonly conversationId?: ConversationId; } - | { readonly type: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } - | { readonly type: "listener_error" }; + | { readonly kind: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } + | { readonly kind: "listener_error" }; + +/** One report to `HarnessOptions.onReport`: what it is about (`ReportInfo`, by `kind`) and the `error` reported. */ +export type Report = ReportInfo & { readonly error: unknown }; /** Whose code a task's `runtime.report()` is about when not the task's own: an extension's, or a host callback. */ export type ReportFrom = - | { readonly type: "extension"; readonly extension?: string } - | { readonly type: "host"; readonly callback: HostCallback }; + | { readonly kind: "extension"; readonly extension?: string } + | { readonly kind: "host"; readonly callback: HostCallback }; /** * A host timer, given as `HarnessOptions.setTimer`: call `callback` once, about `delayMs` from now, and return @@ -268,6 +284,8 @@ export interface TaskRuntime extends DocumentObserver * Harness is closing, or, in a run invocation, the task carries an abort mark. A returned state replaces the task's * state in the same commit; returning nothing leaves it unchanged. `tx.createTask()` defaults to the task's * conversation. Inside the callback read only through `tx`: `snapshot()`, `watchDoc()`, `getTask()`, and the other Session and runtime reads may queue on the line the callback holds, and then never resolve. + * `options.persistence` may defer storing a commit that changes only documents (`CommitPersistence`); one that returns + * a state is immediate. */ commit( change: ( @@ -275,6 +293,7 @@ export interface TaskRuntime extends DocumentObserver current: RunningTask, ) => NextTaskState | undefined | Promise | undefined>, context: Context, + options?: CommitOptions, ): Promise; /** Read a durable memo of this task. */ memo(name: string, context: Context): Promise; @@ -305,6 +324,17 @@ export interface TaskRuntime extends DocumentObserver entry(id: EntryId, context: Context): Promise; /** Undefined when the entry is absent, not visible, or has another kind. */ entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** + * Committed raw history of the task's conversation, fork-aware, as `Conversation.entries()`: every entry, also those a + * compaction summary or reset left behind. Entries never change, so a range bounded by entry IDs reads the same + * before and after a compaction, and in a rerun after a crash. + */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; /** Committed raw active transcript and model context, optionally cut off at the visible entry `at`. */ context(conversationId: ConversationId, context: Context, options?: { readonly at?: EntryId }): Promise; /** The Harness clock. */ @@ -472,7 +502,7 @@ type SubmissionRecordBase = { readonly requestId?: string; }; -/** Durable lifecycle of one admitted user input or passive entry write. */ +/** Durable lifecycle of one admitted user input, passive entry write, or continue. */ export type SubmissionRecord = | (SubmissionRecordBase & { readonly type: "input"; @@ -537,6 +567,35 @@ export type SubmissionRecord = readonly reason: string; readonly detail?: JsonValue; } + )) + | (SubmissionRecordBase & { + /** A run started without a user message; it places no entry and is never queued. */ + readonly type: "continue"; + } & ( + | { + /** Owned by an active run. */ + readonly status: "placed"; + readonly entry?: never; + readonly answer?: never; + readonly reason?: never; + readonly detail?: never; + } + | { + /** The run answered. */ + readonly status: "done"; + readonly entry?: never; + readonly answer: EntryId; + readonly reason?: never; + readonly detail?: never; + } + | { + /** The run ended without an answer. */ + readonly status: "unanswered"; + readonly entry?: never; + readonly answer?: never; + readonly reason: string; + readonly detail?: JsonValue; + } )); /** Terminal status staged for a submission; identity, type, and entry come from its current record. */ @@ -680,8 +739,9 @@ type TaskRecordBase = { readonly endedAt?: number; /** * How many times in a row a crash, eviction, or reset cut the task off while it ran, without the task changing its - * state since; open counts them, and a close does not. At `HarnessSettings.restartLimit` the task is parked as blocked - * until `Harness.retryTask()`. + * state since; open counts them, and a close does not. At `HarnessSettings.restartLimit` the task ends (the built-in + * tasks end themselves, visibly; the scheduler faults any other kind), and a generation reads it to charge an + * interrupted model request as a failed attempt. */ readonly restarts?: number; }; @@ -867,20 +927,31 @@ export type StorageWrite = } | { readonly type: "document.retire"; readonly id: DocumentId }; -/** Committed change of one document incarnation. */ +/** + * Committed change of one document incarnation: `type: "document"` with `kind: "change"` (a creation or an update) or + * `kind: "retire"` (this commit retired the incarnation), or a `document.copy`. + */ export type DocumentCommitChange = | { readonly type: "document"; + readonly kind: "change"; readonly record: DocumentRecord; /** Conversation owning the document; task documents derive it from their task record. Undefined only for Session documents. */ readonly conversationId: ConversationId | undefined; - /** Definition version of `value`; absent when this commit retired the incarnation. */ - readonly version: number | undefined; - /** Exact adopted immutable revision, or `null` when this commit retired the incarnation. */ - readonly value: JsonObject | null; - /** Exact adopted operations for an ordinary update; empty for creation and retirement. */ + /** Definition version of `value`. */ + readonly version: number; + /** Exact adopted immutable revision. */ + readonly value: JsonObject; + /** Exact adopted operations for an ordinary update; empty for creation. */ readonly ops: readonly Op[]; } + | { + readonly type: "document"; + readonly kind: "retire"; + readonly record: DocumentRecord; + /** Conversation owning the document; task documents derive it from their task record. Undefined only for Session documents. */ + readonly conversationId: ConversationId | undefined; + } | { /** Definition-free child initialization; consumers hydrate through watch acquisition. */ readonly type: "document.copy"; @@ -897,7 +968,12 @@ export type TableCommitChange = Extract< export type CommitChange = TableCommitChange | DocumentCommitChange; -/** Every immutable change from one successful Session commit. Change order is unspecified. */ +/** + * Every immutable change one Session storage transaction stored, with its `seq`. Change order is unspecified. A commit + * held in memory (`CommitPersistence`) is published once it is stored: by the next commit that reads or writes its + * document, alone (`seq` of its own), or with that commit, whose document change then carries the held operations + * followed by its own. + */ export type CommitPublication = { readonly seq: Seq; readonly changes: readonly CommitChange[]; @@ -905,7 +981,8 @@ export type CommitPublication = { /** * Transaction surface of one Session commit callback. - * Table reads and creation results are trusted immutable values and may be shared with internal commit state. + * Table reads and creation results are trusted immutable values and may be shared with internal commit state; the + * Session may freeze them (its caches and context ranges keep them), so mutating one can throw. */ export interface Tx { conversation(id: ConversationId): Promise; @@ -921,6 +998,8 @@ export interface Tx { scanEntries(query: EntryQuery, limit: number, cursor?: Cursor): Promise>; /** Newest visible entry of the conversation that carries a `head`. */ latestHeadMarker(conversationId: ConversationId): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; + /** ID of the newest entry visible from the conversation; `undefined` when it has none. */ + latestEntryId(conversationId: ConversationId): Promise; scanTasks( query: TaskQuery, limit: number, @@ -956,7 +1035,7 @@ export interface Tx { */ createSubmission(create: SubmissionCreate): Promise; /** - * Settle a queued or placed submission; only a placed input can be answered, and a settled submission stays + * Settle a queued or placed submission; only a placed input or continue can be answered, and a settled submission stays * unchanged. Resolved against this transaction's latest record of the submission, so it works after table writes. * Run tasks settle the inputs they answer. */ @@ -1005,10 +1084,54 @@ export interface Tx { } /** - * Watch of one committed document incarnation. A retirement delivers `[["r", null]]` with value `null`, then ends the - * watch as `retired`; the Session's close ends it as `ended`, its failure as `failed` with the error that failed it. + * Watch of one document address (kind, scope, and key), from the incarnation found at acquisition on. A retirement is a + * call with `retired` set, the last value the watch was given and no ops (a change made in the retiring commit itself + * is not shown: that commit publishes the retirement only); the watch goes on, and a document created at the address + * later arrives as `[["r", value]]`, folded with what is pending. It ends `ended` when its scope's owner ends: a task + * document's watch once the retirement call of the commit that makes its task terminal returned, every watch at Session + * close (a pending retirement dropped); `failed` with the error that failed the Session. */ -export type DocumentWatch = WatchHandle | null>; +export type DocumentWatch = WatchHandle>; + +/** + * A synchronous observer of committed Durable state, for adapters such as a Chord source (`observeDoc()`, + * `Harness.observeConversation()`, `Harness.observeTaskGraph()`), with the shapes of Chord's watch: `advance` gets a + * `WatchCall`, `end` a `WatchEnd`. `advance` gets every change, unfolded, as memory adopts it: for a stored commit at its + * publication once Storage stored it, for a deferred one before it is stored. A view or the graph gets at most one call + * per commit, a document observer one per change at its address (two for a commit that retires and recreates it); the + * task graph also advances outside any commit's publication, on the Session line, when a registry change or a failed + * migration changes a + * `blocked` reason. Durable's failures are final, so every call is a `value` call; a failure ends the observer. Calls + * may come before the attaching promise resolves; their ops apply from `Observation.value`. `advance` must only record + * the call, synchronously and quickly, never await, call the Session, or throw. A throw is reported as + * `listener_error`, detaches the observer, and ends it `listener_error` with the error. `end` is called once: `ended` + * when the observed scope's owner ends (a task document's task, after its retirement change, inside that commit's + * `adopt`), `failed` with "session closed" when the Session closes (an adapter follows the next Session) or with the + * error that failed the Session; never after `detach()`. + */ +export type Observer = { + advance(call: WatchCall, context: Context): void; + end(end: WatchEnd): void; +}; + +/** + * A synchronous observer of stored commits (`Session.observeCommits()`): `advance` gets each `CommitPublication`, `end` + * the Session's close (`failed` with "session closed", or with the error that failed the Session). + */ +export type CommitObserver = { + advance(publication: CommitPublication, context: Context): void; + end(end: WatchEnd): void; +}; + +/** An attached `CommitObserver`: how to detach it. Idempotent: no `advance` or `end` follows it. */ +export type CommitObservation = { detach(): void }; + +/** An attached `Observer`: the value at attachment, which every later `advance` starts from, and how to detach it. */ +export type Observation = { + readonly value: T; + /** Idempotent: no `advance` or `end` follows it, not even for the rest of a commit underway. */ + detach(): void; +}; /** Committed document reads. */ export type DocumentReader = Pick; @@ -1045,15 +1168,36 @@ export interface DocumentObserver { ): Promise | undefined>; } -/** Why a Session ended: `close()`, or a failed Storage call, whose error it carries. */ -export type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +/** Why a Session ended, by `kind`: `close()`, or a failed Storage call, whose error it carries. */ +export type SessionEnd = { readonly kind: "closed" } | { readonly kind: "failed"; readonly error: unknown }; + +/** + * When a commit reaches Storage. `immediate`, the default: before `commit()` resolves. `deferred`: when every write of + * the commit changes an existing conversation document with `history: "latest"` and `fork: "initial"` (no creation, + * migration, or retirement), the Session adopts it in memory, shows it to watches and views, and resolves, but stores it + * later: with the next commit that reads (`tx.doc()`) or writes that document, before the document leaves memory or + * retires, at `close()`, and at the latest about `withinMs` after the commit (default `settings.progress.persistWithinMs`, + * 1,000 ms; `Infinity` waits for one of the others). A commit that writes anything else is immediate. A crash loses what + * is held, so only state whose loss is acceptable, such as streamed progress, should be deferred, and nothing durable may + * depend on it without reading it in a commit. + */ +export type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; + +/** Options of one Session commit. */ +export type CommitOptions = { readonly persistence?: CommitPersistence }; /** Owner of one mutation line, its records, and its tracked documents. */ export interface Session extends DocumentObserver { /** * Run one atomic transaction on the Session mutation line. Inside the callback read only through `tx`: `snapshot()`, `watchDoc()`, `getTask()`, and the other Session and runtime reads may queue on the line the callback holds, and then never resolve. + * `options.persistence` may defer storing it (`CommitPersistence`). The callback may run while an earlier commit is + * being stored: at its first read or write of what an unpublished commit wrote, it waits until that commit is stored + * and published; code before that runs early, so act on what it computes only once the commit resolves. Resolves + * once the commit is stored and published, after every earlier commit's outcome. */ - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; /** Seal admission, settle admitted commits, then close storage. */ close(context: Context): Promise; /** @@ -1061,8 +1205,61 @@ export interface Session extends DocumentObserver { * which closes the Session itself. A failed Session is reopened from Storage; nothing else is left to clean up. */ readonly closed: Promise; - /** Observe complete commits synchronously after adoption. The listener must not block or call Session APIs. */ - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; + /** + * Attach a synchronous `Observer` to one document address, atomically on the Session line, as `watchDoc()` acquires + * a watch: the observation's value includes every commit before it, and `advance` gets every later one, retirements + * and later incarnations included. `undefined` when the address is absent. `context` governs only the attachment: + * cancelled while attaching, the promise rejects and nothing stays attached. + */ + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + /** + * Observe stored commits, each with exactly what it stored, except that a document the commit retires is published + * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Commits are + * published in line order once Storage stored them, a macrotask apart; a publication queues one dispatch, which runs + * a microtask later, before every wait the commit resolves and every abort it queues, and calls the observers + * attached by then; a commit published while none was attached has no dispatch. There is no order relative to watch listeners. `advance` runs synchronously and must not block or + * call Session APIs; a throw is reported as `listener_error`, detaches the observer, and ends it `listener_error`. + * `end` is called once, when the Session closes (`ended`, after its last publication) or fails (`failed`), never + * after `detach()`. Attaches synchronously; the observation has no `value`: commits are events, not a state, and the + * Session does not know Storage's newest `seq` before its own first commit. + */ + observeCommits(observer: CommitObserver): CommitObservation; /** * Observe close synchronously when it begins, also when a failed Storage call closes the Session. The listener must * not block or call Session APIs. @@ -1113,6 +1310,9 @@ export interface Session extends DocumentObserver { ): Promise | undefined>; } +/** What `Storage.commit()` stored: the number of durable batches, and why the next one was refused, if one was. */ +export type CommitResult = { readonly stored: number; readonly refused?: unknown }; + /** * Atomic persistence boundary for Session records. * @@ -1128,12 +1328,29 @@ export interface Session extends DocumentObserver { */ export interface Storage { /** - * Atomically persist one batch and return its sequence. Once the batch is durable, call `adopt(seq)` exactly once, - * synchronously and before this storage's next commit starts, then resolve with `seq`; a commit that fails before it - * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption of the batch, so memory always shows a - * complete prefix of the log; it must not throw. Once resolved, later reads through this storage observe the batch. + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Every batch has at least one write: a commit without + * writes never reaches Storage, so the Session never sends an empty batch. Stops at the first batch Durable's own checks refuse + * (global IDs, document actions, copy resolution): the batches before it are durable, it and the ones after are not. + * Calls `receive(index, seq)` once per durable batch, in order, synchronously, after all of them are durable and + * before the next commit starts. Resolves with the number of durable batches and the refusal, if any. Rejects for an + * unknown outcome (connection lost, statement or commit failed); `receive` calls made before the rejection still + * stand. `receive` is a durability receipt, not publication: the Session publishes later. Once it resolved, later + * reads through this storage observe the stored batches. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise; + + /** + * Optional. Run `publish`, the Session's publication of one stored or held commit (its cache write-through, + * adoption, and observers), inside this storage's own scope, such as a store's attached host batch, so what the + * publication marks lands together. Call `publish` exactly once, now or soon after; the Session publishes the next + * commit only once it ran. Without it, the Session publishes directly. + */ + publishing?(publish: () => void, context: Context): void; /** * Return a fresh branded candidate from the Session-global numeric ID namespace, greater than every ID minted since @@ -1226,38 +1443,3 @@ export interface Storage { /** Release backend resources; all later operations must reject. */ close(context: Context): Promise; } - -/** - * A write as it appears in a commit log. Fork copies are logged as the creation they became, with the materialized - * base, so replaying a log never needs to read a copy's source. - */ -export type LogWrite = Exclude; - -/** One committed batch at its sequence. Sequences strictly increase within a log and may have gaps. */ -export type LogRecord = { - readonly seq: Seq; - readonly writes: readonly LogWrite[]; -}; - -/** Storage that rebuilds its commit log from what it stores, and writes log records at their sequences. */ -export interface ReplayableStorage extends Storage { - /** Sequence of the last committed batch; `undefined` for a storage without commits. */ - head(context: Context): Promise; - - /** - * Atomically commit `record.writes` at `record.seq`, which must exceed the head. Validation, atomicity, and stamps - * are those of `commit()`; later commits continue after `record.seq`. - */ - replay(record: LogRecord, context: Context): Promise; - - /** - * The changes committed after `after`, as log records in sequence order and rebuilt from the stored state: - * replayed into a storage holding this storage's state at `after`, they produce its current state, and every read, - * historical document reads included, answers the same. Without `after`, the log rebuilds the whole state into an - * empty storage. Superseded states are gone: each task and submission appears once, at its last write, and a - * current-only document keeps only what followed its newest base. The last record is at the head; nothing follows a - * head at `after`, and an `after` past the head rejects. A commit while the log is consumed makes it reject, and the - * caller reads it again. - */ - log(after: Seq | undefined, context: Context): AsyncIterable; -} diff --git a/packages/durable/test-workers/storage.test.ts b/packages/durable/test-workers/storage.test.ts index 0936866cc..210014285 100644 --- a/packages/durable/test-workers/storage.test.ts +++ b/packages/durable/test-workers/storage.test.ts @@ -18,7 +18,7 @@ registerStorageConformance({ describe, expect, it }, "SqliteStorage on Durable O ); describe("Durable Object SQLite schema", () => { - it("migrates a version 1 schema to admit sleeping tasks, keeping tasks and indexes, then adds sequence indexes", async () => { + it("migrates a version 1 schema to admit sleeping tasks, keeping tasks and indexes", async () => { await runInDurableObject(fresh(), async (_instance, state) => { const database = new DurableObjectSqliteDatabase(state.storage); const schema = createSqliteSchema("legacy_"); @@ -35,7 +35,7 @@ describe("Durable Object SQLite schema", () => { await expect(insert(6, "sleeping")).rejects.toThrow(); await openDurableObjectSqliteStorage(state.storage, { prefix: "legacy_" }); - expect(await database.get(`SELECT version FROM ${schema.tables.durableSchema}`)).toEqual({ version: 3 }); + expect(await database.get(`SELECT version FROM ${schema.tables.durableSchema}`)).toEqual({ version: 2 }); expect(await database.get(`SELECT status FROM ${tasks} WHERE id = 5`)).toEqual({ status: "waiting" }); await insert(6, "sleeping"); const indexes = await database.all<{ name: string }>( @@ -47,9 +47,28 @@ describe("Durable Object SQLite schema", () => { schema.indexes.tasksByBackground, schema.indexes.tasksByConversation, schema.indexes.tasksByKind, - schema.indexes.tasksBySeq, schema.indexes.tasksByStatus, ]); }); }); }); + +describe("Durable Object SQLite driver in workerd", () => { + // The cursor's `rowsWritten` counts index writes too (3 for this insert, 20 for the bulk update); `runCounted` must not. + it("reports the rows a statement changed, without index writes", async () => { + await runInDurableObject(fresh(), async (_instance, state) => { + const database = new DurableObjectSqliteDatabase(state.storage); + await database.exec("CREATE TABLE probe (id TEXT PRIMARY KEY, path TEXT NOT NULL, n INTEGER NOT NULL) STRICT, WITHOUT ROWID"); + await database.exec("CREATE INDEX probe_path ON probe (path, id)"); + await database.exec("CREATE INDEX probe_n ON probe (n, id)"); + for (let i = 0; i < 10; i++) await database.run("INSERT INTO probe VALUES (?, ?, ?)", `k${i}`, `a/${i}`, i); + // Outside the bulk statements' range below. + expect(await database.runCounted("INSERT INTO probe VALUES (?, ?, ?)", "x", "b/x", 99)).toBe(1); + expect(await database.runCounted("UPDATE probe SET n = 0 WHERE id = ?", "absent")).toBe(0); + expect( + await database.runCounted("UPDATE probe SET path = 'c/' || substr(path, 3) WHERE path >= 'a/' AND path < 'a0'"), + ).toBe(10); + expect(await database.runCounted("DELETE FROM probe WHERE path >= 'c/' AND path < 'c0'")).toBe(10); + }); + }); +}); diff --git a/packages/durable/test-workers/wake.test.ts b/packages/durable/test-workers/wake.test.ts index a28a739c9..fb991edf7 100644 --- a/packages/durable/test-workers/wake.test.ts +++ b/packages/durable/test-workers/wake.test.ts @@ -59,7 +59,7 @@ describe("a Harness woken by its Durable Object alarm", () => { await eventually(stub, async (_instance, state) => (await state.storage.getAlarm()) === null); }); - it("recovers a turn an eviction cut off mid-stream through the heartbeat alarm", async () => { + it("recovers a turn an eviction cut off mid-stream through the alarms, retrying the charged attempt", async () => { const stub = fresh(); await runInDurableObject(stub, async (instance: TestSession, state) => { instance.script(["hang", "recovered"]); diff --git a/packages/durable/test-workers/worker.ts b/packages/durable/test-workers/worker.ts index 24d0c5ae2..ce6ea8e94 100644 --- a/packages/durable/test-workers/worker.ts +++ b/packages/durable/test-workers/worker.ts @@ -66,6 +66,8 @@ export class TestSession extends DurableObject { { models, registry, + // An eviction during a request is a failed attempt (#11139); its retry need not wait long here. + settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } }, now: () => this.now(), onWake: (wake) => { this.wakes.push(wake); diff --git a/packages/durable/test/chat-support.ts b/packages/durable/test/chat-support.ts index 06b019d3d..1c2946f4a 100644 --- a/packages/durable/test/chat-support.ts +++ b/packages/durable/test/chat-support.ts @@ -16,6 +16,7 @@ import { type HarnessOptions, type HarnessSettings, type Registry, + type ReportInfo, type Storage, type ToolRegistration, } from "@earendil-works/pi-durable"; @@ -28,6 +29,8 @@ export type ChatSetup = { readonly models: Models; readonly registry: Registry; readonly reports: unknown[]; + /** What each report was about, in the order of `reports`. */ + readonly infos: ReportInfo[]; /** Live Harness settings; tests assign fields to change them between decisions. */ readonly settings: { -readonly [K in keyof HarnessSettings]: HarnessSettings[K] }; now: () => number; @@ -39,7 +42,7 @@ export function chatSetup(options: RegisterFauxProviderOptions = {}): ChatSetup const faux = fauxProvider(options); const models = createModels(); models.setProvider(faux.provider); - return { faux, models, registry: createRegistry(), reports: [], settings: {}, now: () => Date.now() }; + return { faux, models, registry: createRegistry(), reports: [], infos: [], settings: {}, now: () => Date.now() }; } /** Open a Harness over `storage` and return its root, configured with the faux model on first creation. */ @@ -60,7 +63,10 @@ export async function openChat( ? {} : { env: typeof options.env === "function" ? options.env : () => options.env as ExecutionEnv }), now: () => setup.now(), - onReport: (error) => setup.reports.push(error), + onReport: ({ error, ...info }) => { + setup.reports.push(error); + setup.infos.push(info); + }, }, context, ); diff --git a/packages/durable/test/codemode-bench.ts b/packages/durable/test/codemode-bench.ts index 982443cdb..d79df3da6 100644 --- a/packages/durable/test/codemode-bench.ts +++ b/packages/durable/test/codemode-bench.ts @@ -5,6 +5,7 @@ * * node --conditions=source --experimental-strip-types test/codemode-bench.ts [N,...] */ + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,7 +13,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { createRegistry, defineExtension, defineTool, Harness } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -22,14 +23,14 @@ type Shape = "sequential" | "parallel"; const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({ i: Type.Number() }), + parameters: obj({ i: num() }), execute: async (args) => ({ output: [{ type: "text", text: String(args.i) }] }), }); const plain = defineTool({ name: "plain", description: "plain", - parameters: Type.Object({ n: Type.Number(), shape: Type.String() }), + parameters: obj({ n: num(), shape: str() }), replay: "safe", execute: async (args, api, callContext) => { if (args.shape === "sequential") { @@ -66,8 +67,11 @@ async function run(mode: "codemode" | "plain", shape: Shape, n: number): Promise const harness = await Harness.open(storage, { models, registry, settings: {} }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); let commits = 0; - harness.subscribeCommits(() => { - commits++; + harness.observeCommits({ + advance: () => { + commits++; + }, + end: () => {}, }); const started = performance.now(); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); diff --git a/packages/durable/test/codemode-describe.test.ts b/packages/durable/test/codemode-describe.test.ts index 99ced0465..85dfdbe99 100644 --- a/packages/durable/test/codemode-describe.test.ts +++ b/packages/durable/test/codemode-describe.test.ts @@ -1,7 +1,8 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CODEMODE_SOURCE_GRAMMAR } from "@earendil-works/pi-codemode/source"; import { defineTool, MemoryStorage, type ToolRegistration } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createCodemodeExtension } from "../src/codemode/codemode.ts"; import { type CodemodeNamespace, codemodeDescription, codemodeSection } from "../src/codemode/describe.ts"; @@ -12,7 +13,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [] }), ...extra, }); diff --git a/packages/durable/test/codemode-imports.test.ts b/packages/durable/test/codemode-imports.test.ts new file mode 100644 index 000000000..df74da0e2 --- /dev/null +++ b/packages/durable/test/codemode-imports.test.ts @@ -0,0 +1,11 @@ +import { expect, test, vi } from "vitest"; +import { createCodemodeExtension } from "../src/codemode/codemode.ts"; + +// The extension runs scripts in the sandbox the host passes, so loading it must not load the WASM VM (#11254). +vi.mock("quickjs-wasi", () => { + throw new Error("the Code Mode extension loaded quickjs-wasi"); +}); + +test("the Code Mode extension loads without pi-codemode's WASM runtime", () => { + expect(createCodemodeExtension).toBeTypeOf("function"); +}); diff --git a/packages/durable/test/codemode.test.ts b/packages/durable/test/codemode.test.ts index 650255e43..c12eba6d2 100644 --- a/packages/durable/test/codemode.test.ts +++ b/packages/durable/test/codemode.test.ts @@ -1,8 +1,8 @@ -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CodemodeRun, CodemodeSandbox } from "@earendil-works/pi-codemode"; import { defineTool, @@ -13,6 +13,7 @@ import { ToolTask, type ToolTaskInput, } from "@earendil-works/pi-durable"; +import { bool, num, obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { type CodemodeGlobal, @@ -24,12 +25,13 @@ import { import type { CodemodeNamespace } from "../src/codemode/describe.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { createReadTool } from "../src/tools/read.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()), id: Type.Optional(Type.Number()) }); +const Parameters = obj({ text: opt(str()), id: opt(num()) }); function tool( name: string, @@ -714,13 +716,13 @@ describe("durable code mode", () => { it("resolves a schema tool's error result that carries its value, and rejects one without with its text", async () => { const chat = codemodeChat(); - const Exit = Type.Object({ output: Type.String(), exitCode: Type.Number() }); + const Exit = obj({ output: str(), exitCode: num() }); addTool( chat.setup.registry, defineTool({ name: "run", description: "run", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, // Like bash on a nonzero exit: an error result with its value. execute: async () => ({ @@ -735,7 +737,7 @@ describe("durable code mode", () => { defineTool({ name: "denied", description: "denied", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, execute: async () => ({ output: [{ type: "text", text: "permission denied" }], isError: true }), }), @@ -766,7 +768,7 @@ describe("durable code mode", () => { defineTool({ name: "picture", description: "picture", - parameters: Type.Object({ caption: Type.Boolean() }), + parameters: obj({ caption: bool() }), execute: async (args) => ({ output: [ ...(args.caption ? [{ type: "text" as const, text: "a cat" }] : []), @@ -928,10 +930,13 @@ describe("durable code mode", () => { ]); const { harness, root } = await openChat(await openNodeSqliteStorage(await sqlitePath()), chat.setup); let journalCommits = 0; - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === "pi.codemode.journal") journalCommits++; - } + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "document" && change.record.kind === "pi.codemode.journal") journalCommits++; + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(await resultText(chat.setup, root)).toEqual({ text: "50", isError: false }); @@ -1044,6 +1049,29 @@ describe("durable code mode output and tool search", () => { expect(message.content.some((item) => item.type === "image")).toBe(true); }); + // #11015 + it("resolves the built-in read with an image's block, which image() shows, and with a text file's text", async () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + const files = await mkdtemp(join(tmpdir(), "pi-codemode-read-")); + directories.add(files); + await writeFile(join(files, "pixel.png"), Buffer.from(png, "base64")); + await writeFile(join(files, "note.txt"), "hello"); + const chat = codemodeChat(); + addTool(chat.setup.registry, createReadTool()); + const { text, message } = await runScript( + `const picture = await tools.read({ path: ${JSON.stringify(join(files, "pixel.png"))} }); + text([picture.type, picture.mimeType, picture.data.length].join(" ")); + image(picture); + text(await tools.read({ path: ${JSON.stringify(join(files, "note.txt"))} }));`, + chat, + ); + expect(message.isError).toBe(false); + expect(text).toContain(`image image/png ${png.length}`); + expect(text).toMatch(/\[Image saved to \S+ \(image\/png, \d+ bytes\)\]/); + expect(text).toContain("hello"); + expect(message.content.some((item) => item.type === "image")).toBe(true); + }); + it("calls a deferred tool the model is not offered, and finds it in ALL_TOOLS", async () => { const chat = codemodeChat(); addTool( @@ -1051,7 +1079,7 @@ describe("durable code mode output and tool search", () => { defineTool({ name: "later", description: "A deferred tool", - parameters: Type.Object({}), + parameters: obj({}), deferred: true, execute: async () => ({ output: [{ type: "text", text: "later ran" }] }), }), diff --git a/packages/durable/test/conversation-transcript.test.ts b/packages/durable/test/conversation-transcript.test.ts new file mode 100644 index 000000000..985941ef9 --- /dev/null +++ b/packages/durable/test/conversation-transcript.test.ts @@ -0,0 +1,358 @@ +// Tests for `ConversationTranscript`, the client-side layer that merges view windows into one transcript (#11327). +import { + ConversationTranscript, + type ConversationView, + type Cursor, + type EntryId, + type EntryLoader, + type EntryRecord, + MemoryStorage, + type TranscriptItem, + type TranscriptView, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { allEntries, chatSetup, openChat } from "./chat-support.ts"; +import { context, onValue } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +const CONVERSATION = { id: 1 } as unknown as ConversationView["conversation"]; + +/** Entries with IDs that have holes: every third ID is missing. */ +function transcript(count: number): EntryRecord[] { + const entries: EntryRecord[] = []; + for (let id = 1; entries.length < count; id++) { + if (id % 3 !== 0) entries.push({ id, conversationId: CONVERSATION.id, kind: "note" } as unknown as EntryRecord); + } + return entries; +} + +/** A loader over `entries` with `Conversation.entries()` semantics, counting its calls. */ +function loader(entries: readonly EntryRecord[]): EntryLoader & { calls: number } { + const load = async (query: Parameters[0], limit: number) => { + load.calls++; + const matching = entries.filter( + (entry) => + (query.minEntryId === undefined || entry.id >= query.minEntryId) && + (query.maxEntryId === undefined || entry.id <= query.maxEntryId), + ); + const ordered = query.order === "ascending" ? matching : [...matching].reverse(); + const items = ordered.slice(0, limit); + return { items, ...(ordered.length > limit ? { next: "more" as unknown as Cursor } : {}) }; + }; + load.calls = 0; + return load; +} + +/** The view of a window: `entries` with the entry before them in `all`. */ +function windowOf( + all: readonly EntryRecord[], + first: number, + last: number, + docs: ConversationView["docs"] = {}, +): ConversationView { + const before = all[first - 1]?.id; + return { + conversation: CONVERSATION, + entries: all.slice(first, last + 1), + ...(before === undefined ? {} : { before }), + docs, + }; +} + +/** Items as a compact list: entry IDs, `gap after..upTo`, and `earlier`. */ +function shape(view: TranscriptView): (number | string)[] { + return view.items.map((item) => + item.kind === "entry" ? item.entry.id : item.kind === "gap" ? `gap ${item.after}..${item.upTo}` : "earlier", + ); +} + +const idsOf = (entries: readonly EntryRecord[]) => entries.map((entry) => entry.id); +const gapOf = (view: TranscriptView) => + view.items.find((item): item is Extract => item.kind === "gap")!; + +describe("ConversationTranscript", () => { + it("shows a first window under an `earlier` marker, or alone when it starts the transcript", () => { + const all = transcript(20); + const later = new ConversationTranscript(loader(all)); + expect(shape(later.merge(windowOf(all, 15, 19)))).toEqual(["earlier", ...idsOf(all.slice(15))]); + expect(later.newest).toBe(all[19]!.id); + const whole = new ConversationTranscript(loader(all)); + expect(shape(whole.merge(windowOf(all, 0, 19)))).toEqual(idsOf(all)); + }); + + it("appends, keeps entries the window dropped, and returns the same items when nothing is added", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + const docsA = { a: { n: 1 } }; + const first = helper.merge(windowOf(all, 10, 19, docsA)); + const grown = helper.merge(windowOf(all, 10, 21, docsA)); + expect(shape(grown)).toEqual(["earlier", ...idsOf(all.slice(10, 22))]); + // The window dropped its oldest; the transcript keeps them. + const spliced = helper.merge(windowOf(all, 18, 21, docsA)); + expect(spliced.items).toBe(grown.items); + expect(spliced).toBe(grown); + // A docs-only change: a new view, the same items. + const docs = { a: { n: 2 } }; + const streamed = helper.merge({ ...windowOf(all, 18, 21), docs }); + expect(streamed).not.toBe(grown); + expect(streamed.items).toBe(grown.items); + expect(streamed.docs).toBe(docs); + expect(first.items).not.toBe(grown.items); + }); + + it("opens a gap when a window does not continue the transcript, and fills it page by page", async () => { + const all = transcript(200); + const load = loader(all); + const helper = new ConversationTranscript(load, { pageSize: 50 }); + helper.merge(windowOf(all, 150, 159)); + // A stall: the next window starts well past the newest entry held. + const stalled = helper.merge(windowOf(all, 190, 199)); + const gap = gapOf(stalled); + expect(gap).toEqual({ kind: "gap", after: all[159]!.id, upTo: all[190]!.id }); + expect(shape(stalled)).toEqual([ + "earlier", + ...idsOf(all.slice(150, 160)), + `gap ${all[159]!.id}..${all[190]!.id}`, + ...idsOf(all.slice(190)), + ]); + // 30 entries are missing; one page loads them all and closes the gap. + const filled = await helper.fillGap(gap, context); + expect(shape(filled)).toEqual(["earlier", ...idsOf(all.slice(150))]); + expect(load.calls).toBe(1); + }); + + it("shrinks a gap from its start while more than a page is missing", async () => { + const all = transcript(200); + const helper = new ConversationTranscript(loader(all), { pageSize: 10 }); + helper.merge(windowOf(all, 100, 109)); + let view = helper.merge(windowOf(all, 190, 199)); + view = await helper.fillGap(gapOf(view), context); + expect(gapOf(view)).toEqual({ kind: "gap", after: all[119]!.id, upTo: all[190]!.id }); + while (view.items.some((item) => item.kind === "gap")) view = await helper.fillGap(gapOf(view), context); + expect(shape(view)).toEqual(["earlier", ...idsOf(all.slice(100))]); + }); + + it("loads earlier pages until the start, then drops the marker", async () => { + const all = transcript(45); + const load = loader(all); + const helper = new ConversationTranscript(load, { pageSize: 20 }); + helper.merge(windowOf(all, 40, 44)); + let view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(["earlier", ...idsOf(all.slice(20))]); + // A page without a next cursor reaches the start. + view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(idsOf(all)); + const calls = load.calls; + expect(await helper.loadEarlier(context)).toBe(view); + expect(load.calls).toBe(calls); + }); + + it("shares one load between concurrent calls of loadEarlier, and of fillGap for one gap", async () => { + const all = transcript(100); + const inner = loader(all); + const gate = deferred(); + const load: EntryLoader = async (query, limit, callContext) => { + await gate.promise; + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 40, 49)); + const stalled = helper.merge(windowOf(all, 90, 99)); + const earlier = [helper.loadEarlier(context), helper.loadEarlier(context)]; + const fills = [helper.fillGap(gapOf(stalled), context), helper.fillGap(gapOf(stalled), context)]; + expect(earlier[0]).toBe(earlier[1]); + expect(fills[0]).toBe(fills[1]); + gate.resolve(); + await Promise.all([...earlier, ...fills]); + expect(inner.calls).toBe(2); + expect(shape(await helper.loadEarlier(context))).toEqual(idsOf(all)); + }); + + it("resumes after `newest`: an empty window right after it adds nothing; one past it opens a gap", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + const view = helper.merge(windowOf(all, 10, 19)); + const resumed: ConversationView = { conversation: CONVERSATION, entries: [], before: helper.newest!, docs: {} }; + expect(helper.merge(resumed).items).toBe(view.items); + // A window that is empty but starts past `newest`: the entries up to its `before` are missing. + const past = helper.merge({ ...resumed, before: all[25]!.id }); + expect(shape(past).at(-1)).toBe(`gap ${all[19]!.id}..${all[25]!.id + 1}`); + expect(helper.newest).toBe(all[25]!.id); + // The next window continues it. + expect(shape(helper.merge(windowOf(all, 26, 27))).slice(-3)).toEqual([ + `gap ${all[19]!.id}..${all[25]!.id + 1}`, + all[26]!.id, + all[27]!.id, + ]); + }); + + it("fills a gap from a window that covers it", () => { + const all = transcript(100); + const helper = new ConversationTranscript(loader(all)); + helper.merge(windowOf(all, 40, 49)); + helper.merge(windowOf(all, 90, 99)); + // A wider window, such as a new watch with a larger `max`, covers the second half of the gap. + const view = helper.merge(windowOf(all, 70, 99)); + expect(shape(view)).toEqual([ + "earlier", + ...idsOf(all.slice(40, 50)), + `gap ${all[49]!.id}..${all[70]!.id}`, + ...idsOf(all.slice(70)), + ]); + }); + + it("rejects loadEarlier before the first merge and returns the view for a gap that is gone", async () => { + const all = transcript(10); + const helper = new ConversationTranscript(loader(all)); + await expect(helper.loadEarlier(context)).rejects.toThrow("merge a view first"); + const view = helper.merge(windowOf(all, 0, 9)); + const stale = { kind: "gap", after: 1 as EntryId, upTo: 2 as EntryId } as const; + expect(await helper.fillGap(stale, context)).toBe(view); + }); +}); + +describe("ConversationTranscript loads", () => { + it("retries a load whose loader threw synchronously, for loadEarlier and fillGap", async () => { + const all = transcript(100); + const inner = loader(all); + let offline = true; + let calls = 0; + const load: EntryLoader = (query, limit, callContext) => { + calls++; + // An adapter that checks its connection before it returns a promise. + if (offline) throw new Error("temporarily offline"); + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 40, 49)); + const stalled = helper.merge(windowOf(all, 90, 99)); + await expect(helper.loadEarlier(context)).rejects.toThrow("temporarily offline"); + await expect(helper.fillGap(gapOf(stalled), context)).rejects.toThrow("temporarily offline"); + offline = false; + expect(shape(await helper.loadEarlier(context))).toEqual([ + ...idsOf(all.slice(0, 50)), + `gap ${all[49]!.id}..${all[90]!.id}`, + ...idsOf(all.slice(90)), + ]); + expect(shape(await helper.fillGap(gapOf(stalled), context))).toEqual(idsOf(all)); + expect(calls).toBe(4); + }); + + it("retries a load that rejected", async () => { + const all = transcript(30); + const inner = loader(all); + let fail = true; + const load: EntryLoader = async (query, limit, callContext) => { + if (fail) throw new Error("timeout"); + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 20, 29)); + await expect(helper.loadEarlier(context)).rejects.toThrow("timeout"); + fail = false; + expect(shape(await helper.loadEarlier(context))).toEqual(idsOf(all)); + }); + + it("keeps one gap for an empty window whose `before` lies inside it", () => { + const all = transcript(100); + const helper = new ConversationTranscript(loader(all)); + helper.merge(windowOf(all, 40, 49)); + const before = helper.merge(windowOf(all, 90, 99)); + const view = helper.merge({ conversation: CONVERSATION, entries: [], before: all[65]!.id, docs: before.docs }); + expect(view.items).toBe(before.items); + expect(view.items.filter((item) => item.kind === "gap")).toHaveLength(1); + }); + + it("shares one load between a stale gap descriptor and the gap it shrank into", async () => { + const all = transcript(100); + const inner = loader(all); + const gate = deferred(); + const load: EntryLoader = async (query, limit, callContext) => { + await gate.promise; + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load, { pageSize: 10 }); + helper.merge(windowOf(all, 10, 19)); + const stale = gapOf(helper.merge(windowOf(all, 90, 99))); + gate.resolve(); + const current = gapOf(await helper.fillGap(stale, context)); + expect(current).not.toEqual(stale); + const loads = inner.calls; + const [a, b] = [helper.fillGap(stale, context), helper.fillGap(current, context)]; + expect(a).toBe(b); + await a; + expect(inner.calls).toBe(loads + 1); + }); + + it("treats an empty page with a next cursor as the end of the range", async () => { + const all = transcript(60); + // A remote loader that always returns a cursor. + const inner = loader(all); + const load: EntryLoader = async (query, limit, callContext) => ({ + ...(await inner(query, limit, callContext)), + next: "more" as unknown as Cursor, + }); + const helper = new ConversationTranscript(load, { pageSize: 100 }); + helper.merge(windowOf(all, 10, 19)); + let view = helper.merge(windowOf(all, 40, 49)); + view = await helper.fillGap(gapOf(view), context); + // The first page held every missing entry; the next one is empty and closes the gap. + view = await helper.fillGap(gapOf(view), context); + expect(gapOf(view)).toBeUndefined(); + await helper.loadEarlier(context); + view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(idsOf(all.slice(0, 50))); + }); + + it("resumes with `resume`: undefined before a merge, `{}` for an empty transcript, `{ from }` after the newest", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + expect(helper.resume).toBeUndefined(); + helper.merge({ conversation: CONVERSATION, entries: [], docs: {} }); + expect(helper.resume).toEqual({}); + helper.merge(windowOf(all, 0, 9)); + expect(helper.resume).toEqual({ from: all[9]!.id + 1 }); + }); +}); + +describe("ConversationTranscript over a Harness", () => { + it("follows the walkthrough: live appends, a stall, fillGap, loadEarlier, and a resume with `from`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + for (let index = 0; index < 120; index++) await note(); + const helper = new ConversationTranscript((query, limit, callContext) => + root.entries(query, limit, undefined, callContext), + ); + const window = { min: 10, max: 20 }; + const watch = await root.watch(context, window); + let view = helper.merge(watch.value); + watch.start( + onValue(async ({ value }) => { + view = helper.merge(value); + }), + ); + for (let index = 0; index < 15; index++) await note(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(view.items.some((item) => item.kind === "gap")).toBe(false); + expect(view.items.filter((item) => item.kind === "entry")).toHaveLength(25); + // A drop: the client misses 40 entries, more than `max`, and resumes after the newest it has. + await watch.stop(); + for (let index = 0; index < 40; index++) await note(); + const resumed = await root.watch(context, { ...window, ...helper.resume }); + view = helper.merge(resumed.value); + expect(view.items.filter((item) => item.kind === "gap")).toHaveLength(1); + while (view.items.some((item) => item.kind === "gap")) view = await helper.fillGap(gapOf(view), context); + while (view.items[0]?.kind === "earlier") view = await helper.loadEarlier(context); + const entries = view.items.flatMap((item) => (item.kind === "entry" ? [item.entry.id] : [])); + expect(entries).toEqual(idsOf(await allEntries(root))); + // A short drop resumes without a gap. + for (let index = 0; index < 3; index++) await note(); + const again = await root.watch(context, { ...window, ...helper.resume }); + expect(again.value.entries).toHaveLength(3); + view = helper.merge(again.value); + expect(view.items.some((item) => item.kind !== "entry")).toBe(false); + await resumed.stop(); + await again.stop(); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/conversation-view-bench.ts b/packages/durable/test/conversation-view-bench.ts new file mode 100644 index 000000000..e3b781850 --- /dev/null +++ b/packages/durable/test/conversation-view-bench.ts @@ -0,0 +1,233 @@ +/** + * Conversation view benchmark: a long conversation (thousands of entries) and the watches of its view. + * + * node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts + * + * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap + * (`SqliteStorage`; with `PI_DURABLE_BENCH_STORAGE=store`, Durable's storage on Chord's store, `StoreStorage`): + * - `start`: acquire a watch of a conversation of N entries (the mount is built every time; the watch is stopped). + * - `append`: K started watches; commit entries one per commit, or in bursts of 50 per commit, and wait for delivery. + * - `resume`: a watch with `from` 20 entries back, as a client that reconnects after a short drop. + * - `stream`: K started watches; commits that change only `pi.live` (a streamed answer), no entries. + * - `long-lived`: one started watch while 20,000 entries are appended; heap after GC, and whether the first entry + * delivered is still reachable. + */ +import { execFile } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import type { Context } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { type Conversation, createRegistry, type EntryId, Harness, LiveDoc, type Storage } from "../src/index.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +type Scenario = + | { kind: "start"; entries: number } + | { kind: "append"; entries: number; watches: number; burst: number; commits: number } + | { kind: "resume"; entries: number } + | { kind: "stream"; entries: number; watches: number; commits: number } + | { kind: "long-lived"; entries: number }; + +const TEXT = "The quick brown fox jumps over the lazy dog. ".repeat(10); +type Watch = Awaited>; +/** `watch(context, options)`; an older Harness ignores the options. */ +type WatchWith = (context: Context, options?: { from?: EntryId }) => Promise; + +async function main(): Promise { + const [, , encoded] = process.argv; + if (encoded !== undefined) { + const result = await run(JSON.parse(encoded) as Scenario); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + const scenarios: Scenario[] = [ + { kind: "start", entries: 5000 }, + { kind: "resume", entries: 5000 }, + { kind: "append", entries: 5000, watches: 1, burst: 1, commits: 2000 }, + { kind: "append", entries: 5000, watches: 4, burst: 1, commits: 2000 }, + { kind: "append", entries: 5000, watches: 4, burst: 50, commits: 200 }, + { kind: "stream", entries: 5000, watches: 4, commits: 5000 }, + { kind: "long-lived", entries: 20000 }, + ]; + const rows: Record[] = []; + for (const scenario of scenarios) { + const { stdout } = await promisify(execFile)( + process.execPath, + [...process.execArgv, fileURLToPath(import.meta.url), JSON.stringify(scenario)], + { maxBuffer: 1 << 20 }, + ); + rows.push({ scenario: label(scenario), ...JSON.parse(stdout.trim().split("\n").at(-1)!) }); + } + console.table(rows); +} + +function label(scenario: Scenario): string { + switch (scenario.kind) { + case "start": + return `watch start, ${scenario.entries} entries`; + case "resume": + return `resume 20 back, ${scenario.entries} entries`; + case "append": + return `append x${scenario.burst} per commit, ${scenario.watches} watches`; + case "stream": + return `pi.live only, ${scenario.watches} watches`; + case "long-lived": + return `long-lived watch, ${scenario.entries} appends`; + } +} + +function gc(): void { + (globalThis as { gc?: () => void }).gc?.(); +} + +function heapMiB(): number { + gc(); + gc(); + return Math.round((process.memoryUsage().heapUsed / 2 ** 20) * 10) / 10; +} + +async function appendEntries(conversation: Conversation, count: number, perCommit: number): Promise { + for (let done = 0; done < count; done += perCommit) { + await conversation.commit(async (tx) => { + for (let index = done; index < Math.min(count, done + perCommit); index++) { + await tx.appendEntry(conversation.id, { kind: "note", data: { text: `${index} ${TEXT}` } }); + } + }, context); + } +} + +/** Resolve once every watch delivered a value whose newest entry is `id`. */ +function delivered(watches: readonly Watch[], id: () => EntryId | undefined): Promise { + return new Promise((resolve) => { + const check = (): void => { + const target = id(); + if (watches.every((watch) => watch.value.entries.at(-1)?.id === target)) resolve(); + else setTimeout(check, 1); + }; + check(); + }); +} + +async function run(scenario: Scenario): Promise> { + const directory = mkdtempSync(join(tmpdir(), "pi-durable-view-bench-")); + let storage: Storage | undefined; + try { + const path = join(directory, "session.sqlite"); + storage = + process.env.PI_DURABLE_BENCH_STORAGE === "store" + ? await openStoreStorage(path) + : await openNodeSqliteStorage(path); + const models = createModels(); + const harness = await Harness.open(storage, { models, registry: createRegistry() }, context); + const root = await harness.root(context); + await appendEntries(root, scenario.entries, 500); + const newest = async (): Promise => (await root.entries({}, 1, undefined, context)).items[0]!.id; + const watchWith = root.watch.bind(root) as unknown as WatchWith; + const heapBefore = heapMiB(); + const cpu = process.cpuUsage(); + const started = performance.now(); + const result: Record = {}; + if (scenario.kind === "start" || scenario.kind === "resume") { + const from = scenario.kind === "resume" ? (((await newest()) - 19) as EntryId) : undefined; + const times: number[] = []; + let size = 0; + for (let round = 0; round < 20; round++) { + const at = performance.now(); + const watch = await watchWith(context, from === undefined ? undefined : { from }); + times.push(performance.now() - at); + size = watch.value.entries.length; + await watch.stop(); + } + times.sort((a, b) => a - b); + result.p50Ms = round(times[10]!); + result.windowEntries = size; + } else if (scenario.kind === "append") { + const watches = await Promise.all(Array.from({ length: scenario.watches }, () => root.watch(context))); + for (const watch of watches) watch.start(async () => {}); + let last: EntryId | undefined; + const at = performance.now(); + for (let index = 0; index < scenario.commits; index++) { + const ids = await root.commit(async (tx) => { + const appended: EntryId[] = []; + for (let n = 0; n < scenario.burst; n++) { + appended.push((await tx.appendEntry(root.id, { kind: "note", data: { text: TEXT } })).id); + } + return appended; + }, context); + last = ids.at(-1); + } + await delivered(watches, () => last); + result.perCommitUs = round(((performance.now() - at) * 1000) / scenario.commits); + result.windowEntries = watches[0]!.value.entries.length; + result.heapWithWatchesMiB = heapMiB(); + for (const watch of watches) await watch.stop(); + } else if (scenario.kind === "stream") { + const watches = await Promise.all(Array.from({ length: scenario.watches }, () => root.watch(context))); + let calls = 0; + for (const watch of watches) + watch.start(async () => { + calls++; + }); + const at = performance.now(); + for (let index = 0; index < scenario.commits; index++) { + await root.commit(async (tx) => { + (await tx.doc(LiveDoc, root.id)).compactionFailure = { + taskId: 1 as never, + trigger: "manual", + error: `${index}`, + }; + }, context); + } + // Every watch delivered the final value, not just one of them. + const last = `${scenario.commits - 1}`; + const done = () => + watches.every( + (watch) => + (watch.value.docs["pi.live"] as { compactionFailure?: { error: string } } | undefined) + ?.compactionFailure?.error === last, + ); + while (calls < scenario.watches || !done()) await new Promise((resolve) => setTimeout(resolve, 1)); + result.perCommitUs = round(((performance.now() - at) * 1000) / scenario.commits); + for (const watch of watches) await watch.stop(); + } else { + const watch = await root.watch(context); + let first: WeakRef | undefined; + watch.start(async (call) => { + if (call.kind !== "value") return; + const entry = call.value.entries.at(-1); + if (first === undefined && entry !== undefined) first = new WeakRef(entry); + }); + await appendEntries(root, 1, 1); + const id = await newest(); + await delivered([watch], () => id); + await appendEntries(root, scenario.entries, 1); + const last = await newest(); + await delivered([watch], () => last); + result.windowEntries = watch.value.entries.length; + result.heapWithWatchMiB = heapMiB(); + // A WeakRef's target survives the job that created or read it; collect again in a later task. + await new Promise((resolve) => setTimeout(resolve, 10)); + gc(); + result.firstEntryCollected = first?.deref() === undefined ? 1 : 0; + await watch.stop(); + } + const usage = process.cpuUsage(cpu); + result.cpuMs = round((usage.user + usage.system) / 1000); + result.wallMs = round(performance.now() - started); + result.heapBeforeMiB = heapBefore; + await harness.close(context); + return result; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} + +await main(); diff --git a/packages/durable/test/env-node-conformance.test.ts b/packages/durable/test/env-node-conformance.test.ts index 38e8de61b..b7f36b1c1 100644 --- a/packages/durable/test/env-node-conformance.test.ts +++ b/packages/durable/test/env-node-conformance.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { afterEach, describe, expect, it } from "vitest"; -import { getOrThrow, type WatchChange } from "../src/env/index.ts"; +import { type FileChange, getOrThrow } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { registerEnvConformance } from "../src/testing/index.ts"; @@ -72,17 +72,17 @@ describe("NodeExecutionEnv watch limits", () => { }); getOrThrow(await env.remove("tree/c", { recursive: true }, context)); - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow( await env.watch([{ path: "tree", recursive: true }], (change) => changes.push(change), context), ); getOrThrow(await env.createDir("tree/d", undefined, context)); const deadline = Date.now() + 3000; - while (!changes.some((change) => "error" in change) && Date.now() < deadline) { + while (!changes.some((change) => change.kind === "failed") && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 20)); } - expect(changes.at(-1)).toMatchObject({ error: { code: "invalid" } }); - await watcher.close(context); + expect(changes.at(-1)).toMatchObject({ kind: "failed", error: { code: "invalid" } }); + await watcher.stop(context); } finally { await removeTempDir(cwd); } diff --git a/packages/durable/test/examples/04-watches.ts b/packages/durable/test/examples/04-watches.ts index bc76fc6ec..be3c0f405 100644 --- a/packages/durable/test/examples/04-watches.ts +++ b/packages/durable/test/examples/04-watches.ts @@ -32,7 +32,11 @@ console.log("watch baseline:", notesWatch.value); const busy = Promise.withResolvers(); const done = Promise.withResolvers(); let calls = 0; -notesWatch.start(async (value, ops, _deliveryContext) => { +notesWatch.start(async (call, _deliveryContext) => { + // Calls are tagged: a `failed` call is a failure a source recovers from, which Durable never sends (its failures + // end the watch, see `closed`). + if (call.kind !== "value") return; + const { value, ops } = call; console.log("watch update:", value, JSON.stringify(ops)); // The first call stays busy, as a slow consumer would, while three more commits land. if (++calls === 1) await busy.promise; diff --git a/packages/durable/test/examples/05-harness.ts b/packages/durable/test/examples/05-harness.ts index ad33c0d0c..e0fd947fb 100644 --- a/packages/durable/test/examples/05-harness.ts +++ b/packages/durable/test/examples/05-harness.ts @@ -1,9 +1,10 @@ // Open a Harness with a registry of extensions. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/05-harness.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -23,7 +24,7 @@ const context = BACKGROUND_CONTEXT; const read = defineTool({ name: "read", description: "Read a file", - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `contents of ${args.path}` }] }), }); const Files = defineExtension({ name: "files", tools: [read] }); diff --git a/packages/durable/test/examples/06-configuration.ts b/packages/durable/test/examples/06-configuration.ts index 03184b6d8..fb50401db 100644 --- a/packages/durable/test/examples/06-configuration.ts +++ b/packages/durable/test/examples/06-configuration.ts @@ -1,9 +1,10 @@ // Per-conversation agent choices and Harness-wide settings. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/06-configuration.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -25,7 +26,7 @@ function exampleTool(name: string, description: string): AppTool { const tool = defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); return { ...tool, snippet: `Use ${name} for files.` }; diff --git a/packages/durable/test/examples/09-registry-reload.ts b/packages/durable/test/examples/09-registry-reload.ts index 4135650ee..e1ee8319e 100644 --- a/packages/durable/test/examples/09-registry-reload.ts +++ b/packages/durable/test/examples/09-registry-reload.ts @@ -1,9 +1,10 @@ // Reload extension code through the registry. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/09-registry-reload.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage, wrapTool } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -12,7 +13,7 @@ function exampleTool(name: string, description: string) { return defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); } diff --git a/packages/durable/test/examples/10-extension-state.ts b/packages/durable/test/examples/10-extension-state.ts index f2d518214..bfd6613e7 100644 --- a/packages/durable/test/examples/10-extension-state.ts +++ b/packages/durable/test/examples/10-extension-state.ts @@ -1,10 +1,11 @@ // An extension that keeps its own per-conversation document. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/10-extension-state.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineDoc, @@ -38,7 +39,7 @@ const Todo = defineExtension({ defineTool({ name: "todo", description: "Add an item to your todo list", - parameters: Type.Object({ item: Type.String() }), + parameters: obj({ item: str() }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(Todos, api.conversationId)).items.push(args.item); diff --git a/packages/durable/test/examples/15-real-model.ts b/packages/durable/test/examples/15-real-model.ts index b5cbe1244..e43c6b427 100644 --- a/packages/durable/test/examples/15-real-model.ts +++ b/packages/durable/test/examples/15-real-model.ts @@ -45,7 +45,9 @@ if (process.env.OPENAI_API_KEY === undefined) { process.stdout.write(text.slice(printed.length)); printed = text; }; - liveWatch.start(async (value) => { + liveWatch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; const block = value?.generation?.message?.content.find((content) => content.type === "text"); if (block?.type === "text") printText(block.text); }); diff --git a/packages/durable/test/examples/18-json.ts b/packages/durable/test/examples/18-json.ts index f1348cfe1..2e2b801c7 100644 --- a/packages/durable/test/examples/18-json.ts +++ b/packages/durable/test/examples/18-json.ts @@ -1,11 +1,11 @@ -// JSON mode: stream one prompt's run as JSON lines, like `pi --mode json`. Two modes: -// --events (default): the experimental agent events of watchEvents(), starting with a `snapshot` event. -// --ops: the raw ConversationView frames, starting with the view itself; each later line holds one commit's ops. +// JSON mode: stream one prompt's run as JSON lines, like `pi --mode json`. Two modes, both of the ConversationView: +// --ops (default): the raw frames, starting with the view itself; each later line holds the ops of the commits since. +// --state: per call, the entries appended since the previous one and the current `pi.live`, state a renderer draws. // --storage sqlite (default) | jsonl | memory: sqlite writes a temporary database file, jsonl a temporary directory; // neither is deleted, and the path is printed to stderr at the end. // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: -// node --conditions=source --experimental-strip-types test/examples/18-json.ts --events "What is in this directory?" +// node --conditions=source --experimental-strip-types test/examples/18-json.ts --state "What is in this directory?" import { mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -14,22 +14,14 @@ import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; import { NodeExecutionEnv } from "../../src/env/node.ts"; -import { - createRegistry, - defineExtension, - Harness, - MemoryStorage, - type Storage, - section, - watchEvents, -} from "../../src/index.ts"; +import { createRegistry, defineExtension, Harness, MemoryStorage, type Storage, section } from "../../src/index.ts"; import { openNodeJsonlStorage } from "../../src/storage/jsonl/node.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; import { createBashTool, createReadTool } from "../../src/tools/index.ts"; const context = BACKGROUND_CONTEXT; const args = process.argv.slice(2); -const mode = args.includes("--ops") ? "ops" : "events"; +const mode = args.includes("--state") ? "state" : "ops"; const storageIndex = args.indexOf("--storage"); const storageKind = storageIndex < 0 ? "sqlite" : args[storageIndex + 1]; const prompt = @@ -77,20 +69,23 @@ const env = new NodeExecutionEnv({ cwd: process.cwd() }); const harness = await Harness.open(storage, { models, registry, env: () => env }, context); const root = await harness.root(context, { agent: { model } }); -// Attach before submitting, so the stream covers the whole run. -let stop: () => Promise; -if (mode === "events") { - const stream = await watchEvents(harness, root.id, context); - print(stream.snapshot); - stream.start(async (events) => { - for (const event of events) print(event); - }); - stop = () => stream.stop(); -} else { - const watch = await root.watch(context); +// Attach before submitting, so the watch covers the whole run. +const watch = await root.watch(context); +if (mode === "ops") { print({ view: watch.value }); - watch.start(async (_value, ops) => print({ ops })); - stop = () => watch.stop(); + watch.start(async (call) => { + if (call.kind === "value") print({ ops: call.ops }); + }); +} else { + let newest = watch.value.entries.at(-1)?.id ?? 0; + print({ entries: watch.value.entries, live: watch.value.docs["pi.live"] }); + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + const entries = value.entries.filter((entry) => entry.id > newest); + newest = value.entries.at(-1)?.id ?? newest; + print({ ...(entries.length === 0 ? {} : { entries }), live: value.docs["pi.live"] }); + }); } const submission = await root.submit({ type: "input", content: prompt }, context); @@ -98,6 +93,6 @@ await submission.wait(context); await harness.waitForIdle(context); // Let the last batch reach the listener before stopping. await new Promise((resolve) => setTimeout(resolve, 0)); -await stop(); +await watch.stop(); await harness.close(context); if (location !== undefined) console.error(`${storageKind} storage: ${location}`); diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 122ad70ee..59eb2d0d1 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -1,20 +1,19 @@ // Late join: a client attaches while a run is already underway. It gets the current state first, the conversation -// view or the snapshot event, and then only what changes after that. +// view, and then only what changes after that. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/20-late-join.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { - type AgentEvent, createRegistry, defineExtension, defineTool, Harness, type LiveState, MemoryStorage, - watchEvents, } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -29,7 +28,7 @@ registry.install( defineTool({ name: "count", description: "Counts to ten", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (let n = 1; n <= 10; n++) { api.output(`${n}\n`); @@ -55,7 +54,8 @@ const submission = await root.submit({ type: "input", content: "Count to ten, th // Join while the tool is halfway through. await pause(500); -// Structural client: the view holds the committed transcript and documents, including the running tool's output. +// The view holds the newest entries and the documents, including the running tool's output and, later, the answer +// being streamed. const view = await root.watch(context); const live = view.value.docs["pi.live"] as LiveState; console.log( @@ -63,38 +63,25 @@ console.log( view.value.entries.map((entry) => entry.kind), ); console.log("view tool slot:", live.tools?.[0]?.status, JSON.stringify(live.tools?.[0]?.output)); -view.start(async (value) => { - const slot = (value.docs["pi.live"] as LiveState).tools?.[0]; +let shown = ""; +view.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + const state = value.docs["pi.live"] as LiveState; + const slot = state.tools?.[0]; if (slot?.status === "running") console.log("view output now:", JSON.stringify(slot.output)); -}); - -// Event client: the snapshot event carries the same state; later events apply on top of it. -const stream = await watchEvents(harness, root.id, context); -console.log( - "snapshot tools:", - stream.snapshot.tools.map((slot) => `${slot.name} ${slot.status}`), -); -let output = stream.snapshot.tools[0]?.output ?? ""; -stream.start(async (events: readonly AgentEvent[]) => { - for (const event of events) { - if (event.type === "tool_execution_update" && event.output !== undefined) { - output = - "set" in event.output - ? event.output.set - : output.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - console.log("event output now:", JSON.stringify(output)); - } else if (event.type === "message_update") { - const deltas = event.changes.flatMap((change) => (change.type === "text_delta" ? [change.delta] : [])); - console.log("event text delta:", JSON.stringify(deltas.join(""))); - } else { - console.log("event:", event.type); - } - } + const partial = state.generation?.message?.content.find((block) => block.type === "text")?.text ?? ""; + if (partial !== shown && partial !== "") console.log("view answer now:", JSON.stringify(partial)); + shown = partial; + if (state.run === undefined) + console.log( + "view entries:", + value.entries.map((entry) => entry.kind), + ); }); await submission.wait(context); await harness.waitForIdle(context); await pause(0); -await stream.stop(); await view.stop(); await harness.close(context); diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index c766803fb..7f624a018 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -1,17 +1,18 @@ // A foreground subagent tool: the parent's tool call creates a child conversation it owns, runs one task there, and // returns the child's answer. Aborting the tool call aborts the child. A UI finds the child through the tool's running -// details and shows its events under the call. +// details and shows its view under the call. // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/21-subagent-foreground.ts + import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { obj, str } from "@earendil-works/pi-schema"; import { - type AgentEvent, AssistantEntry, type ConversationId, configure, @@ -19,11 +20,12 @@ import { defineExtension, defineTool, type EntryId, + type EntryRecord, type Extension, Harness, + type LiveState, MemoryStorage, type ToolExecutionApi, - watchEvents, } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -42,7 +44,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String({ description: "What the subagent should do" }) }), + parameters: obj({ task: str({ description: "What the subagent should do" }) }), // Safe to rerun after a crash: a rerun finds the child it already created and the submission it already made. replay: "safe", execute: async (args, api, callContext) => { @@ -99,26 +101,30 @@ registry.install(Subagent); const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); const root = await harness.root(context, { agent: { model } }); -// ─── UI: the parent's events, with each subagent's events indented under its call ─── +// ─── UI: the parent's transcript, with each subagent's indented under its call ─── -const print = (indent: string, event: AgentEvent): void => { - if (event.type === "message_end" && event.entry.kind === "pi.assistant") { - const message = event.entry.model?.[0] as AssistantMessage; - const text = message.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join(""); - if (text !== "") console.log(`${indent}assistant: ${text}`); - } else if (event.type === "tool_execution_start") { - console.log(`${indent}tool ${event.toolName}(${JSON.stringify(event.args)})`); +/** Print an assistant entry's text and tool calls. */ +const print = (indent: string, entry: EntryRecord): void => { + const message = entry.model?.[0]; + if (entry.kind !== "pi.assistant" || message?.role !== "assistant") return; + for (const block of (message as AssistantMessage).content) { + if (block.type === "text" && block.text !== "") console.log(`${indent}assistant: ${block.text}`); + if (block.type === "toolCall") console.log(`${indent}tool ${block.name}(${JSON.stringify(block.arguments)})`); } }; const attached = new Set(); const attach = async (id: ConversationId, indent: string): Promise => { attached.add(id); - const stream = await watchEvents(harness, id, context); - stream.start(async (events) => { - for (const event of events) { - print(indent, event); - if (event.type !== "tool_execution_update") continue; - const child = (event.details as { conversationId?: ConversationId } | undefined)?.conversationId; + const watch = await (await harness.conversation(id, context))!.watch(context); + let newest = watch.value.entries.at(-1)?.id ?? 0; + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + for (const entry of value.entries) if (entry.id > newest) print(indent, entry); + newest = value.entries.at(-1)?.id ?? newest; + // A running subagent call names its child conversation in its details. + for (const slot of (value.docs["pi.live"] as LiveState).tools ?? []) { + const child = (slot.details as { conversationId?: ConversationId } | undefined)?.conversationId; if (child !== undefined && !attached.has(child)) await attach(child, `${indent} `); } }); @@ -131,6 +137,6 @@ const submission = await root.submit( ); await submission.wait(context); await harness.waitForIdle(context); -// Event callbacks run after their commit; let the last ones print. +// Watch callbacks run after their commit; let the last ones print. await new Promise((resolve) => setTimeout(resolve, 0)); await harness.close(context); diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 19fba5dc5..f01ee89b4 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -5,16 +5,17 @@ // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/22-subagent-background.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { bool, obj, oneOf, opt, str } from "@earendil-works/pi-schema"; import { - type AgentEvent, AssistantEntry, type ConversationId, configure, @@ -27,7 +28,6 @@ import { Harness, LiveDoc, type TaskId, - watchEvents, } from "../../src/index.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; @@ -136,11 +136,11 @@ const subagentTool = defineTool({ "Manage persistent subagents that work in the background. Actions: spawn (name, message), send (name, message; " + "followUp: true queues it after the current answer instead of steering), stop (name: aborts its current work), " + "status (name, or all subagents without one). Answers are reported back to you when they arrive.", - parameters: Type.Object({ - action: Type.Union([Type.Literal("spawn"), Type.Literal("send"), Type.Literal("stop"), Type.Literal("status")]), - name: Type.Optional(Type.String()), - message: Type.Optional(Type.String()), - followUp: Type.Optional(Type.Boolean()), + parameters: obj({ + action: oneOf(["spawn", "send", "stop", "status"]), + name: opt(str()), + message: opt(str()), + followUp: opt(bool()), }), // A call interrupted by a crash is not rerun: repeating `stop` could stop newer work. The model sees that the call // was interrupted and can check with `status`. @@ -284,11 +284,15 @@ function contentText(content: string | readonly { type: string; text?: string }[ /** Print the main conversation's messages as they are committed. Subagents work in their own conversations. */ async function follow(harness: Harness, id: ConversationId): Promise { - const stream = await watchEvents(harness, id, context); - stream.start(async (events: readonly AgentEvent[]) => { - for (const event of events) { - if (event.type !== "message_end") continue; - const message = event.entry.model?.[0]; + const watch = await (await harness.conversation(id, context))!.watch(context); + let newest = watch.value.entries.at(-1)?.id ?? 0; + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + const entries = value.entries.filter((entry) => entry.id > newest); + newest = value.entries.at(-1)?.id ?? newest; + for (const entry of entries) { + const message = entry.model?.[0]; if (message?.role === "user") { const text = contentText(message.content); // Input is either the user or a subagent's report, which arrives whenever the subagent is done. diff --git a/packages/durable/test/examples/26-plan-mode.ts b/packages/durable/test/examples/26-plan-mode.ts index b0bedd056..0eaf5b939 100644 --- a/packages/durable/test/examples/26-plan-mode.ts +++ b/packages/durable/test/examples/26-plan-mode.ts @@ -2,13 +2,14 @@ // extension's own document, and switching back restores the full tool set. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/26-plan-mode.ts + import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../../src/env/node.ts"; import { createRegistry, @@ -39,7 +40,7 @@ const PlanDoc = defineDoc<{ steps: string[] }>({ const submitPlan = defineTool({ name: "submit_plan", description: "Submit the plan as a list of steps.", - parameters: Type.Object({ steps: Type.Array(Type.String()) }), + parameters: obj({ steps: arr(str()) }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(PlanDoc, api.conversationId)).steps = args.steps; diff --git a/packages/durable/test/examples/30-reload-and-restart.ts b/packages/durable/test/examples/30-reload-and-restart.ts index 6813a7659..4918cea2f 100644 --- a/packages/durable/test/examples/30-reload-and-restart.ts +++ b/packages/durable/test/examples/30-reload-and-restart.ts @@ -2,13 +2,15 @@ // with; stored choices are names, so they survive a restart and bind to whatever code the new process installs. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/30-reload-and-restart.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type ToolResultMessage, Type } from "@earendil-works/pi-ai"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../../src/index.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; @@ -24,7 +26,7 @@ function loadVersioned(version: string) { defineTool({ name: "version", description: "Report the tool's code version", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { started(); await gate; diff --git a/packages/durable/test/examples/31-transcript.ts b/packages/durable/test/examples/31-transcript.ts new file mode 100644 index 000000000..bdbf7f1f2 --- /dev/null +++ b/packages/durable/test/examples/31-transcript.ts @@ -0,0 +1,110 @@ +// A client's transcript over the windowed conversation view: `ConversationTranscript` merges every value of a watch +// into the client's whole transcript, marks entries a stall skipped as a gap, and loads them, and earlier pages, on +// demand. A client that drops resumes after the newest entry it has. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/31-transcript.ts +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { + ConversationTranscript, + createRegistry, + Harness, + MemoryStorage, + type TranscriptItem, + type TranscriptView, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; +const harness = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: createRegistry() }, + context, +); +const root = await harness.root(context); +let count = 0; +/** Commit `n` chat lines, one entry each. */ +const chat = async (n: number): Promise => { + for (let index = 0; index < n; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "chat", data: `line ${++count}` }), context); + } +}; +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** One line per item: ranges of entries, gaps, and the "load earlier" marker. */ +function render(label: string, view: TranscriptView): void { + const rows: string[] = []; + let run: [number, number] | undefined; + const flush = () => { + if (run !== undefined) rows.push(run[0] === run[1] ? `${run[0]}` : `${run[0]}..${run[1]}`); + run = undefined; + }; + for (const item of view.items) { + if (item.kind === "entry") { + const line = Number(String(item.entry.data).slice("line ".length)); + run = run === undefined ? [line, line] : [run[0], line]; + continue; + } + flush(); + rows.push(item.kind === "gap" ? "[load more]" : "[load earlier]"); + } + flush(); + console.log(`${label.padEnd(28)} ${rows.join(" ")}`); +} + +const firstGap = (view: TranscriptView) => + view.items.find((item): item is Extract => item.kind === "gap"); + +await chat(200); + +// Attach: the window holds the newest `min` entries; the transcript shows "load earlier" above them. +const transcript = new ConversationTranscript((query, limit, ctx) => root.entries(query, limit, undefined, ctx), { + pageSize: 20, +}); +const window = { min: 10, max: 20 }; +let watch = await root.watch(context, window); +let view = transcript.merge(watch.value); +render("attach", view); +watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + view = transcript.merge(value); +}); + +// Live: appends. The window drops its oldest past `max`; the transcript keeps them. +await chat(15); +await settle(); +render("15 more, live", view); + +// The user scrolls up: one page. +view = await transcript.loadEarlier(context); +render("load earlier", view); + +// A drop: the client misses 40 entries, more than `max`, and resumes after the newest it has. The window starts at the +// newest `min` again, and `before` shows the transcript a gap. +await watch.stop(); +await chat(40); +watch = await root.watch(context, { ...window, ...transcript.resume }); +view = transcript.merge(watch.value); +render("resume after 40 missed", view); +watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; + view = transcript.merge(value); +}); + +// "load more" fills the gap a page at a time. +for (let gap = firstGap(view); gap !== undefined; gap = firstGap(view)) { + view = await transcript.fillGap(gap, context); + render("fill gap", view); +} + +// A short drop resumes without a gap: the window starts right after the newest entry. +await watch.stop(); +await chat(3); +watch = await root.watch(context, { ...window, ...transcript.resume }); +console.log(`${"resume after 3 missed".padEnd(28)} window: ${watch.value.entries.length} entries`); +view = transcript.merge(watch.value); +render("", view); + +await watch.stop(); +await harness.close(context); diff --git a/packages/durable/test/examples/32-tests-after-edits.ts b/packages/durable/test/examples/32-tests-after-edits.ts new file mode 100644 index 000000000..240bfec31 --- /dev/null +++ b/packages/durable/test/examples/32-tests-after-edits.ts @@ -0,0 +1,141 @@ +// Run the project's tests at the end of every run that edited files, with an `afterRun` hook: the run waits for the +// tests, a failure starts a fix run, and input sent meanwhile waits for them too. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/32-tests-after-edits.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; +import { + createRegistry, + defineExtension, + defineTask, + defineTool, + GenerationTask, + Harness, + hook, + LiveDoc, + MemoryStorage, + runEndTask, + ToolResultEntry, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// ─── Product code: the project, an edit tool, and the test runner ───────────── + +// The project under test, in memory: the tests pass once `add` adds. +const files = new Map([["math.ts", "export const add = (a, b) => a - b;"]]); +const runTests = () => { + const passed = files.get("math.ts")!.includes("a + b"); + return { passed, output: passed ? "1 passed" : "add(1, 2): expected 3, got -1" }; +}; + +const edit = defineTool({ + name: "edit", + description: "Replace a file's content", + parameters: obj({ path: str(), content: str() }), + execute: async (args) => { + files.set(args.path, args.content); + return { output: [{ type: "text", text: `wrote ${args.path}` }] }; + }, +}); + +// A durable task: it runs once the run answered, and the run ends after it. A crash resumes it. +const RunTests = defineTask<{ attempt: number }, { phase: "run" }, boolean>({ + name: "app.run-tests", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + const result = runTests(); + console.log(` [tests, attempt ${task.input.attempt}] ${result.output}`); + // The handle is fetched before the commit: inside it, runtime reads would wait for the commit itself. + const conversation = (await runtime.conversation(runtime.conversationId, ctx))!; + await runtime.commit(async (tx) => { + // Admitted with this commit, before its own table writes: placed when the run ends. + if (!result.passed && task.input.attempt < 3) { + await conversation.submit(tx, { + type: "input", + content: `The tests failed: ${result.output}. Fix it.`, + whenBusy: "followUp", + requestId: `fix-${task.input.attempt}`, + }); + } + await tx.appendEntry(runtime.conversationId, { kind: "app.tests", data: result }); + return { status: "terminal", outcome: { status: "completed", result: result.passed } }; + }, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }) as const, ctx), +}); + +let attempts = 0; +const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tools: [edit], + tasks: [RunTests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + // The run's own entries, from its first input to its answer; a compaction never removes them. + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, ctx); + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && message.toolName === "edit"; + }); + // Each fix run asks again with a fresh turn budget: the extension caps the loop itself. + return edited ? { tasks: [runEndTask(RunTests, { attempt: ++attempts })] } : undefined; + }, + }), + ], +}); + +// ─── Host setup ───────────────────────────────────────────────────────────── + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + // The first run edits, wrongly. + fauxAssistantMessage(fauxToolCall("edit", { path: "math.ts", content: "export const add = (a, b) => a - b + 0;" }), { + stopReason: "toolUse", + }), + fauxAssistantMessage("I cleaned up add."), + // The user's question, queued first, runs first; it edits nothing, so no tests run after it. + fauxAssistantMessage("Not yet: add subtracts. The fix is queued next."), + // The fix run the failed tests started. + fauxAssistantMessage(fauxToolCall("edit", { path: "math.ts", content: "export const add = (a, b) => a + b;" }), { + stopReason: "toolUse", + }), + fauxAssistantMessage("Fixed: add adds now."), +]); + +const registry = createRegistry(); +registry.install(TestsAfterEdits); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + +const submission = await root.submit({ type: "input", content: "Clean up math.ts." }, context); +// Resolves with the answer; the conversation stays busy while the tests run. +console.log("answered:", (await submission.wait(context)).status); +const live = await harness.snapshot(LiveDoc, root.id, context); +console.log("busy with after-run tasks:", live?.run?.afterRun !== undefined); +// Sent while the tests run: it waits for them. Follow-ups run in queue order, so it runs before the fix run. +await root.submit({ type: "input", content: "Are the tests green now?" }, context); + +await root.waitForIdle(context); +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (entry.kind === "app.tests") console.log("tests:", (entry.data as { output: string }).output); + else if (message?.role === "user") console.log(">", message.content); + else if (message?.role === "assistant") { + const text = message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""); + if (text !== "") console.log("agent:", text); + } +} +await harness.close(context); diff --git a/packages/durable/test/examples/33-turn-limit-and-continue.ts b/packages/durable/test/examples/33-turn-limit-and-continue.ts new file mode 100644 index 000000000..817bd1fcb --- /dev/null +++ b/packages/durable/test/examples/33-turn-limit-and-continue.ts @@ -0,0 +1,74 @@ +// A per-conversation turn limit stops a long run after a number of tool rounds; a continue resumes it without a user +// message, with a fresh budget. A host learns that a run stopped from `pi.live.runFailure`. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/33-turn-limit-and-continue.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { num, obj } from "@earendil-works/pi-schema"; +import { + ConversationBusy, + createRegistry, + defineExtension, + defineTool, + Harness, + LiveDoc, + MemoryStorage, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const step = defineTool({ + name: "step", + description: "Do one step of the work", + parameters: obj({ n: num() }), + execute: async (args) => ({ output: [{ type: "text", text: `step ${args.n} done` }] }), +}); +const call = (n: number) => fauxAssistantMessage(fauxToolCall("step", { n }), { stopReason: "toolUse" }); + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +// Four steps, then the answer. Two tool rounds per run are allowed. +faux.setResponses([call(1), call(2), call(3), call(4), fauxAssistantMessage("All four steps are done.")]); + +const registry = createRegistry(); +registry.install(defineExtension({ name: "steps", tools: [step] })); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +// Stored on the conversation, like its model: children and forks inherit it. +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" }, maxTurns: 2 } }); + +const first = await (await root.submit({ type: "input", content: "Do the four steps." }, context)).wait(context); +const failure = (await harness.snapshot(LiveDoc, root.id, context))?.runFailure; +console.log("first run:", first.status, first.reason, `(${failure?.detail})`); + +// An idle conversation runs a continue on the context as it is, the round's results last, with a fresh budget. +const resumed = await root.submit({ type: "continue" }, context); +// It never queues: while that run works, another continue is refused. +try { + await root.submit({ type: "continue" }, context); +} catch (error) { + if (!(error instanceof ConversationBusy)) throw error; + console.log("second continue while busy: ConversationBusy"); +} +const second = await resumed.wait(context); +console.log("first continue:", second.status, second.status === "done" ? "" : second.reason); +const third = await (await root.submit({ type: "continue" }, context)).wait(context); +const live = await harness.snapshot(LiveDoc, root.id, context); +console.log("second continue:", third.status, live?.runFailure === undefined ? "(runFailure cleared)" : ""); + +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (message?.role === "user") console.log(">", message.content); + if (message?.role === "toolResult") + console.log(" tool:", message.content[0]?.type === "text" ? message.content[0].text : ""); + if (message?.role === "assistant" && message.stopReason === "stop") { + console.log("agent:", message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("")); + } +} +// A continue after an answer has nothing to answer: it ends without a model request. +const after = await (await root.submit({ type: "continue" }, context)).wait(context); +console.log("continue after the answer:", after.status, after.reason); +await harness.close(context); diff --git a/packages/durable/test/examples/34-submit-in-commit.ts b/packages/durable/test/examples/34-submit-in-commit.ts new file mode 100644 index 000000000..5cdfa918f --- /dev/null +++ b/packages/durable/test/examples/34-submit-in-commit.ts @@ -0,0 +1,104 @@ +// A host lets people message an agent while it works: a message must join the current run as a steer and never start a +// run of its own. `submit(tx, draft)` admits it inside the host's own commit, together with the host's check and its own +// state; `whenIdle: "reject"` refuses it when no run is working. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/34-submit-in-commit.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; +import { + ConversationIdle, + createRegistry, + defineDoc, + defineExtension, + defineTool, + Harness, + MemoryStorage, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The host's own state: who may message this conversation, and what they sent. +const Visitors = defineDoc<{ muted: string[]; sent: { from: string; text: string }[] }>({ + kind: "app.visitors", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ muted: [], sent: [] }), + checkpointWhen: () => false, +}); + +// A tool that waits until released, so messages arrive while the run works. +let release!: () => void; +const working = new Promise((resolve) => { + release = resolve; +}); +const build = defineTool({ + name: "build", + description: "Build the project", + parameters: obj({}), + execute: async () => { + await working; + return { output: [{ type: "text", text: "built" }] }; + }, +}); + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage(fauxToolCall("build", {}), { stopReason: "toolUse" }), + fauxAssistantMessage("Built, and I added the test Ana asked for."), +]); +const registry = createRegistry(); +registry.install(defineExtension({ name: "build", tools: [build] })); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); +await root.commit(async (tx) => { + (await tx.doc(Visitors, root.id)).muted.push("bob"); +}, context); + +/** A visitor's message: checked, admitted as a steer of the current run, and recorded, in one commit or not at all. */ +async function message(from: string, text: string, requestId: string): Promise { + try { + return await harness.commit(async (tx) => { + const visitors = await tx.doc(Visitors, root.id); + if (visitors.muted.includes(from)) return `${from} is muted`; + // Before the commit's own table writes; document writes may come before or after. + const id = await root.submit(tx, { + type: "input", + content: `${from}: ${text}`, + whenBusy: "steer", + whenIdle: "reject", + requestId, + }); + visitors.sent.push({ from, text }); + return `${from}'s message is submission ${id}`; + }, context); + } catch (error) { + if (error instanceof ConversationIdle) return `${from}: the agent isn't working now`; + throw error; + } +} + +console.log(await message("ana", "Add a test for the build.", "ana-1")); // idle: refused, nothing written +const run = await root.submit({ type: "input", content: "Build the project." }, context); +console.log(await message("ana", "Add a test for the build.", "ana-2")); // busy: joins the run as a steer +console.log(await message("bob", "Delete everything.", "bob-1")); // the host's own check refuses it +release(); +await run.wait(context); +await root.waitForIdle(context); + +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (message?.role === "user") console.log(">", message.content); + if (message?.role === "assistant" && message.stopReason === "stop") { + console.log("agent:", message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("")); + } +} +console.log("recorded:", (await harness.snapshot(Visitors, root.id, context))?.sent); +await harness.close(context); diff --git a/packages/durable/test/examples/35-durable-on-the-store.ts b/packages/durable/test/examples/35-durable-on-the-store.ts new file mode 100644 index 000000000..958a9d1b4 --- /dev/null +++ b/packages/durable/test/examples/35-durable-on-the-store.ts @@ -0,0 +1,108 @@ +// Durable on the store: a session's worker keeps everything in one store. Durable's Session (a Harness) opens +// `StoreStorage` over the host's store under its object's type name, next to a Chord object's own table and documents; +// both commit through the store's one commit queue (one seq each), one id allocator numbers them all, and one export +// carries both. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/35-durable-on-the-store.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + createHost, + defineFacet, + defineTable, + local, + method, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Notes = defineTable("notes", { version: 1, columns: { id: str(), text: str() }, key: ["id"] }); + +// The harness object: local with empty params, so it lives as long as the host and may persist (P3). +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { note: method(str()), say: method(str()) }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + // Its own table, under its type name ... + const notes = self.table(Notes); + // ... and Durable's records under the same owner name: claims per table and per document namespace. + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + return { + note: async (text, ctx) => { + await self.tx(async (tx) => tx.put(notes, { id: `n${text.length}`, text }), ctx); + return null; + }, + say: async (text, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind: "example.said", data: { text } } }, ctx); + return null; + }, + }; + }); +}); + +const directory = await mkdtemp(join(tmpdir(), "chord-example-26-")); +const path = join(directory, "session.sqlite"); +// 1. The worker's start code opens the store and gives it to its host. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 2. Chord's commit and Durable's commit, each one seq of the same store. +const before = store.lastSeq; +await chat.note("hello", context); +await chat.say("hello, Durable", context); +console.log("seqs taken:", store.lastSeq - before); + +// 3. One archive holds both: Chord's table, Durable's tables and documents, all under the owner name. +const owners = new Map(); +for await (const part of store.export(context)) { + const owner = + part.kind === "table" || part.kind === "put" + ? part.table.split(":")[0] + : part.kind === "document.create" + ? part.record.owner + : undefined; + if (owner !== undefined) owners.set(`${owner} ${part.kind}`, (owners.get(`${owner} ${part.kind}`) ?? 0) + 1); + if (part.kind === "manifest") console.log("manifest:", { lastSeq: part.lastSeq, nextId: part.nextId }); +} +console.log("parts by owner and kind:", Object.fromEntries(owners)); + +// 4. The app closes the host, then the store; a reopen finds both. +await host.close(context); +await store.close(context); +const reopened = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const storage = await StoreStorage.open(reopened, Chat.name, context); +const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); +console.log( + "Durable entries after reopen:", + page.items.map((entry) => entry.kind), +); +await storage.close(context); +await reopened.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/durable/test/examples/36-durable-views.ts b/packages/durable/test/examples/36-durable-views.ts new file mode 100644 index 000000000..2f2ace488 --- /dev/null +++ b/packages/durable/test/examples/36-durable-views.ts @@ -0,0 +1,91 @@ +// Durable's views as Chord sources: `self.follow(attach)` turns Durable's synchronous observers (the conversation view, +// the task graph) into watchables, members and derive sources of the object that opened Durable. With Durable on the +// host's store, Durable adopts its commits inside the store's commit step, in the host's batch: a derive over two views +// of one commit computes once, and a watcher gets one frame per commit. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/36-durable-views.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + int, + local, + method, + obj, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { + // What a UI shows: the kinds of the transcript's entries and the number of tasks, from two views. + summary: obj({ kinds: arr(str()), tasks: int() }), + write: method(str()), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + // 1. Follow two of Durable's views; they attach once the natives opened. + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx, { min: 50 }), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + // 2. A derive over both: one compute per Durable commit. + const summary = self.derive([view, graph], ([v, g]) => ({ + kinds: v.entries.map((entry) => entry.kind), + tasks: Object.keys(g.tasks).length, + })); + return { + summary, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); +}); + +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 3. Watch the member: one frame per Durable commit, with its ops. +const watch = await chat.summary.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("frame:", call.value.kinds, JSON.stringify(call.ops)); +}); +await chat.write("example.first", context); +await chat.write("example.second", context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("member value:", chat.summary.value); + +// 4. Closing the host detaches the observers; the store is the app's. +await watch.stop(); +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/examples/37-harness-registry.ts b/packages/durable/test/examples/37-harness-registry.ts new file mode 100644 index 000000000..404a61a0d --- /dev/null +++ b/packages/durable/test/examples/37-harness-registry.ts @@ -0,0 +1,172 @@ +// A Durable harness in Chord objects, p3's pattern (Chord v2 slice 8): the object that opens Durable on the host's store +// takes its extensions from a Chord registry (`registryReader`), so any plugin extends the harness by contributing an +// extension; and it contributes its Harness to a second registry, through which another object follows its views +// (`self.follow`). When that Harness's Session closes, the follower fails and follows the next Harness, woken by its +// contribution (`self.retryOn`). +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/37-harness-registry.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + local, + method, + type Native, + openStore, + type RegistryTable, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// 1. The core's registries: extensions in, Durable's registry snapshot out; and the Harness, for other objects. +const Extensions = defineRegistry("example.extensions", { version: 1 }); +const Harnesses = defineRegistry, Harness | undefined>("example.harnesses", { version: 1 }); + +/** Builds Durable's snapshot without throwing: an extension Durable refuses (a task name taken) is reported, left out. */ +function snapshotOf( + table: RegistryTable, + report: (conflict: { message: string; plugins: string[] }) => void, +) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Core = local("example.harness", { + version: 1, + params: {}, + members: { greet: method(undefined, str()), close: method() }, +}); +const Transcript = local("example.transcript", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +// 2. The core facet: the harness object opens Durable on the store with its extensions read from the registry. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const core = defineFacet((host) => { + host.serve(Extensions, { compose: (table, { report }) => snapshotOf(table, report) }); + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Core, (_, self) => { + // A declared dependency: the object activates (and opens Durable) once the registry has a value. + const extensions = self.registry(Extensions); + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Core.name, open), + { models: createModels(), registry: registryReader(extensions, open) }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + // Following its own task graph, the object fails with its Session (S1), and its retry opens the next Harness. + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + greet: async (ctx) => { + // The composed snapshot, read at use: is the task's extension installed? + if (extensions.value.task(Greet.definition.name) === undefined) return "no greeter installed"; + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + const id = await root!.commit( + (tx) => tx.createTask(Greet, undefined, { ownership: { kind: "conversation" } }), + ctx, + ); + harness.value.resume(); + return JSON.stringify((await harness.value.waitForTask(id, ctx)).state.outcome); + }, + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + }; + }); + // 3. Another object follows the harness's conversation view through the registry. + host.serve(Transcript, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value?.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +}); + +// 4. Two plugins contribute extensions with the same task name, which Durable refuses: the first by plugin name +// (copycat) is installed, the other reported. +const Greet = defineTask({ + name: "example.greet", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit(async (tx) => { + await tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "example.greeted", data: {} }); + return { status: "terminal", outcome: { status: "completed", result: "hello" } }; + }, taskContext); + }, + }, + abort: async () => {}, +}); +const greeter = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "greeter", tasks: [Greet] })), +); +const copycat = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "copycat", tasks: [Greet] })), +); + +const host = createHost({ + store, + onError: (error) => console.log("onError:", error.kind, error.kind === "registry_conflict" ? error.message : ""), +}); +await host.load( + [ + { name: "@example/core", facet: core }, + { name: "@example/greeter", facet: greeter }, + { name: "@example/copycat", facet: copycat }, + ], + context, +); + +// 5. The task comes from a contributed extension; the transcript object sees its entry. (It failed once at start, before +// the harness object contributed its Harness: `activation_failed`, then the contribution woke it.) +const harness = host.hold(Core, {}, context); +const transcript = host.hold(Transcript, {}, context); +console.log("greet:", await harness.greet(context)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript:", transcript.kinds.value); + +// 6. The Session closes: both objects fail; a retry of the harness object opens the next Harness (on the same store: +// the transcript is still there), whose contribution wakes the follower. +await harness.close(context).catch((error: Error) => console.log("close:", error.message)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript after the close:", transcript.state.value.kind); +harness.retry(); +await new Promise((resolve) => setTimeout(resolve, 50)); +console.log("transcript after the retry:", transcript.state.value.kind, transcript.kinds.value); + +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/harness-after-run.test.ts b/packages/durable/test/harness-after-run.test.ts new file mode 100644 index 000000000..7201fe6af --- /dev/null +++ b/packages/durable/test/harness-after-run.test.ts @@ -0,0 +1,1205 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Draft, JsonValue } from "@earendil-works/chord"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; +import { + CompactionTask, + ConversationBusy, + type ConversationHandle, + defineExtension, + defineTask, + defineTool, + type EntryRecord, + type GenerationHooks, + GenerationTask, + hook, + InboxDoc, + LiveDoc, + type LiveState, + MemoryStorage, + type RunEndTask, + type RunSummary, + runEndTask, + type Storage, + type TaskId, + type Tx, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { afterEach, describe, expect, it } from "vitest"; +import { endRun } from "../src/harness/live.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, completed, type Deferred, deferred, eventually, settled } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-after-run-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over SQLite at `path`; `kill()` simulates a crash: every later storage call hangs, so nothing more lands. */ +async function openKillable(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +/** What the after-run task and hook of a test did, and how they behave. */ +type Probe = { + /** Labels of the after-run task's runs, in order, reruns included. */ + runs: string[]; + /** Summaries the `afterRun` hook was asked with. */ + asked: RunSummary[]; + aborted: number; + /** Holds every run of the after-run task until resolved. */ + gate?: Deferred; + /** Holds the hook until resolved. */ + hookGate?: Deferred; + /** Admitted in the after-run task's commit, before its own entry. */ + admit?: (handle: ConversationHandle, tx: Tx) => Promise; +}; + +function probe(): Probe { + return { runs: [], asked: [], aborted: 0 }; +} + +/** The after-run task: records its run, waits for the gate, then appends a `test.tests` entry and completes. */ +function testsTask(state: Probe) { + return defineTask<{ label: string }, { phase: "run" }, string>({ + name: "test.run-tests", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + state.runs.push(task.input.label); + if (state.gate !== undefined) await Promise.race([state.gate.promise, aborted(runtime.signal)]); + const handle = + state.admit === undefined ? undefined : await runtime.conversation(runtime.conversationId, ctx); + await runtime.commit(async (tx) => { + if (state.admit !== undefined) await state.admit(handle!, tx); + await tx.appendEntry(runtime.conversationId, { kind: "test.tests", data: task.input.label }); + return completed(task.input.label); + }, ctx); + }, + }, + abort: async (_task, runtime, ctx) => { + state.aborted++; + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }) as const, ctx); + }, + }); +} + +/** + * Install the after-run extension: its task and an `afterRun` hook, by default asking for one task per run. Returns + * the task. + */ +function install(setup: ChatSetup, state: Probe, afterRun?: GenerationHooks["afterRun"]) { + const Tests = testsTask(state); + const handler: GenerationHooks["afterRun"] = + afterRun ?? + (async (run) => { + state.asked.push(run); + if (state.hookGate !== undefined) await state.hookGate.promise; + return { tasks: [runEndTask(Tests, { label: `run ${state.asked.length}` })] }; + }); + setup.registry.install( + defineExtension({ name: "after-run", tasks: [Tests], hooks: [hook(GenerationTask, { afterRun: handler })] }), + ); + return Tests; +} + +const Parameters = obj({}); +const tools = defineExtension({ + name: "tools", + tools: [ + defineTool({ + name: "edit", + description: "Edit a file", + parameters: Parameters, + execute: async () => ({ output: [{ type: "text", text: "edited" }] }), + }), + defineTool({ + name: "finish", + description: "End the run", + parameters: Parameters, + execute: async () => ({ output: [{ type: "text", text: "finished" }], control: { terminate: true } }), + }), + defineTool({ + name: "handoff", + description: "End the run in a new context", + parameters: Parameters, + execute: async () => ({ + output: [{ type: "text", text: "handed off" }], + control: { terminate: true, handoff: "fresh start" }, + }), + }), + ], +}); + +function call(name: string, id: string) { + return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); +} + +function kindOf(entries: readonly EntryRecord[], kind: string): EntryRecord[] { + return entries.filter((entry) => entry.kind === kind); +} + +/** Texts of the user messages of the conversation's entries, in order. */ +function userTexts(entries: readonly EntryRecord[]): string[] { + return entries.flatMap((entry) => + (entry.model ?? []).flatMap((message) => (message.role === "user" ? [textOf(message) ?? ""] : [])), + ); +} + +describe("afterRun", () => { + // #11141 + it("waits for the tasks an answer's hooks ask for: inputs done at the answer, the run busy until they end", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + // The input settles with the answer, before the task finishes. + expect((await submission.wait(context)).status).toBe("done"); + await eventually(() => state.runs.length === 1); + const live = (await harness.snapshot(LiveDoc, root.id, context))!; + expect(live.run?.inputs).toEqual([]); + expect(live.run?.afterRun?.tasks).toHaveLength(1); + const child = await harness.getTask(live.run!.afterRun!.tasks![0]!, context); + expect(child?.owner).toBe(live.run!.taskId); + expect(child?.background).toBe(false); + const idle = root.waitForIdle(context); + expect(await settled(idle)).toBe(false); + + const entries = await allEntries(root); + const answer = kindOf(entries, "pi.assistant").at(-1)!; + expect(state.asked).toEqual([ + { reason: "answer", first: kindOf(entries, "pi.user")[0]!.id, answer: answer.id, last: answer.id }, + ]); + state.gate.resolve(); + await idle; + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + expect((await allEntries(root)).at(-1)?.kind).toBe("test.tests"); + const generation = await harness.getTask(live.run!.taskId, context); + expect(generation?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed", result: { entryId: answer.id } }, + }); + await harness.close(context); + }); + + // #11141 + it("asks after a terminate round, the run's last entry its newest result or its handoff's reset", async () => { + for (const [tool, lastKind] of [ + ["finish", "pi.tool-result"], + ["handoff", "pi.reset"], + ] as const) { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call(tool, "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + const assistant = kindOf(entries, "pi.assistant").at(-1)!; + expect(state.asked).toEqual([ + { + reason: "terminate", + first: kindOf(entries, "pi.user")[0]!.id, + answer: assistant.id, + last: kindOf(entries, lastKind).at(-1)!.id, + }, + ]); + expect(state.runs).toEqual(["run 1"]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + } + }); + + // #11141 + it("ends the run in one commit without an afterRun hook, and after the hook when it asks for nothing", async () => { + const plain = chatSetup(); + const state = probe(); + plain.registry.install(defineExtension({ name: "tasks-only", tasks: [testsTask(state)] })); + plain.faux.setResponses([fauxAssistantMessage("done")]); + const opened = await openChat(new MemoryStorage(), plain); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + // No hook: the answer's commit ended the run. + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + + const setup = chatSetup(); + install(setup, state, async (run) => { + state.asked.push(run); + return undefined; + }); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + expect(state.runs).toEqual([]); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + // #11141 + it("queues input during the tasks and places it after them: writes first, then steers and follow-ups", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + const Tests = testsTask(state); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run) => { + state.asked.push(run); + // Only the first run gets a task. + return state.asked.length === 1 ? { tasks: [runEndTask(Tests, { label: "run 1" })] } : undefined; + }, + }), + ], + }), + ); + // The task admits its result for the model in its own commit. + state.admit = (handle, tx) => + handle.submit(tx, { + type: "write", + entry: { kind: "test.result", model: [{ role: "user", content: "tests passed", timestamp: 1 }] }, + }); + setup.faux.setResponses([ + fauxAssistantMessage("first"), + fauxAssistantMessage("second"), + fauxAssistantMessage("third"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + + const steer = await root.submit({ type: "input", content: "steer", whenBusy: "steer" }, context); + const followUp = await root.submit({ type: "input", content: "later" }, context); + await expect(root.submit({ type: "input", content: "x", whenBusy: "reject" }, context)).rejects.toBeInstanceOf( + ConversationBusy, + ); + await expect(root.submit({ type: "continue" }, context)).rejects.toBeInstanceOf(ConversationBusy); + // Busy, so not idle: it queues and starts the next run after the tasks. + const idleSteer = await root.submit( + { type: "input", content: "steer 2", whenBusy: "steer", whenIdle: "reject" }, + context, + ); + const note = await root.submit({ type: "write", entry: { kind: "test.note" } }, context); + expect((await note.status(context)).status).toBe("done"); + expect((await steer.status(context)).status).toBe("queued"); + expect(await harness.snapshot(InboxDoc, root.id, context)).toMatchObject({ items: { length: 3 } }); + + state.gate.resolve(); + await root.waitForIdle(context); + for (const submission of [steer, followUp, idleSteer]) { + expect((await submission.wait(context)).status).toBe("done"); + } + const entries = await allEntries(root); + const kinds = entries.filter((entry) => entry.kind !== "pi.system").map((entry) => entry.kind); + expect(kinds).toEqual([ + "pi.user", + "pi.assistant", + "test.note", + "test.tests", + "test.result", + "pi.user", + "pi.user", + "pi.assistant", + "pi.user", + "pi.assistant", + ]); + expect(userTexts(entries)).toEqual(["go", "tests passed", "steer", "later", "steer 2"]); + await harness.close(context); + }); + + // #11141 + it("places a reset queued during the tasks after them", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + await root.reset(undefined, context); + expect(kindOf(await allEntries(root), "pi.reset")).toEqual([]); + state.gate.resolve(); + await root.waitForIdle(context); + const kinds = (await allEntries(root)).map((entry) => entry.kind); + expect(kinds.slice(-2)).toEqual(["test.tests", "pi.reset"]); + await harness.close(context); + }); + + // #11141 + it("asks once a run really ends: not while onYield continues it, and when a queued steer drops the continuation", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + let yields = 0; + setup.registry.install( + defineExtension({ + name: "again", + hooks: [hook(GenerationTask, { onYield: async () => (++yields <= 2 ? { continue: "again" } : undefined) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("1"), fauxAssistantMessage("2"), fauxAssistantMessage("3")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + expect(state.asked).toEqual([ + { + reason: "answer", + first: kindOf(entries, "pi.user")[0]!.id, + answer: kindOf(entries, "pi.assistant").at(-1)!.id, + last: kindOf(entries, "pi.assistant").at(-1)!.id, + }, + ]); + await harness.close(context); + + // A steer queued while the answer's hooks run: the continuation is dropped, and the run ends after its task. + const dropped = chatSetup(); + const second = probe(); + install(dropped, second); + let root2: Awaited>["root"] | undefined; + dropped.registry.install( + defineExtension({ + name: "steering", + hooks: [ + hook(GenerationTask, { + onYield: async () => { + if (second.asked.length > 0) return undefined; + await root2!.submit({ type: "input", content: "steer", whenBusy: "steer" }, context); + return { continue: "again" }; + }, + }), + ], + }), + ); + dropped.faux.setResponses([fauxAssistantMessage("1"), fauxAssistantMessage("2")]); + const opened = await openChat(new MemoryStorage(), dropped); + root2 = opened.root; + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await opened.root.waitForIdle(context); + expect(second.asked).toHaveLength(2); + expect(second.runs).toEqual(["run 1", "run 2"]); + expect(userTexts(await allEntries(opened.root))).toEqual(["go", "steer"]); + await opened.harness.close(context); + }); + + // #11141 + it("asks the hooks again after a crash before the tasks exist, and never once they do", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); // a crash while the hook runs, in `finish` + state.hookGate = undefined; + state.gate = deferred(); + opened = await openKillable(path, setup); + opened.harness.resume(); + await eventually(() => state.runs.length === 1); + expect(state.asked).toHaveLength(2); + opened.kill(); // a crash while the task runs, in `drain` + state.gate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + // The task ran again; the hook was not asked again, and only one task exists. + expect(state.asked).toHaveLength(2); + expect(state.runs).toEqual(["run 2", "run 2"]); + expect(kindOf(await allEntries(opened.root), "test.tests")).toHaveLength(1); + const tasks = (await opened.harness.inspect(context)).tasks; + expect(tasks).toEqual([]); + await opened.harness.close(context); + + // An orderly close in `drain` resumes the same way. + const closing = await sqlitePath(); + const other = probe(); + other.gate = deferred(); + const setup2 = chatSetup(); + install(setup2, other); + setup2.faux.setResponses([fauxAssistantMessage("done")]); + let reopened = await openKillable(closing, setup2); + await (await reopened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => other.runs.length === 1); + await reopened.harness.close(context); + other.gate = undefined; + reopened = await openKillable(closing, setup2); + await reopened.root.waitForIdle(context); + expect(other.asked).toHaveLength(1); + expect(other.runs).toEqual(["run 1", "run 1"]); + await reopened.harness.close(context); + }); + + // #11141 + it("ends an answered run without a failure when Stop comes while the hooks run or the tasks run", async () => { + for (const stage of ["finish", "drain"] as const) { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + state.gate = deferred(); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, _api, ctx) => { + state.asked.push(run); + if (stage === "finish") await aborted(ctx.abortSignal!); + return { tasks: [runEndTask(Tests, { label: "run 1" })] }; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await submission.wait(context); + await eventually(() => (stage === "finish" ? state.asked.length === 1 : state.runs.length === 1)); + await root.abort(context); + const live = await harness.snapshot(LiveDoc, root.id, context); + expect(live?.run).toBeUndefined(); + expect(live?.runFailure).toBeUndefined(); + expect((await submission.status(context)).status).toBe("done"); + expect(state.aborted).toBe(stage === "drain" ? 1 : 0); + expect(state.runs).toEqual(stage === "drain" ? ["run 1"] : []); + await harness.close(context); + } + }); + + // #11141 + it("ends the run normally when one of its tasks is aborted on its own", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + const live = (await harness.snapshot(LiveDoc, root.id, context))!; + await harness.abortTask(live.run!.afterRun!.tasks![0]!, context); + await root.waitForIdle(context); + expect(state.aborted).toBe(1); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect((await harness.getTask(live.run!.taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed" }, + }); + await harness.close(context); + }); + + // #11141 + it("reports broken requests and hooks and creates the rest", async () => { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + const Throwing = defineTask({ + name: "test.throwing", + version: 1, + initial: () => { + throw new Error("initial failed"); + }, + phases: { run: async () => {} }, + abort: async () => {}, + }); + // Its `initial()` starts at a phase the definition does not have. + const BadStart = defineTask({ + name: "test.bad-start", + version: 1, + initial: () => ({ phase: "nowhere" }) as unknown as { phase: "run" }, + phases: { run: async () => {} }, + abort: async () => {}, + }); + const requests: RunEndTask[] = [ + { task: { definition: { ...Tests.definition, name: "test.unknown" } }, input: null }, + { task: GenerationTask, input: {} }, + { task: Tests, input: { label: () => "not JSON" } as unknown as JsonValue }, + runEndTask(Throwing, null), + runEndTask(BadStart, null), + runEndTask(Tests, { label: "ok" }), + ]; + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests, Throwing, BadStart], + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: requests }) })], + }), + ); + setup.registry.install( + defineExtension({ + name: "broken", + hooks: [ + hook(GenerationTask, { + afterRun: async () => { + throw new Error("hook failed"); + }, + }), + ], + }), + ); + setup.registry.install( + defineExtension({ + name: "not-a-list", + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: "x" as unknown as RunEndTask[] }) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(state.runs).toEqual(["ok"]); + // One report per hook, against the hook's extension; the requests of one hook come together. + expect(setup.infos.filter((info) => info.kind === "extension_error")).toEqual([ + expect.objectContaining({ extension: "after-run" }), + expect.objectContaining({ extension: "broken" }), + expect.objectContaining({ extension: "not-a-list" }), + ]); + const rejected = setup.reports[0] as AggregateError; + expect(rejected.errors.map((error: Error) => error.message)).toEqual([ + "Task test.unknown is not installed", + "An afterRun hook cannot start the built-in task pi.generation", + "Value contains a non-JSON function; expected strict JSON", + "initial failed", + "Task test.bad-start starts at the unknown phase nowhere", + ]); + expect((setup.reports[1] as Error).message).toBe("hook failed"); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + // #11141 + it("creates a requested task from the installed definition, calling its initial() once, off the line", async () => { + const setup = chatSetup(); + const calls = { installed: 0, token: 0 }; + const define = (version: number, count: () => void) => + defineTask({ + name: "test.versioned", + version, + initial: () => { + count(); + return { phase: "run" }; + }, + phases: { run: async (_task, runtime, ctx) => runtime.commit(() => completed(null), ctx) }, + abort: async () => {}, + }); + const Installed = define(2, () => calls.installed++); + // The hook's extension holds an older copy of the task. + const Stale = define(1, () => calls.token++); + setup.registry.install(defineExtension({ name: "tasks", tasks: [Installed] })); + setup.registry.install( + defineExtension({ + name: "asker", + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: [runEndTask(Stale, null)] }) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const created = await harness.commit( + (tx) => tx.scanTasks({ conversationId: root.id, kind: "test.versioned" }, 10), + context, + ); + expect(created.items).toHaveLength(1); + expect(created.items[0]).toMatchObject({ version: 2, state: { status: "terminal" } }); + expect(calls).toEqual({ installed: 1, token: 0 }); + await harness.close(context); + }); + + // #11141 + it("lets a hook read the run's history, tool results included", async () => { + const setup = chatSetup(); + const kinds: string[] = []; + setup.registry.install(tools); + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const page = await api.entries( + { minEntryId: run.first, maxEntryId: run.last, order: "ascending" }, + 100, + undefined, + ctx, + ); + kinds.push(...page.items.map((entry) => entry.kind)); + expect((await api.entry(run.answer, ctx))?.kind).toBe("pi.assistant"); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(kinds.filter((kind) => kind !== "pi.system")).toEqual([ + "pi.user", + "pi.assistant", + "pi.tool-result", + "pi.assistant", + ]); + await harness.close(context); + }); + + // #11141 + it("lets a hook read the run's history after a compaction summarized the start of the run", async () => { + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 1, backgroundTokens: 0 }; + const gate = deferred(); + const started = deferred(); + const runs: string[][] = []; + setup.registry.install( + defineExtension({ + name: "slow", + tools: [ + defineTool({ + name: "edit", + description: "Edit a file", + parameters: Parameters, + execute: async () => { + started.resolve(); + await gate.promise; + return { output: [{ type: "text", text: "edited" }] }; + }, + }), + ], + hooks: [ + hook(CompactionTask, { beforeCompact: async () => ({ summary: "earlier work" }) }), + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const page = await api.entries( + { minEntryId: run.first, maxEntryId: run.last, order: "ascending" }, + 100, + undefined, + ctx, + ); + runs.push(page.items.map((entry) => entry.kind)); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + // Its summary is placed at the round's post-tools boundary, in the middle of the run. + const compaction = await root.compact(undefined, context); + await harness.waitForTask(compaction, context); + gate.resolve(); + await submission.wait(context); + await root.waitForIdle(context); + const active = (await root.context(context)).entries.map((entry) => entry.kind); + // The active context lost the run's input to the summary; the run's history did not. + expect(active).not.toContain("pi.user"); + expect(runs).toHaveLength(1); + expect(runs[0]!.filter((kind) => kind !== "pi.system")).toEqual([ + "pi.user", + "pi.assistant", + "pi.tool-result", + "pi.compaction", + "pi.assistant", + ]); + await harness.close(context); + }); + + // #11141 + it("ends an answered run completed, without a failure, once finish was cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + await opened.root.submit({ type: "input", content: "later" }, context); + opened.kill(); + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + const live = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + expect(live?.run).toBeUndefined(); + expect(live?.runFailure).toBeUndefined(); + // Queued items stay, as after a failed run. + expect((await opened.harness.snapshot(InboxDoc, opened.root.id, context))?.items).toHaveLength(1); + expect(setup.infos).toContainEqual( + expect.objectContaining({ kind: "task_interrupted", reason: "restart_limit" }), + ); + await opened.harness.close(context); + }); + + // #11141 + it("keeps the run busy while an after-run task cannot run, until Stop orphans it", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + opened.kill(); + // Reopened without the extension: its task is blocked. + const bare = chatSetup(); + opened = await openKillable(path, bare); + opened.harness.resume(); + const live = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!; + expect(live.run?.afterRun?.tasks).toHaveLength(1); + await eventually(async () => + (await opened.harness.inspect(context)).tasks.some((task) => task.state.kind === "blocked"), + ); + expect(await settled(opened.root.waitForIdle(context))).toBe(false); + await opened.root.abort(context); + const after = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + expect(after?.run).toBeUndefined(); + expect(after?.runFailure).toBeUndefined(); + expect((await opened.harness.getTask(live.run!.afterRun!.tasks![0]!, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "orphaned" }, + }); + await opened.harness.close(context); + }); + + // #11141 + it("starts a fix run with a follow-up a task admits, after the tasks", async () => { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run) => { + state.asked.push(run); + return state.asked.length === 1 ? { tasks: [runEndTask(Tests, { label: "run 1" })] } : undefined; + }, + }), + ], + }), + ); + state.admit = (handle, tx) => + handle.submit(tx, { type: "input", content: "fix the tests", requestId: "fix-1", whenBusy: "followUp" }); + setup.faux.setResponses([fauxAssistantMessage("done"), fauxAssistantMessage("fixed")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + expect(entries.filter((entry) => entry.kind !== "pi.system").map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.assistant", + "test.tests", + "pi.user", + "pi.assistant", + ]); + expect(userTexts(entries)).toEqual(["go", "fix the tests"]); + expect(state.asked).toHaveLength(2); + // The fix run's history starts with its own input. + expect(state.asked[1]!.first).toBe(kindOf(entries, "pi.user")[1]!.id); + await harness.close(context); + }); + + // #11141 + it("gives a continue after a failed run the failed run's history", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + const failed = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(failed.status).toBe("unanswered"); + const user = kindOf(await allEntries(root), "pi.user")[0]!; + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toMatchObject({ + reason: "turn_limit", + first: user.id, + }); + expect(state.asked).toEqual([]); + await harness.commit((tx) => root.submit(tx, { type: "continue" }), context); + await root.waitForIdle(context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + expect(state.asked).toEqual([{ reason: "answer", first: user.id, answer: answer.id, last: answer.id }]); + await harness.close(context); + }); + + // #11141 + it("ends the run without asking when the hooks are deselected before finish runs", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); + setup.registry.uninstall(defineExtension({ name: "after-run" })); + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + expect(state.runs).toEqual([]); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + // #11141 + it("starts a fork of a conversation running its after-run tasks idle", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + await root.submit({ type: "input", content: "later" }, context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + const fork = await root.fork(answer.id, { ownership: { kind: "ownerless" } }, context); + expect((await harness.snapshot(LiveDoc, fork.id, context))?.run).toBeUndefined(); + expect((await harness.snapshot(InboxDoc, fork.id, context))?.items ?? []).toEqual([]); + state.gate.resolve(); + await root.waitForIdle(context); + await harness.close(context); + }); + + // #11141 + it("gives a run without a recorded first entry its answer as first", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + let root: Awaited>["root"] | undefined; + setup.registry.install( + defineExtension({ + name: "legacy", + hooks: [ + hook(GenerationTask, { + // As a run started before `first` existed. + onYield: async () => { + await root!.commit(async (tx) => { + delete (await tx.doc(LiveDoc, root!.id)).run!.first; + }, context); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + expect(state.asked).toEqual([{ reason: "answer", first: answer.id, answer: answer.id, last: answer.id }]); + await opened.harness.close(context); + }); + + // #11141 + it("ends the run normally when its after-run task is cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + const generation = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!.run!.taskId; + // The task is cut off, and the waiting generation is not: the scheduler faults the task, and the run ends. + opened.kill(); + state.gate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect((await opened.harness.getTask(generation, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed" }, + }); + expect((await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.runFailure).toBeUndefined(); + expect(state.runs).toEqual(["run 1"]); + expect(setup.infos).toContainEqual(expect.objectContaining({ kind: "task_faulted", reason: "restart_limit" })); + await opened.harness.close(context); + }); + + // #11141 + it("reports no task_interrupted for a task that ends normally after the limit was lowered", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = { started: 0 }; + const Worker = defineTask({ + name: "test.worker", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, ctx) => { + state.started++; + if (state.started === 1) return new Promise(() => {}); // the run a crash cuts off + // The host lowers the limit while the rerun runs. + setup.settings.restartLimit = 1; + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); + setup.registry.install(defineExtension({ name: "worker", tasks: [Worker] })); + let opened = await openKillable(path, setup); + const id = await opened.root.commit( + (tx) => tx.createTask(Worker, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + await eventually(() => state.started === 1); + opened.kill(); + opened = await openKillable(path, setup); + expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + expect(setup.infos.filter((info) => info.kind === "task_interrupted")).toEqual([]); + setup.settings.restartLimit = undefined; + await opened.harness.close(context); + }); + + // #11141 + it("reports no task_interrupted for a generation that ends normally after the limit was lowered in finish", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); // `finish` cut off once + const rerun = deferred(); + state.hookGate = rerun; + opened = await openKillable(path, setup); + opened.harness.resume(); + await eventually(() => state.asked.length === 2); + // The host lowers the limit while the hook runs; the phase had started under the old one and ends normally. + setup.settings.restartLimit = 1; + rerun.resolve(); + await opened.root.waitForIdle(context); + expect(state.runs).toEqual(["run 2"]); + expect(setup.infos.filter((info) => info.kind === "task_interrupted")).toEqual([]); + await opened.harness.close(context); + }); + + // #11141 + it("asks after a terminate round at the turn limit: terminate wins", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("finish", "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + expect((await (await root.submit({ type: "input", content: "go" }, context)).wait(context)).status).toBe("done"); + await root.waitForIdle(context); + expect(state.asked).toMatchObject([{ reason: "terminate" }]); + expect(state.runs).toEqual(["run 1"]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + await harness.close(context); + }); + + // #11141 + it("starts a continue's history at its first tool round when nothing failed and nothing was placed", async () => { + const setup = chatSetup(); + const kinds: string[][] = []; + setup.registry.install(tools); + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + kinds.push((await api.entries(range, 100, undefined, ctx)).items.map((entry) => entry.kind)); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + // A host writes the user message itself, then continues. + await root.submit( + { type: "write", entry: { kind: "app.prompt", model: [{ role: "user", content: "go", timestamp: 1 }] } }, + context, + ); + await (await root.submit({ type: "continue" }, context)).wait(context); + await root.waitForIdle(context); + expect(kinds).toEqual([["pi.assistant", "pi.tool-result", "pi.assistant"]]); + await harness.close(context); + }); + + // #11141 + it("leaves a summary queued during the tasks stale behind a terminate round's handoff", async () => { + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 1, backgroundTokens: 0 }; + const state = probe(); + install(setup, state); + const gate = deferred(); + const started = deferred(); + setup.registry.install( + defineExtension({ + name: "slow-handoff", + tools: [ + defineTool({ + name: "handoff", + description: "End the run in a new context", + parameters: Parameters, + execute: async () => { + started.resolve(); + await gate.promise; + return { + output: [{ type: "text", text: "handed off" }], + control: { terminate: true, handoff: "fresh start" }, + }; + }, + }), + ], + hooks: [hook(CompactionTask, { beforeCompact: async () => ({ summary: "earlier work" }) })], + }), + ); + setup.faux.setResponses([call("handoff", "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const compaction = await root.compact(undefined, context); + await harness.waitForTask(compaction, context); + gate.resolve(); + await submission.wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + // The handoff's reset landed with the answer; the summary, cutting before it, settled stale at the run's end. + expect(kindOf(entries, "pi.compaction")).toEqual([]); + expect(entries.map((entry) => entry.kind).slice(-2)).toEqual(["pi.reset", "test.tests"]); + expect(state.asked[0]).toMatchObject({ reason: "terminate", last: kindOf(entries, "pi.reset")[0]!.id }); + await harness.close(context); + }); + + // #11141 + it("stores the answer's held progress with commit A, and asks again after an orderly close in finish", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ tokensPerSecond: 200, tokenSize: { min: 1, max: 1 } }); + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + const state = probe(); + const Tests = testsTask(state); + let hookGate: Deferred | undefined = deferred(); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, _api, ctx) => { + state.asked.push(run); + if (hookGate !== undefined) await Promise.race([hookGate.promise, aborted(ctx.abortSignal!)]); + return { tasks: [runEndTask(Tests, { label: "run" })] }; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("x".repeat(40))]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); + opened = await openKillable(path, setup); + const stored = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!; + expect(stored.generation).toBeUndefined(); + expect(stored.run?.afterRun).toEqual({}); + expect(textOf(kindOf(await allEntries(opened.root), "pi.assistant")[0]!.model![0])).toBe("x".repeat(40)); + opened.harness.resume(); + await eventually(() => state.asked.length === 2); + await opened.harness.close(context); + hookGate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(3); + expect(state.runs).toEqual(["run"]); + await opened.harness.close(context); + }); +}); + +describe("endRun", () => { + const tx = { settleSubmission: () => {} } as unknown as Tx; + + // #11141 + it("leaves a run it does not own alone, and ends an answered run without a failure", () => { + const live = { + run: { taskId: 1 as TaskId, inputs: [], turns: 0, afterRun: { tasks: [5 as TaskId] } }, + } as unknown as Draft; + endRun(tx, live, 2 as TaskId, { status: "unanswered", reason: "faulted" }); + expect(live.run?.afterRun).toEqual({ tasks: [5] }); + endRun(tx, live, 1 as TaskId, { status: "unanswered", reason: "aborted" }); + expect(live.run).toBeUndefined(); + expect(live.runFailure).toBeUndefined(); + }); + + // #11141 + it("copies a failed run's first entry into runFailure", () => { + const live = { run: { taskId: 1 as TaskId, inputs: [], turns: 0, first: 7 } } as unknown as Draft; + endRun(tx, live, 1 as TaskId, { status: "unanswered", reason: "model_error" }); + expect(live.runFailure).toMatchObject({ reason: "model_error", first: 7 }); + }); +}); diff --git a/packages/durable/test/harness-blocked.test.ts b/packages/durable/test/harness-blocked.test.ts index 6cd234cc6..b51231103 100644 --- a/packages/durable/test/harness-blocked.test.ts +++ b/packages/durable/test/harness-blocked.test.ts @@ -12,6 +12,7 @@ import { } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { context, flush } from "./session-support.ts"; import { completed, deferred, eventually, openTasks } from "./task-support.ts"; @@ -104,7 +105,7 @@ describe("recovery at open", () => { ...rest, state: { status: "completing", outcome: { status: "completed", result: null } }, } as const; - await raw.commit([{ type: "task", value: completing }], () => {}, context); + await commitWrites(raw, [{ type: "task", value: completing }], context); await raw.close(context); // That pass's commit fails, as on a platform reset. @@ -125,14 +126,15 @@ describe("recovery at open", () => { }); const wakes: Wake[] = []; const { harness } = await openTasks(failing, [], { registry, onWake: (wake) => wakes.push(wake) }); - expect((await harness.closed).reason).toBe("failed"); + expect((await harness.closed).kind).toBe("failed"); // The host was told work is underway, and never that it ended: its recovery alarm stays. expect(wakes).toEqual([{ active: true }]); }); }); describe("tasks that cannot run", () => { - it("parks a task cut off restartLimit times in a row, shows why, and runs it again on retryTask()", async () => { + // #11139 + it("faults an extension's task cut off restartLimit times in a row instead of running it again", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const registry = createRegistry(); @@ -150,31 +152,32 @@ describe("tasks that cannot run", () => { } const { harness, infos } = opened; expect((await harness.getTask(id, context))?.restarts).toBe(3); - expect(infos).toEqual([ - { type: "task_parked", task: { id, kind: "test.crash-loop", conversationId: root.id }, extension: "loop" }, - ]); + // Nothing is parked: the task is ready, and the host is asked to run, which ends it. const inspection = await harness.inspect(context); expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ - kind: "blocked", - reason: "restart_limit", + kind: "pending", + migrates: false, }); - const graph = await harness.watchTaskGraph(context); - // A started watch keeps its value current. - graph.start(async () => {}); - expect(graph.value.tasks[String(id)]?.blocked).toBe("restart_limit"); - // Parked work needs no process: nothing is active. - await eventually(() => !harness.wake().active); - expect(harness.wake()).toEqual({ active: false }); + expect(harness.wake().active).toBe(true); harness.resume(); - // A scheduling pass runs within a few turns; it leaves the parked task out. - for (let turn = 0; turn < 5; turn++) await flush(); + const settled = await harness.waitForTask(id, context); + expect(settled.state.outcome).toEqual({ + status: "faulted", + error: { + message: `Task ${id} (test.crash-loop) was cut off 3 times without progress`, + detail: { reason: "restart_limit" }, + }, + }); expect(state.started).toBe(3); - - state.crashing = false; - await harness.retryTask(id, context); - expect((await harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); - expect(state.started).toBe(4); - await graph.stop(); + await eventually(() => infos.length > 0); + expect(infos).toEqual([ + { + kind: "task_faulted", + task: { id, kind: "test.crash-loop", conversationId: root.id }, + extension: "loop", + reason: "restart_limit", + }, + ]); await harness.close(context); }); @@ -264,11 +267,11 @@ describe("tasks that cannot run", () => { expect(settled.restarts).toBeUndefined(); expect(aborts).toBe(1); await eventually(() => opened.infos.length > 0); - expect(opened.infos).toMatchObject([{ type: "task_faulted", task: { id }, reason: "restart_limit" }]); + expect(opened.infos).toMatchObject([{ kind: "task_faulted", task: { id }, reason: "restart_limit" }]); await opened.harness.close(context); }); - it("runs a parked task once a raised restartLimit no longer parks it", async () => { + it("reads restartLimit when it reserves the task: a raised one runs it again", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const registry = createRegistry(); @@ -284,24 +287,17 @@ describe("tasks that cannot run", () => { await eventually(() => state.started === 1); opened.kill(); // a crash: a close would not count opened = await openWith(path, registry, settings); - const graph = await opened.harness.watchTaskGraph(context); - // A started watch keeps its value current. - graph.start(async () => {}); - opened.harness.resume(); - await eventually(() => graph.value.tasks[String(id)]?.blocked === "restart_limit"); - // Let a scheduling pass leave the parked task out. - for (let turn = 0; turn < 5; turn++) await flush(); + expect((await opened.harness.getTask(id, context))?.restarts).toBe(1); state.crashing = false; - // The host's live settings change; the next pass runs the task. + // The host's live settings change before the task is reserved; it runs instead of ending. settings.restartLimit = Number.POSITIVE_INFINITY; opened.harness.resume(); expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); expect(state.started).toBe(2); - await graph.stop(); await opened.harness.close(context); }); - it("still runs the abort handler of a parked task", async () => { + it("runs the abort handler of a task at the limit", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const aborted: TaskId[] = []; @@ -362,10 +358,12 @@ describe("tasks that cannot run", () => { opened = await openWith(path, registry, { restartLimit: 2 }); } expect((await opened.harness.getTask(id, context))?.restarts).toBe(2); - expect((await opened.harness.inspect(context)).tasks[0]?.state).toEqual({ - kind: "blocked", - reason: "restart_limit", + opened.harness.resume(); + expect((await opened.harness.waitForTask(id, context)).state.outcome).toMatchObject({ + status: "faulted", + error: { detail: { reason: "restart_limit" } }, }); + expect(started).toBe(2); await opened.harness.close(context); }); diff --git a/packages/durable/test/harness-callers.test.ts b/packages/durable/test/harness-callers.test.ts index ff9a8311e..bbf4e7db9 100644 --- a/packages/durable/test/harness-callers.test.ts +++ b/packages/durable/test/harness-callers.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -7,6 +7,7 @@ import { type NestedToolExecutionResult, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -16,7 +17,7 @@ function tool(name: string, extra: Partial = {}): ToolRegistra return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -34,7 +35,7 @@ function prober(names: readonly string[]) { const registration = defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { for (const name of names) received.set(name, await api.executeTool(name, {}, callContext)); return {}; diff --git a/packages/durable/test/harness-cancellation-barrier.test.ts b/packages/durable/test/harness-cancellation-barrier.test.ts index 086787e43..9564e39b0 100644 --- a/packages/durable/test/harness-cancellation-barrier.test.ts +++ b/packages/durable/test/harness-cancellation-barrier.test.ts @@ -1,5 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { + type CommitResult, createSession, defineTask, MemoryStorage, @@ -8,6 +9,7 @@ import { type TaskId, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { context, flush } from "./session-support.ts"; import { aborted, deferred, eventually, openTasks } from "./task-support.ts"; @@ -26,19 +28,25 @@ const Dormant = defineTask({ }, }); -/** Holds every commit that abort-marks `target`, while set, until released. */ +/** Holds every commit call that abort-marks `target`, while set, until released. */ class HoldMarks extends MemoryStorage { target: TaskId | undefined; readonly held = deferred(); readonly release = deferred(); - override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) { + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { if ( - writes.some((write) => write.type === "task" && write.value.id === this.target && write.value.abortRequested) + batches + .flat() + .some((write) => write.type === "task" && write.value.id === this.target && write.value.abortRequested) ) { this.held.resolve(); await this.release.promise; } - return super.commit(writes, adopt, commitContext); + return super.commit(batches, receive, commitContext); } } @@ -127,7 +135,8 @@ describe("work below a cancelled owner", () => { policy: "failFast", } as const; const failure = { status: "failed", error: { message: "failed" } } as const; - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: { ...owner, abortRequested: true, abortReason: "restart", state: waiting } }, { @@ -135,7 +144,6 @@ describe("work below a cancelled owner", () => { value: { ...failedRecord, state: { status: "terminal", outcome: failure } } as typeof failedRecord, }, ], - () => {}, context, ); // The child's definition is missing: a restart mark would keep it waiting, a request orphans it. @@ -217,9 +225,9 @@ describe("work below a cancelled owner", () => { ); const record = (await storage.task(holder, context))!; const completing = { status: "completing", outcome: { status: "completed", result: null } } as const; - await storage.commit( + await commitWrites( + storage, [{ type: "task", value: { ...record, state: completing } as typeof record }], - () => {}, context, ); const { harness } = await openTasks(storage, [Dormant]); diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index 61f1f1d14..1be32ae35 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -9,33 +9,37 @@ import { type Message, type SimpleStreamOptions, type TranscriptContext, - Type, type UserMessage, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type CompactionPolicy, type CompactionResult, CompactionTask, type ContextView, type Conversation, + createSession, + defineDoc, defineTask, defineTool, type EntryRecord, type Harness, + type JsonObject, LiveDoc, + type LiveState, MemoryStorage, ProviderDoc, + type SubmissionId, type TaskId, UsageDoc, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { selectCut, serializeConversation } from "../src/harness/compaction.ts"; import { orderToolResults } from "../src/harness/context.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, toolsNamed, waitFor } from "./chat-support.ts"; import { addHooks, addSection, addTask, addTool } from "./harness-support.ts"; +import { viewHistory } from "./live-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; @@ -467,7 +471,7 @@ describe("manual compaction", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: "waited" }] }), }), ); @@ -882,7 +886,7 @@ describe("compaction outcomes", () => { defineTool({ name: "read", description: "read", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -962,7 +966,7 @@ describe("background threshold compaction", () => { expect(task).toMatchObject({ background: true, input: { reason: "threshold" } }); expect(task!.owner).toBeUndefined(); expect((await live(chat)).compactions).toEqual([ - { taskId: task!.id, reason: "threshold", blocking: false, attempt: 1 }, + { taskId: task!.id, trigger: "threshold", blocking: false, attempt: 1 }, ]); // Background work: neither conversation idle nor Esc waits for or stops it. await chat.root.waitForIdle(context); @@ -1006,40 +1010,37 @@ describe("background threshold compaction", () => { await chat.harness.close(context); }); - it("records a failure in pi.live, the snapshot, and compaction_end until a summary is produced", async () => { + it("records a failure in pi.live, in the commit that ends the compaction, until a summary is produced", async () => { const chat = await open({ contextWindow: 2000 }); await history(chat); chat.setup.settings.compaction = BACKGROUND; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(failure("bad request")); await turn(chat, text("u4", 100), text("a4", 100)); - await waitFor(() => events.some((event) => event.type === "compaction_end")); + await waitFor(async () => (await live(chat)).compactionFailure !== undefined); const recorded = (await live(chat)).compactionFailure; - expect(recorded).toMatchObject({ reason: "threshold", error: "Summarization failed: bad request" }); - expect(events.find((event) => event.type === "compaction_end")).toEqual({ - type: "compaction_end", - taskId: recorded!.taskId, - reason: "threshold", - error: "Summarization failed: bad request", - }); - const reopened = await watchEvents(chat.harness, chat.root.id, context); - expect(reopened.snapshot.compactionFailure).toEqual(recorded); - await reopened.stop(); - // A compaction that produces a summary removes the failure, and ends without an error. + expect(recorded).toMatchObject({ trigger: "threshold", error: "Summarization failed: bad request" }); + // The value that drops the compaction's status records its failure. + const ended = (taskId: TaskId) => { + const states = views.live(); + const listed = states.findIndex((state) => state.compactions?.some((status) => status.taskId === taskId)); + return states.find( + (state, index) => index > listed && !state.compactions?.some((status) => status.taskId === taskId), + )!; + }; + expect(ended(recorded!.taskId).compactionFailure).toEqual(recorded); + // A late watch starts with it. + const late = await chat.root.watch(context); + expect((late.value.docs["pi.live"] as LiveState).compactionFailure).toEqual(recorded); + await late.stop(); + // A compaction that produces a summary removes the failure in the commit that ends it. chat.setup.settings.compaction = MANUAL; chat.faux.summaries.push(summary()); const manual = await chat.root.compact(undefined, context); expect((await result(chat, manual)).status).toBe("completed"); expect((await live(chat)).compactionFailure).toBeUndefined(); - await waitFor(() => events.some((event) => event.type === "compaction_end" && event.taskId === manual)); - expect(events.find((event) => event.type === "compaction_end" && event.taskId === manual)).not.toHaveProperty( - "error", - ); - await stream.stop(); + expect(ended(manual).compactionFailure).toBeUndefined(); + views.detach(); await chat.harness.close(context); }); @@ -1084,7 +1085,7 @@ describe("blocking threshold compaction", () => { checkpoint: { phase: "prepare", attempt: 1, compacted: child!.id }, }); expect((await live(chat)).compactions).toEqual([ - { taskId: child!.id, reason: "threshold", blocking: true, attempt: 1 }, + { taskId: child!.id, trigger: "threshold", blocking: true, attempt: 1 }, ]); // Nothing was appended before the wait. expect((await kinds(chat.root)).at(-1)).toBe("pi.user"); @@ -1144,7 +1145,7 @@ describe("blocking threshold compaction", () => { const state = await live(chat); expect(state.compactions).toBeUndefined(); expect(state.compactionFailure).toMatchObject({ - reason: "threshold", + trigger: "threshold", error: "Summarization failed: bad request", }); await chat.harness.close(context); @@ -1179,21 +1180,22 @@ describe("blocking threshold compaction", () => { chat.setup.settings.compaction = BLOCKING; const reached = deferred(); chat.faux.summaries.push(gated(deferred(), summary(), reached)); - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); const input = await chat.root.submit({ type: "input", content: text("u4", 200) }, context); await reached.promise; const [child] = await compactionTasks(chat); await chat.root.abort(context); expect(await input.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect((await chat.harness.getTask(child!.id, context))?.state).toMatchObject({ outcome: { status: "aborted" } }); - await waitFor(() => events.some((event) => event.type === "run_end")); - const types = events.map((event) => event.type); - expect(types.indexOf("compaction_end")).toBeLessThan(types.indexOf("run_end")); - await stream.stop(); + views.detach(); + // The compaction's status leaves before the run ends. + const states = views.live(); + const busy = states.findIndex((state) => state.compactions !== undefined); + const compactionEnd = states.findIndex((state, index) => index > busy && state.compactions === undefined); + const runEnd = states.findIndex((state, index) => index > busy && state.run === undefined); + expect(busy).toBeGreaterThan(0); + expect(compactionEnd).toBeLessThan(runEnd); + expect(states[runEnd]!.runFailure).toMatchObject({ inputs: [input.id], reason: "aborted" }); await chat.harness.close(context); }); @@ -1341,7 +1343,7 @@ describe("compaction estimates and interactions", () => { defineTool({ name: "slow", description: "slow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { toolReached.resolve(); await toolGate.promise; @@ -1448,9 +1450,11 @@ describe("compaction estimates and interactions", () => { expect(view.messages.slice(1).map(userText)).toEqual([text("u3", 100), text("a3", 100)]); // The parent is untouched. expect((await kinds(chat.root)).includes("pi.compaction")).toBe(false); - // The fork's view keeps the parent entries the summary kept. + // The fork's view shows the parent entries in ID order, the summary after them, its head on a parent entry. const watch = await fork.watch(context); - expect(watch.value.entries).toEqual(view.entries); + expect(watch.value.entries).toEqual(await allEntries(fork)); + expect(watch.value.entries.at(-1)).toEqual(view.head); + expect(watch.value.entries.some((entry) => entry.id === u3.id)).toBe(true); await watch.stop(); await chat.harness.close(context); }); @@ -1565,24 +1569,20 @@ function defineChildTask(gate: Deferred) { }); } -describe("compaction events and live status", () => { - it("reports start and end, and the retry backoff in a late joiner's snapshot", async () => { +describe("compaction live status", () => { + it("lists a compaction from start to end, with its retry backoff in a late joiner's view", async () => { const chat = await open(); await history(chat); chat.setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(failure("overloaded")); const id = await chat.root.compact(undefined, context); await waitFor(async () => (await live(chat)).compactions?.[0]?.retry !== undefined); - const late = await watchEvents(chat.harness, chat.root.id, context); - expect(late.snapshot.compactions).toEqual([ + const late = await chat.root.watch(context); + expect((late.value.docs["pi.live"] as LiveState).compactions).toEqual([ { taskId: id, - reason: "manual", + trigger: "manual", blocking: false, attempt: 1, retry: { at: expect.any(Number), error: "overloaded" }, @@ -1590,12 +1590,12 @@ describe("compaction events and live status", () => { ]); await late.stop(); await chat.harness.abortTask(id, context); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - expect(events.filter((event) => event.type.startsWith("compaction_"))).toEqual([ - { type: "compaction_start", taskId: id, reason: "manual", blocking: false }, - { type: "compaction_end", taskId: id, reason: "manual" }, - ]); - await stream.stop(); + await waitFor(async () => (await live(chat)).compactions === undefined); + views.detach(); + const listed = views.live().flatMap((state) => state.compactions ?? []); + expect(listed[0]).toEqual({ taskId: id, trigger: "manual", blocking: false, attempt: 1 }); + // An abort is not a failure. + expect(views.live().at(-1)!.compactionFailure).toBeUndefined(); await chat.harness.close(context); }); @@ -1893,15 +1893,21 @@ describe("compaction and the inbox", () => { await chat.harness.close(context); }); - it("keeps the kept entries mounted in the conversation view", async () => { + it("appends the summary to the conversation view in place, its head on a kept entry", async () => { const chat = await open(); await history(chat); const watch = await chat.root.watch(context); + const before = watch.value.entries; watch.start(async () => {}); chat.faux.summaries.push(summary()); await result(chat, await chat.root.compact(undefined, context)); - await waitFor(() => watch.value.entries[0]?.kind === "pi.compaction"); - expect(watch.value.entries).toEqual((await chat.root.context(context)).entries); + await waitFor(() => watch.value.entries.at(-1)?.kind === "pi.compaction"); + // Nothing leaves the window: the summary is appended where it was written. + expect(watch.value.entries.slice(0, before.length)).toEqual(before); + expect(watch.value.entries).toEqual(await allEntries(chat.root)); + const head = (await chat.root.context(context)).head!; + expect(watch.value.entries.at(-1)).toEqual(head); + expect(watch.value.entries.some((entry) => entry.id === head.head)).toBe(true); await watch.stop(); await chat.harness.close(context); }); @@ -1988,7 +1994,7 @@ describe("compaction edge cases", () => { expect((await kinds(chat.root)).includes("pi.compaction")).toBe(false); const state = await live(chat); expect(state.compactions).toBeUndefined(); - expect(state.compactionFailure).toMatchObject({ reason: "threshold", error: "no credentials" }); + expect(state.compactionFailure).toMatchObject({ trigger: "threshold", error: "no credentials" }); await chat.harness.close(context); }); @@ -2028,41 +2034,36 @@ describe("compaction edge cases", () => { await chat.harness.close(context); }); - it("orders the events of a summary placed at once", async () => { + it("places a summary at once in the commit that ends its compaction", async () => { const chat = await open(); await history(chat); - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(summary()); await result(chat, await chat.root.compact(undefined, context)); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - const end = events.findIndex((event) => event.type === "compaction_end"); - const batch = events.slice(end - 2, end + 3).map((event) => event.type); - expect(batch).toEqual(["message_start", "message_end", "compaction_end", "submission", "usage_changed"]); - await stream.stop(); + views.detach(); + const states = views.live(); + const listed = states.findIndex((state) => state.compactions !== undefined); + const ended = states.findIndex((state, index) => index > listed && state.compactions === undefined); + const before = views.views[ended - 1]!; + const after = views.views[ended]!; + expect(after.entries.slice(before.entries.length).map((entry) => entry.kind)).toEqual(["pi.compaction"]); await chat.harness.close(context); }); - it("puts compaction_start last in the batch of the preparation commit", async () => { + it("lists a background compaction in the preparation commit, with the entries it appends", async () => { const chat = await open({ contextWindow: 2000 }); await history(chat); chat.setup.settings.compaction = BACKGROUND; - const events: AgentEvent[][] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push([...batch]); - }); + const views = await viewHistory(chat.harness, chat.root.id); // A new section makes the preparation commit append a system entry next to the compaction's status. addSection(chat.setup.registry, "extra", () => "extra"); chat.faux.summaries.push(gated(deferred(), summary())); await turn(chat, text("u4", 100), "a4"); - await waitFor(() => events.some((batch) => batch.some((event) => event.type === "compaction_start"))); - const batch = events.find((candidate) => candidate.some((event) => event.type === "compaction_start"))!; - expect(batch.map((event) => event.type)).toEqual(["message_start", "message_end", "compaction_start"]); - await stream.stop(); + await waitFor(async () => (await live(chat)).compactions !== undefined); + views.detach(); + const started = views.live().findIndex((state) => state.compactions !== undefined); + const appended = views.views[started]!.entries.slice(views.views[started - 1]!.entries.length); + expect(appended.map((entry) => entry.kind)).toEqual(["pi.system"]); await chat.harness.abortTask((await compactionTasks(chat))[0]!.id, context); await chat.harness.close(context); }); @@ -2129,7 +2130,7 @@ describe("compaction edge cases", () => { return summary(); }); const id = await chat.root.compact(undefined, context); - expect(await chat.harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await chat.harness.closed).toEqual({ kind: "failed", error: failure }); await chat.harness.close(context); // Nothing of the failed commit landed: the task is still live, without usage, entry, or submission. storage.reopen(); @@ -2283,9 +2284,11 @@ describe("compaction pinning, silent overflow, and late policy changes", () => { chat.faux.summaries.push(gated(deferred(), summary(), reached)); const id = await chat.root.compact(undefined, context); await reached.promise; - const stream = await watchEvents(chat.harness, chat.root.id, context); - expect(stream.snapshot.compactions).toEqual([{ taskId: id, reason: "manual", blocking: false, attempt: 1 }]); - await stream.stop(); + const watch = await chat.root.watch(context); + expect((watch.value.docs["pi.live"] as LiveState).compactions).toEqual([ + { taskId: id, trigger: "manual", blocking: false, attempt: 1 }, + ]); + await watch.stop(); await chat.harness.abortTask(id, context); await chat.harness.close(context); }); @@ -2349,18 +2352,13 @@ describe("blocked compaction", () => { const id = await root.commit(async (tx) => { const taskId = await tx.createTask(newer, { reason: "manual" }, { ownership: { kind: "conversation" } }); const live = await tx.doc(LiveDoc, root.id); - live.compactions = [{ taskId, reason: "manual", blocking: false, attempt: 1 }]; + live.compactions = [{ taskId, trigger: "manual", blocking: false, attempt: 1 }]; return taskId; }, context); await harness.close(context); ({ harness, root } = await openChat(await openNodeSqliteStorage(path), setup)); harness.resume(); const chat = { harness, root, setup, faux: script(setup) }; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); const inspection = await chat.harness.inspect(context); expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ kind: "blocked", @@ -2372,8 +2370,7 @@ describe("blocked compaction", () => { reason: "task_too_old", }); expect((await live(chat)).compactions).toBeUndefined(); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - await stream.stop(); + expect((await live(chat)).compactionFailure).toEqual({ taskId: id, trigger: "manual", error: "task_too_old" }); await chat.harness.close(context); }); }); @@ -2456,3 +2453,95 @@ describe("per-model compaction overrides", () => { await chat.harness.close(context); }); }); + +describe("pi.live migrations", () => { + it("reads a version 1 value with each compaction's `reason` as `trigger`, and stores it migrated on its next write", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + // `pi.live` as version 1 wrote it. + const LiveV1 = defineDoc({ + kind: "pi.live", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), + checkpointWhen: () => false, + }); + const conversationId = await session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await session.commit(async (tx) => { + const live = await tx.doc(LiveV1, conversationId); + live.run = { taskId: 9, inputs: [4] }; + live.compactions = [{ taskId: 7, reason: "threshold", blocking: true, attempt: 2 }]; + live.compactionFailure = { taskId: 5, reason: "manual", error: "no model" }; + }, context); + // The version 2 token reloads the stored value and migrates it. + const stored = async () => { + const record = await storage.findDocument( + { kind: "pi.live", scope: { kind: "conversation", conversationId } }, + "current", + context, + ); + return (await storage.document(record!.id, "current", context))!; + }; + const migrated: LiveState = { + // A run in flight across the upgrade counts its turns from 0 (#11043). + run: { taskId: 9 as TaskId, inputs: [4 as SubmissionId], turns: 0 }, + compactions: [{ taskId: 7 as TaskId, trigger: "threshold", blocking: true, attempt: 2 }], + compactionFailure: { taskId: 5 as TaskId, trigger: "manual", error: "no model" }, + }; + const read = await session.snapshot(LiveDoc, conversationId, context); + expect(read).toEqual(migrated); + expect(Object.keys(read!.compactions![0]!)).toEqual(["taskId", "trigger", "blocking", "attempt"]); + // A read migrates in memory only: Storage keeps the version 1 value. + expect(await stored()).toMatchObject({ + version: 1, + value: { compactions: [{ reason: "threshold" }], compactionFailure: { reason: "manual" } }, + }); + await session.commit(async (tx) => { + (await tx.doc(LiveDoc, conversationId)).compactions![0]!.attempt = 3; + }, context); + expect(await stored()).toMatchObject({ + version: 3, + value: { compactions: [{ trigger: "threshold", attempt: 3 }], compactionFailure: { trigger: "manual" } }, + }); + await session.close(context); + }); + + // #11043: an unreleased build stored version 2 without `run.turns`; a run in flight then counted NaN turns. + it("gives a version 2 run without turns a count of 0", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + const LiveV2 = defineDoc({ + kind: "pi.live", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), + checkpointWhen: () => false, + }); + const conversationId = await session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await session.commit(async (tx) => { + const live = await tx.doc(LiveV2, conversationId); + live.run = { taskId: 9, inputs: [4] }; + live.compactions = [{ taskId: 7, trigger: "threshold", blocking: true, attempt: 2 }]; + }, context); + expect(await session.snapshot(LiveDoc, conversationId, context)).toEqual({ + run: { taskId: 9, inputs: [4], turns: 0 }, + compactions: [{ taskId: 7, trigger: "threshold", blocking: true, attempt: 2 }], + }); + // A version 2 run that has a count keeps it. + await session.commit(async (tx) => { + (await tx.doc(LiveV2, conversationId)).run = { taskId: 9, inputs: [4], turns: 3 }; + }, context); + expect((await session.snapshot(LiveDoc, conversationId, context))?.run?.turns).toBe(3); + await session.close(context); + }); +}); diff --git a/packages/durable/test/harness-context-range.test.ts b/packages/durable/test/harness-context-range.test.ts new file mode 100644 index 000000000..d44cab3db --- /dev/null +++ b/packages/durable/test/harness-context-range.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { advanceRange, readContext, readContextFrom } from "../src/harness/context.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { MemoryStorage } from "../src/storage/memory.ts"; +import { context } from "./session-support.ts"; + +describe("kept context ranges", () => { + it("is not advanced when an entry was committed while its scan ran off the line", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + class PausedStorage extends MemoryStorage { + pause = false; + override async scanEntries(...args: Parameters) { + const page = await super.scanEntries(...args); + if (this.pause) { + this.pause = false; + entered.resolve(); + await release.promise; + } + return page; + } + } + const storage = new PausedStorage(); + const session = new SessionImpl(storage); + const conversation = await session.commit( + (tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), + context, + ); + await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "first" }), context); + storage.pause = true; + const read = readContextFrom(session, storage, conversation.id, context, undefined, undefined); + await entered.promise; + await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "missed" }), context); + release.resolve(); + const { range } = await read; + expect(range?.current).toBe(false); + const last = await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "last" }), context); + // Not current: left as it is, so the next read extends it from Storage instead of skipping "missed". + expect(advanceRange(range!, [last])).toBe(range); + const again = await readContextFrom(session, storage, conversation.id, context, undefined, range); + const fresh = await readContext(session, storage, conversation.id, context); + expect(again.view.entries.map((entry) => entry.kind)).toEqual(["first", "missed", "last"]); + expect(again.view).toEqual(fresh); + expect(again.range?.current).toBe(true); + await session.close(context); + }); +}); diff --git a/packages/durable/test/harness-context.test.ts b/packages/durable/test/harness-context.test.ts index f3a9ada0e..c0e6d13b5 100644 --- a/packages/durable/test/harness-context.test.ts +++ b/packages/durable/test/harness-context.test.ts @@ -444,18 +444,18 @@ describe("conversation context", () => { const initial = await read(); expect(initial.view).toEqual(await root.context(taskContext)); - // Three new entries: the bounds probe reads one row, the extension the three new ones. + // Three new entries: the kept range took them as they were committed, so the read scans nothing. await write({ kind: "message", model: [user("new")] }); await write({ kind: "note", data: { text: "display only" } }); await write({ kind: "message", model: [assistant("answer")] }); const extended = await read(); - expect(extended.rows).toBe(4); + expect(extended.rows).toBe(0); expect(extended.view).toEqual(await root.context(taskContext)); // A newer edit of an older entry applies to the extended range. await write({ kind: "edit", edits: [{ target: first.id, action: "omit" }] }); const edited = await read(); - expect(edited.rows).toBe(2); + expect(edited.rows).toBe(0); expect(edited.view).toEqual(await root.context(taskContext)); expect(edited.view.messages.map(describeMessage)).not.toContain("user:first"); @@ -559,9 +559,9 @@ describe("conversation context", () => { await probe("idle"); clock += 600_000; await probe("expired"); - // Bounds probes read one row each. The child scans its own new entry; the waiting parent's later invocation and - // the probe within the retention period scan nothing new. After it, the probe reads the whole transcript again. - expect(rows).toEqual({ parent: 22, child: 2, after: 1, idle: 1, expired: 23 }); + // The Session knows the bounds, and the kept range takes each committed entry, so only a first read scans: the + // parent's, and the probe's after the retention period, which reads the whole transcript again. + expect(rows).toEqual({ parent: 21, child: 0, after: 0, idle: 0, expired: 22 }); await harness.close(context); }); @@ -732,8 +732,8 @@ describe("conversation context", () => { expect((await read(root)).messages[0]).toBe("user:first"); await append(root.id, { kind: "pi.system", model: [system({ plan: "Plan." })] }); let read2 = await read(root); - // One bounds probe and the added entry: the range was extended, not derived again. - expect(read2.rows).toBe(2); + // Nothing: the kept range took the added entry as it was committed. + expect(read2.rows).toBe(0); expect(read2.messages.slice(0, 3)).toEqual(["system:plan", "user:first", "assistant:old 0"]); // A system message the cached range already has keeps a later first `pi.system` entry in place. @@ -743,7 +743,7 @@ describe("conversation context", () => { expect((await read(fork)).messages).toEqual(["user:first", "assistant:answer", "system:note"]); await append(fork.id, { kind: "pi.system", model: [system({ note: null })] }); read2 = await read(fork); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:note", "system:note"]); // An omitted system message still rules out a later baseline; the extended read keeps that. @@ -761,7 +761,7 @@ describe("conversation context", () => { expect((await read(omitted)).messages).toEqual(["user:first", "assistant:answer"]); await append(omitted.id, { kind: "pi.system", model: [system({ legacy: null })] }); read2 = await read(omitted); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:legacy"]); // A first `pi.system` entry without a system message leaves no baseline for a later one to take. @@ -771,7 +771,7 @@ describe("conversation context", () => { expect((await read(empty)).messages).toEqual(["user:first", "assistant:answer"]); await append(empty.id, { kind: "pi.system", model: [system({ s: "Delta." })] }); read2 = await read(empty); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:s"]); await runtime.commit(() => DONE, taskContext); }, @@ -907,7 +907,7 @@ describe("conversation context", () => { await late; probeStart.resolve(); expect((await harness.waitForTask(probeId, context)).state.outcome).toEqual(DONE.outcome); - expect(rows).toEqual({ probe: 22 }); + expect(rows).toEqual({ probe: 21 }); await harness.close(context); }); @@ -934,7 +934,7 @@ describe("conversation context", () => { vi.advanceTimersByTime(1_000); expect(vi.getTimerCount()).toBe(idle); await probe("second"); - expect(rows).toEqual({ first: 22, second: 22 }); + expect(rows).toEqual({ first: 21, second: 21 }); await harness.close(context); }); @@ -954,7 +954,7 @@ describe("conversation context", () => { expect((await harness.waitForTask(id, context)).state.outcome).toEqual(DONE.outcome); await root.waitForIdle(context); } - expect(rows).toEqual({ first: 22, second: 22 }); + expect(rows).toEqual({ first: 21, second: 21 }); await harness.close(context); }); }); diff --git a/packages/durable/test/harness-continue.test.ts b/packages/durable/test/harness-continue.test.ts new file mode 100644 index 000000000..627e6218f --- /dev/null +++ b/packages/durable/test/harness-continue.test.ts @@ -0,0 +1,423 @@ +// `submit({ type: "continue" })`: a run without a new user message (#11043). + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { withTelemetry } from "@earendil-works/chord/context"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type Message, +} from "@earendil-works/pi-ai"; +import { + AssistantEntry, + type Conversation, + ConversationBusy, + configure, + defineTool, + Harness, + InboxDoc, + LiveDoc, + MemoryStorage, + type SettledSubmissionRecord, + type Storage, + UserEntry, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-continue-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over the SQLite file at `path` and start scheduling; `kill()` simulates a crash. */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + opened.harness.resume(); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +const failure = (): AssistantMessage => + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + +/** A response step that records the roles of the context it was asked with, then answers `text`. */ +function answering(text: string, seen: Message["role"][][]): FauxResponseStep { + return (request) => { + seen.push(request.messages.map((message) => message.role)); + return fauxAssistantMessage([fauxText(text)]); + }; +} + +async function answerText(root: Conversation, settled: SettledSubmissionRecord): Promise { + if (settled.status !== "done" || settled.type === "write") throw new Error(`Unexpected ${settled.status}`); + const answer = await root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context); + return textOf(answer?.model?.[0]); +} + +async function userEntries(root: Conversation): Promise { + return (await allEntries(root)).filter((entry) => UserEntry.is(entry)).length; +} + +describe("continue", () => { + // #11043 + it("answers a run that failed, without a new user message", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([failure, answering("recovered", seen)]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "Analyze this" }, context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.reason).toBe("model_error"); + + const resumed = await root.submit({ type: "continue" }, context); + expect((await resumed.status(context)).status).toBe("placed"); + const settled = await resumed.wait(context); + expect(settled.type).toBe("continue"); + expect(await answerText(root, settled)).toBe("recovered"); + expect(seen.at(-1)?.at(-1)).toBe("user"); + // The failed input stays unanswered; no user entry was added. + expect(await first.status(context)).toMatchObject({ status: "unanswered" }); + expect(await userEntries(root)).toBe(1); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("answers the tool results of a round aborted midway", async () => { + const setup = chatSetup(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "wait", + description: "Waits until aborted", + parameters: obj({}), + execute: async (_args, _api, callContext) => { + started.resolve(); + return aborted(callContext.abortSignal!); + }, + }), + ); + const seen: Message["role"][][] = []; + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("wait", {}, { id: "c1" })], { stopReason: "toolUse" }), + answering("after the abort", seen), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.abort(context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("after the abort"); + expect(seen.at(-1)?.at(-1)).toBe("toolResult"); + await harness.close(context); + }); + + // #11043 + it("skips system messages when deciding there is something to answer", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([fauxAssistantMessage("first answer"), failure, answering("resumed", seen)]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ instructions: "Be thorough." }, context); + await (await root.submit({ type: "input", content: "one" }, context)).wait(context); + // Changed instructions append a system delta after the next user message, before its request. + await root.configure({ instructions: "Be brief." }, context); + const second = await root.submit({ type: "input", content: "two" }, context); + expect(await second.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + const roles = (await root.context(context)).messages.map((message) => message.role); + expect(roles.at(-1)).toBe("system"); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("resumed"); + await harness.close(context); + }); + + // #11043 + it("is rejected while busy, after request ID deduplication", async () => { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([failure, busy.step]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const resumed = await root.submit({ type: "continue", requestId: "resume-1" }, context); + await busy.reached; + await expect(root.submit({ type: "continue" }, context)).rejects.toBeInstanceOf(ConversationBusy); + // The same request ID returns the existing submission instead of rejecting. + expect((await root.submit({ type: "continue", requestId: "resume-1" }, context)).id).toBe(resumed.id); + await harness.close(context); + }); + + // #11043 + it("ends without a request when there is nothing to answer", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage("answered")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const empty = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(empty).toMatchObject({ type: "continue", status: "unanswered", reason: "nothing_to_answer" }); + await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); + const calls = setup.faux.state.callCount; + // A context ending in an answer has nothing to continue. + const after = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(after).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect(setup.faux.state.callCount).toBe(calls); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.reason).toBe("nothing_to_answer"); + await harness.close(context); + }); + + // #11043 + it("continues an answer with tool calls cut off at its output limit, which ends in their missing results", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([ + fauxAssistantMessage([fauxText("calling"), fauxToolCall("echo", {}, { id: "c1" })], { stopReason: "length" }), + answering("resumed", seen), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + // The run answered: a cut-off answer is an answer, and its calls never ran. + expect((await (await root.submit({ type: "input", content: "go" }, context)).wait(context)).status).toBe("done"); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("resumed"); + expect(seen.at(-1)?.slice(-2)).toEqual(["assistant", "toolResult"]); + await harness.close(context); + }); + + // #11043 + it("does not continue a text answer cut off at its output limit", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("cut")], { stopReason: "length" })]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "write a lot" }, context)).wait(context); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("answers a run interrupted by a host that died while the model answered", async () => { + const path = await sqlitePath(); + const first = chatSetup({ tokensPerSecond: 20, tokenSize: { min: 1, max: 1 } }); + first.faux.setResponses([fauxAssistantMessage("z".repeat(400))]); + first.settings.progress = { persistWithinMs: 0 }; + let opened = await open(path, first); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor( + async () => + (await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.generation?.message !== undefined, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + opened.kill(); + + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const seen: Message["role"][][] = []; + setup.faux.setResponses([answering("after the crash", seen)]); + opened = await open(path, setup); + const interrupted = await (await opened.harness.submission(id, context))!.wait(context); + expect(interrupted).toMatchObject({ status: "unanswered", reason: "interrupted" }); + const settled = await (await opened.root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(opened.root, settled)).toBe("after the crash"); + // The aborted partial is not model context: the request ends in the user message. + expect(seen.at(-1)?.at(-1)).toBe("user"); + await opened.harness.close(context); + }); + + // #11043 + it("places a reset a failed run kept, and finds nothing to answer in the context it starts", async () => { + const setup = chatSetup(); + const gate = deferred(); + const reached = deferred(); + setup.faux.setResponses([ + async () => { + reached.resolve(); + await gate.promise; + return failure(); + }, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await reached.promise; + await root.reset(undefined, context); + gate.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items).toHaveLength(1); + + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items ?? []).toEqual([]); + expect((await root.context(context)).messages.filter((message) => message.role !== "system")).toEqual([]); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("starts the inputs queued behind a continue with nothing to answer under afterFailedRun continue", async () => { + const setup = chatSetup(); + setup.settings.afterFailedRun = "continue"; + setup.faux.setResponses([fauxAssistantMessage("answered"), fauxAssistantMessage("second answered")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "one" }, context)).wait(context); + const resumed = await root.submit({ type: "continue" }, context); + const queued = await root.submit({ type: "input", content: "two", whenBusy: "followUp" }, context); + expect((await queued.status(context)).status).toBe("queued"); + expect(await resumed.wait(context)).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + const settled = await queued.wait(context); + expect(await answerText(root, settled)).toBe("second answered"); + await harness.close(context); + }); + + // #11043 + it("is submitted by a tool through the handle of a conversation it owns, and rejected there while busy", async () => { + const setup = chatSetup(); + const busy = deferred(); + const release = deferred(); + const outcome: { busy?: unknown; first?: string; resumed?: string } = {}; + addTool( + setup.registry, + defineTool({ + name: "delegate", + description: "Delegates", + parameters: obj({}), + execute: async (_args, api, callContext) => { + const child = await api.commit(async (tx) => { + const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); + await configure(tx, created.id, { model: { provider: "faux", modelId: "faux-1" } }); + return created.id; + }, callContext); + const handle = (await api.conversation(child, callContext))!; + const first = await (await handle.submit({ type: "input", content: "work" }, callContext)).wait( + callContext, + ); + outcome.first = first.status; + const resumed = await handle.submit({ type: "continue" }, callContext); + await busy.promise; + outcome.busy = await handle.submit({ type: "continue" }, callContext).catch((error: unknown) => error); + release.resolve(); + const settled = await resumed.wait(callContext); + outcome.resumed = settled.status; + return { output: [{ type: "text", text: settled.status }] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }), + failure, + async () => { + busy.resolve(); + await release.promise; + return fauxAssistantMessage("child recovered"); + }, + fauxAssistantMessage("done"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(outcome).toMatchObject({ first: "unanswered", resumed: "done" }); + expect(outcome.busy).toBeInstanceOf(ConversationBusy); + await harness.close(context); + }); + + // #11043 + it("starts a root trace linked to its caller, as an input does", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + setup.faux.setResponses([failure, fauxAssistantMessage("recovered")]); + const harness = await Harness.open( + new MemoryStorage(), + { models: setup.models, registry: setup.registry, settings: setup.settings, telemetry }, + context, + ); + const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const id = await telemetry.startSpan({ name: "caller" }, async (caller) => { + const callContext = withTelemetry(caller, context); + const resumed = await root.submit({ type: "continue" }, callContext); + await resumed.wait(callContext); + return resumed.id; + }); + await harness.waitForIdle(context); + await waitFor(() => telemetry.getSpans().every((span) => span.settled)); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find( + (span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id, + )!; + expect(submit).toMatchObject({ parentId: null, attributes: { result: "placed" } }); + expect(submit.links).toEqual([{ context: caller.context, attributes: {} }]); + expect((await (await harness.submission(id, context))!.status(context)).trace).toEqual(submit.context); + expect( + spans.some( + (span) => + span.name === "pi.durable.phase" && + span.attributes.phase === "prepare" && + JSON.stringify(span.parentContext) === JSON.stringify(submit.context), + ), + ).toBe(true); + await harness.close(context); + }); + + // #11043 + it("places inputs a failed run left queued first, and they join its run", async () => { + const setup = chatSetup(); + const gate = deferred(); + const reached = deferred(); + setup.faux.setResponses([ + async () => { + reached.resolve(); + await gate.promise; + return failure(); + }, + fauxAssistantMessage("both handled"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.submit({ type: "input", content: "first" }, context); + await reached.promise; + const queued = await root.submit({ type: "input", content: "second", whenBusy: "followUp" }, context); + gate.resolve(); + await harness.waitForIdle(context); + expect((await queued.status(context)).status).toBe("queued"); + + const resumed = await root.submit({ type: "continue" }, context); + const live = await harness.snapshot(LiveDoc, root.id, context); + expect(live?.run?.inputs).toEqual([queued.id, resumed.id]); + const [byQueued, byContinue] = [await queued.wait(context), await resumed.wait(context)]; + expect(await answerText(root, byContinue)).toBe("both handled"); + expect(byQueued).toMatchObject({ status: "done", answer: (byContinue as { answer: number }).answer }); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts new file mode 100644 index 000000000..07116cc08 --- /dev/null +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -0,0 +1,476 @@ +// Deferred progress commits (`PROGRESS_COMMIT`): the partial answer and tool output reach watches at once and Storage at +// the next commit that reads or writes `pi.live` or within `settings.progress.persistWithinMs`. Crashes are simulated +// by failing the Harness: from that moment nothing more reaches Storage, as when the process dies; the reopened Harness +// resumes from what was stored. + +import { applyImmutable } from "@earendil-works/chord/delta"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; +import { + AssistantEntry, + type ConversationId, + defineTask, + defineTool, + type EntryRecord, + type Harness, + LiveDoc, + type LiveState, + ToolResultEntry, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { describe, expect, it } from "vitest"; +import type { PartialMessage } from "../src/harness/partial-changes.ts"; +import type { SessionImpl } from "../src/session/session.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; +import { addTask, addTool } from "./harness-support.ts"; +import { ControlledStorage, context, followCommits } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +/** A slow stream: 4 characters 80 times a second. */ +function slowSetup(persistWithinMs: number): ChatSetup { + const setup = chatSetup({ tokensPerSecond: 80, tokenSize: { min: 4, max: 4 } }); + setup.settings.progress = { persistWithinMs }; + // A request a crash interrupted is a failed attempt (#11139): retry at once. + setup.settings.retry = { enabled: true, maxRetries: 5, baseDelayMs: 1 }; + return setup; +} + +const LONG = "streamed ".repeat(400); + +/** Open (or reopen) a Harness over `storage` and let it run tasks. */ +async function open(storage: ControlledStorage, setup: ChatSetup) { + const opened = await openChat(storage.reopen(), setup); + opened.harness.resume(); + return { ...opened, memory: memoryPartials(opened.harness) }; +} + +/** The partial text of every `pi.live` revision memory adopted, the last one last. */ +function memoryPartials(harness: Harness): string[] { + const texts: string[] = []; + followCommits(harness, (publication) => { + for (const change of publication.changes) { + if (change.type !== "document" || change.kind !== "change" || change.record.kind !== LiveDoc.definition.kind) + continue; + // Publications carry untyped document values; this one is `pi.live`. + const message = (change.value as LiveState)?.generation?.message; + if (message !== undefined) texts.push(partialText(message) ?? ""); + } + }); + return texts; +} + +/** Fail the Harness now: nothing more is stored, held changes are lost with it. */ +async function crash(harness: Harness): Promise { + // `fail()` is internal, so the public type lacks it; every Harness is a `SessionImpl`. + (harness as unknown as SessionImpl).fail(new Error("crash")); + await harness.closed; +} + +/** `pi.live` as Storage has it, read past the Session, also after a crash closed it. */ +async function storedLive(storage: ControlledStorage, conversationId: ConversationId): Promise { + storage.reopen(); + const address = { kind: LiveDoc.definition.kind, scope: { kind: "conversation", conversationId } } as const; + const record = await storage.findDocument(address, "current", context); + if (record === undefined) return undefined; + // Storage values are untyped; this one is `pi.live`. + return (await storage.document(record.id, "current", context))?.value as LiveState | undefined; +} + +/** The text of a partial message's first text block. */ +function partialText(message: PartialMessage | undefined): string | undefined { + const block = message?.content.find((candidate) => candidate.type === "text"); + return block?.type === "text" ? block.text : undefined; +} + +function storedText(live: LiveState | undefined): string | undefined { + const message = live?.generation?.message; + return message === undefined ? undefined : (partialText(message) ?? ""); +} + +/** Assistant entries: their text and stop reason. */ +function answers(entries: readonly EntryRecord[]): { text: string | undefined; stopReason: string | undefined }[] { + return entries + .filter((entry) => AssistantEntry.is(entry)) + .map((entry) => { + const message = entry.model![0]!; + return { text: textOf(message), stopReason: message.role === "assistant" ? message.stopReason : undefined }; + }); +} + +describe("generation partials", () => { + it("hold partials in memory and store them together, at most every persistWithinMs", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(400); + setup.faux.setResponses([fauxAssistantMessage(LONG.slice(0, 640))]); + const { harness, root, memory } = await open(storage, setup); + const from = storage.commits.length; + await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); + const writesLive = storage.commits + .slice(from) + .filter((batch) => batch.some((write) => write.type === "document.change")); + // About 2 s of streaming at 100 ms per partial: some 20 partials in memory, a store about every 400 ms. + expect(memory.length).toBeGreaterThan(12); + expect(writesLive.length).toBeLessThan(memory.length / 1.5); + await harness.close(context); + }); + + it("converts what was stored into the aborted entry after a crash, and retries the charged attempt", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + // A replica fed by observeCommits() alone, as a replication feed would be. + let replica = await storedLive(storage, opened.root.id); + opened.harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if ( + change.type === "document" && + change.kind === "change" && + change.record.kind === LiveDoc.definition.kind + ) { + replica = applyImmutable(replica, change.ops); + } + } + }, + end: () => {}, + }); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); + const inMemory = opened.memory.at(-1)!; + await crash(opened.harness); + const stored = storedText(await storedLive(storage, opened.root.id))!; + // What Storage has is a state memory had, behind it by at most the window, and what the replica has. + expect(opened.memory).toContain(stored); + expect(inMemory.startsWith(stored)).toBe(true); + expect(replica).toEqual(await storedLive(storage, opened.root.id)); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([ + { text: stored, stopReason: "aborted" }, + { text: "answer", stopReason: "stop" }, + ]); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + it("leaves no aborted entry for a stream that crashed before its first store", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await crash(opened.harness); + // The attempt is presentation, deferred like the partials: none of it was stored. + expect((await storedLive(storage, opened.root.id))?.generation).toBeUndefined(); + expect(storedText(await storedLive(storage, opened.root.id))).toBeUndefined(); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([{ text: "answer", stopReason: "stop" }]); + await opened.harness.close(context); + }); + + it("converts a stored partial once through two crashes, before and after the conversion", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); + await crash(opened.harness); + const stored = storedText(await storedLive(storage, opened.root.id))!; + + // Crash again before anything ran: nothing was converted, so nothing is lost or doubled. + opened = { ...(await openChat(storage.reopen(), setup)), memory: [] }; + await crash(opened.harness); + expect(storedText(await storedLive(storage, opened.root.id))).toBe(stored); + + // Crash again while the resend streams: its conversion is stored, its own partials only held. + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + opened = await open(storage, setup); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await crash(opened.harness); + expect(storedText(await storedLive(storage, opened.root.id))).toBeUndefined(); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([ + { text: stored, stopReason: "aborted" }, + { text: "answer", stopReason: "stop" }, + ]); + await opened.harness.close(context); + }); + + it("stores everything held at a clean close", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await opened.harness.close(context); + const atClose = opened.memory.at(-1)!; + expect(storedText(await storedLive(storage, opened.root.id))).toBe(atClose); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + await (await opened.harness.submission(submission.id, context))!.wait(context); + expect(answers(await allEntries(opened.root))[0]).toEqual({ text: atClose, stopReason: "aborted" }); + await opened.harness.close(context); + }); + + it("resumes polling a deferred response after a crash", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); + let now = 1_000; + setup.now = () => now; + setup.faux.setResponses([fauxAssistantMessage("deferred answer")]); + let opened = await open(storage, setup); + setup.settings.stream = { deferred: true }; + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await storedLive(storage, opened.root.id))?.generation?.deferred !== undefined); + await crash(opened.harness); + opened = await open(storage, setup); + now = 61_000; + opened.harness.resume(); + const settled = await (await opened.harness.submission(id, context))!.wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + const answer = await opened.root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context); + expect(textOf(answer?.model?.[0])).toBe("deferred answer"); + expect(setup.faux.state.deferredFetchCount).toBe(1); + await opened.harness.close(context); + }); +}); + +describe("tool output", () => { + /** + * A tool that writes a line every 20 ms until its invocation is cancelled on its first run. A later run waits for + * cancellation while `stall` is set (`stalled` resolves then), and otherwise writes one line and returns. + */ + function streamingTool(replay: "safe" | "unsafe") { + const lines: string[] = []; + const state = { runs: 0, lines, stall: false, stalled: deferred() }; + const registration = defineTool({ + name: "work", + description: "work", + parameters: obj({}), + replay, + execute: async (_args, api, callContext) => { + const run = ++state.runs; + const signal = callContext.abortSignal!; + if (run > 1 && state.stall) { + state.stalled.resolve(); + return aborted(signal); + } + if (run > 1) { + api.output(`rerun ${run}\n`); + return {}; + } + for (let line = 0; !signal.aborted; line++) { + api.output(`line ${line}\n`); + state.lines.push(`line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return aborted(signal); + }, + }); + return { registration, state }; + } + + async function storedOutput(storage: ControlledStorage, conversationId: ConversationId) { + return (await storedLive(storage, conversationId))?.tools?.[0]?.output; + } + + function results(entries: readonly EntryRecord[]): string[] { + return entries + .filter((entry) => ToolResultEntry.is(entry)) + .map((entry) => { + const message = entry.model![0]!; + if (message.role !== "toolResult") return ""; + return message.content.map((item) => (item.type === "text" ? item.text : "")).join("|"); + }); + } + + // #11139: a generation's request is charged at once after a crash, so its partials never stream with a count; a + // replay-safe tool reruns and streams its output with one. + it("never counts progress commits as progress of a task cut off again and again", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(50); + let runs = 0; + addTool( + setup.registry, + defineTool({ + name: "work", + description: "work", + parameters: obj({}), + replay: "safe", + execute: async (_args, api, callContext) => { + runs++; + for (let line = 0; !callContext.abortSignal!.aborted; line++) { + api.output(`run ${runs} line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return aborted(callContext.abortSignal!); + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + await opened.root.submit({ type: "input", content: "go" }, context); + for (let crashes = 1; crashes <= 2; crashes++) { + // This run stored output of its own, a progress commit, before it was cut off. + await waitFor( + async () => (await storedOutput(storage, opened.root.id))?.startsWith(`run ${crashes} `) === true, + ); + await crash(opened.harness); + opened = await open(storage, setup); + const tool = (await opened.harness.commit((tx) => tx.scanTasks({ kind: "pi.tool" }, 1), context)).items[0]!; + expect(tool.restarts).toBe(crashes); + } + await opened.harness.close(context); + }); + + it("answers a tool without replay that crashed with the output stored by then", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + const { registration, state } = streamingTool("unsafe"); + addTool(setup.registry, registration); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await waitFor(async () => ((await storedOutput(storage, opened.root.id))?.length ?? 0) > 20); + await crash(opened.harness); + const stored = (await storedOutput(storage, opened.root.id))!; + expect(state.lines.join("").startsWith(stored)).toBe(true); + + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(state.runs).toBe(1); + expect(results(await allEntries(opened.root))).toEqual([ + `${stored}|\n[error] Tool work was interrupted and may have partially run\n`, + ]); + await opened.harness.close(context); + }); + + it("reruns a replay-safe tool after a crash with its stored progress cleared, also through a second crash", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + const { registration, state } = streamingTool("safe"); + addTool(setup.registry, registration); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await waitFor(async () => ((await storedOutput(storage, opened.root.id))?.length ?? 0) > 20); + await crash(opened.harness); + + // The rerun's first commit stores the cleared progress; it crashes again before writing any of its own. + state.stall = true; + opened = await open(storage, setup); + await state.stalled.promise; + expect(await storedOutput(storage, opened.root.id)).toBeUndefined(); + await crash(opened.harness); + expect(await storedOutput(storage, opened.root.id)).toBeUndefined(); + + state.stall = false; + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(state.runs).toBe(3); + expect(results(await allEntries(opened.root))).toEqual(["rerun 3\n"]); + await opened.harness.close(context); + }); +}); + +describe("compaction", () => { + it("reruns a summarization cut off by a crash from its checkpoint", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 150, backgroundTokens: 0 }; + const summarizing = deferred(); + setup.faux.setResponses([ + ...["a1", "a2", "a3"].map((label) => fauxAssistantMessage(`${label} ${"word ".repeat(100)}`)), + async (_request, options) => { + summarizing.resolve(); + return aborted(options!.signal!); + }, + ]); + let opened = await open(storage, setup); + for (const label of ["u1", "u2", "u3"]) { + await (await opened.root.submit({ type: "input", content: `${label} ${"word ".repeat(100)}` }, context)).wait( + context, + ); + } + const id = await opened.root.compact(undefined, context); + await summarizing.promise; + await crash(opened.harness); + + setup.faux.setResponses([fauxAssistantMessage("SUMMARY")]); + opened = await open(storage, setup); + expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + await waitFor(async () => (await allEntries(opened.root)).some((entry) => entry.kind === "pi.compaction")); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); +}); + +describe("reading pi.live in a commit", () => { + it("stores the held partial a third-party task's commit read, with what it derived", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + const Audit = defineTask<{ conversationId: ConversationId }, { phase: "run" }, null>({ + name: "test.audit", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + const live = await tx.doc(LiveDoc, task.input.conversationId); + const text = partialText(live.generation?.message) ?? ""; + await tx.appendEntry(task.input.conversationId, { kind: "test.audit", data: text }); + return { status: "terminal", outcome: { status: "completed", result: null } }; + }, ctx); + }, + }, + abort: async () => {}, + }); + addTask(setup.registry, Audit); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + const conversationId = opened.root.id; + const audit = await opened.harness.commit( + (tx) => tx.createTask(Audit, { conversationId }, { ownership: { kind: "conversation" }, conversationId }), + context, + ); + await opened.harness.waitForTask(audit, context); + const derived = String((await allEntries(opened.root)).find((entry) => entry.kind === "test.audit")!.data); + expect(derived.length).toBeGreaterThan(40); + // The read stored the partial it saw, in the same storage transaction as the entry. + const batch = storage.commits.find((writes) => + writes.some((write) => write.type === "entry" && write.value.kind === "test.audit"), + )!; + expect(batch.some((write) => write.type === "document.change")).toBe(true); + await crash(opened.harness); + expect(storedText(await storedLive(storage, conversationId))).toBe(derived); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + await (await opened.harness.submission(submission.id, context))!.wait(context); + expect(answers(await allEntries(opened.root))[0]).toEqual({ text: derived, stopReason: "aborted" }); + await opened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts deleted file mode 100644 index 8f561a205..000000000 --- a/packages/durable/test/harness-events.test.ts +++ /dev/null @@ -1,779 +0,0 @@ -import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; -import { runInNewContext } from "node:vm"; -import type { Draft } from "@earendil-works/chord"; -import { - type AssistantMessage, - type FauxResponseStep, - fauxAssistantMessage, - fauxText, - fauxThinking, - fauxToolCall, - Type, -} from "@earendil-works/pi-ai"; -import { - AgentDoc, - type AgentEvent, - type AgentEventStream, - type Conversation, - defineTool, - type Harness, - LiveDoc, - type LiveState, - MemoryStorage, - type MessageChange, - type SnapshotEvent, - UsageDoc, - watchEvents, -} from "@earendil-works/pi-durable"; -import { describe, expect, it } from "vitest"; -import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; -import { addTool } from "./harness-support.ts"; -import { context, documentChanges } from "./session-support.ts"; -import { aborted, deferred } from "./task-support.ts"; - -/** Attach and start an event stream that records every delivered batch. */ -async function listen(harness: Harness, conversation: Conversation) { - const stream = await watchEvents(harness, conversation.id, context); - const batches: AgentEvent[][] = []; - stream.start(async (events) => { - batches.push([...events]); - }); - return { stream, batches, events: () => batches.flat() }; -} - -async function drained(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -function slow(): ChatSetup { - return chatSetup({ tokensPerSecond: 400, tokenSize: { min: 1, max: 1 } }); -} - -/** Rebuild the streamed text of block 0 from `message_start` and the text deltas that follow it. */ -function streamedText(events: readonly AgentEvent[]): string { - let text = ""; - for (const event of events) { - if (event.type === "message_start" && event.message.role === "assistant") { - const block = event.message.content[0]; - text = block?.type === "text" ? block.text : ""; - } - if (event.type !== "message_update") continue; - for (const change of event.changes) { - if (change.type === "text_start" && change.contentIndex === 0 && change.block.type === "text") { - text = change.block.text; - } - if (change.type === "text_delta" && change.contentIndex === 0) text += change.delta; - } - } - return text; -} - -type Mutable = { [key: string]: unknown }; - -/** Apply message changes to a copy of `message`, as an events-only consumer would. */ -function applyChanges(message: AssistantMessage, changes: readonly MessageChange[]): AssistantMessage { - let next = structuredClone(message); - for (const change of changes) { - if (change.type === "message") next = structuredClone(change.message); - else if (change.type === "block") next.content[change.contentIndex] = structuredClone(change.block); - else if (change.type.endsWith("_start") && "block" in change) { - next.content.splice(change.contentIndex, 0, structuredClone(change.block)); - } else if (change.type === "text_delta" || change.type === "thinking_delta") { - const block = next.content[change.contentIndex] as unknown as Mutable; - const field = change.type === "text_delta" ? "text" : "thinking"; - block[field] = `${block[field] as string}${change.delta}`; - } else if (change.type === "toolcall_delta") { - let target = (next.content[change.contentIndex] as unknown as Mutable).arguments as Mutable; - for (const segment of change.path.slice(0, -1)) target = target[segment] as Mutable; - const last = change.path.at(-1)!; - target[last] = `${target[last] as string}${change.delta}`; - } - } - return next; -} - -/** Committed partials of `conversation`'s generation, one per commit that has one. */ -function partialsOf(harness: Harness): AssistantMessage[] { - const partials: AssistantMessage[] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; - if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); - } - }); - return partials; -} - -describe("agent events", () => { - it("streams a run as lifecycle events and text deltas that rebuild the answer", async () => { - const setup = slow(); - const text = "streamed answer text ".repeat(20); - setup.faux.setResponses([fauxAssistantMessage([fauxText(text)])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - expect(stream.snapshot).toMatchObject({ type: "snapshot", entries: [], tools: [], inbox: [] }); - const partials: string[] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; - if (change.record.kind === "pi.live" && message !== undefined) partials.push(textOf(message) ?? ""); - } - }); - const submission = await root.submit({ type: "input", content: "hi" }, context); - await submission.wait(context); - await drained(); - const all = events(); - const types = all.map((event) => event.type).filter((type, index, list) => type !== list[index - 1]); - expect(types).toEqual([ - "message_start", - "message_end", - "submission", - "run_start", - "turn_start", - "message_start", - "message_update", - "message_end", - "turn_end", - "run_end", - "submission", - "usage_changed", - ]); - expect(all.find((event) => event.type === "run_start")).toEqual({ type: "run_start", inputs: [submission.id] }); - // After each event, the rebuilt text equals the committed partial of that commit. - const rebuilt = all.flatMap((event, index) => - (event.type === "message_start" && event.message.role === "assistant") || event.type === "message_update" - ? [streamedText(all.slice(0, index + 1))] - : [], - ); - expect(rebuilt).toEqual(partials); - expect(partials.length).toBeGreaterThan(1); - const end = all.filter((event) => event.type === "message_end").at(-1)!; - expect(end.type === "message_end" && textOf(end.entry!.model![0])).toBe(text); - await stream.stop(); - await harness.close(context); - }); - - it("reports tool start, output appends, and the result entry", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "print", - description: "Prints", - parameters: Type.Object({ n: Type.Number() }), - execute: async (_args, api) => { - api.output("one\n"); - await new Promise((resolve) => setTimeout(resolve, 150)); - api.output("two\n"); - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("print", { n: 1 }, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - // The output events rebuild the slot's retained window. - const output = (): string => { - let text = ""; - for (const event of events()) { - if (event.type !== "tool_execution_update" || event.output === undefined) continue; - if ("set" in event.output) text = event.output.set; - else text = text.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - } - return text; - }; - await waitFor(() => output() === "one\ntwo\n"); - gate.resolve(); - await submission.wait(context); - await drained(); - const tool = events().filter((event) => event.type.startsWith("tool_execution")); - expect(tool[0]).toEqual({ - type: "tool_execution_start", - toolCallId: "c1", - toolName: "print", - taskId: expect.any(Number), - args: { n: 1 }, - }); - const end = tool.at(-1)!; - expect(end).toMatchObject({ type: "tool_execution_end", toolCallId: "c1", entry: { kind: "pi.tool-result" } }); - // As in the coding agent, the tool ends directly before its result message. - const all = events(); - const endIndex = all.indexOf(end); - expect(all.slice(endIndex, endIndex + 3).map((event) => event.type)).toEqual([ - "tool_execution_end", - "message_start", - "message_end", - ]); - expect(all[endIndex + 1]).toMatchObject({ message: { role: "toolResult", toolCallId: "c1" } }); - // Two turns: the tool round, and the answer. - expect(events().filter((event) => event.type === "turn_start")).toHaveLength(2); - expect(events().filter((event) => event.type === "turn_end")).toHaveLength(2); - await stream.stop(); - await harness.close(context); - }); - - it("reports queued submissions, inbox changes, and retries", async () => { - const setup = chatSetup(); - const release = deferred(); - const held: FauxResponseStep = async (_request, options) => { - await Promise.race([release.promise, aborted(options!.signal!)]); - return fauxAssistantMessage([fauxText("first")]); - }; - const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); - setup.faux.setResponses([held, error, fauxAssistantMessage([fauxText("second")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; - const { stream, events } = await listen(harness, root); - await root.submit({ type: "input", content: "a" }, context); - const followUp = await root.submit({ type: "input", content: "f" }, context); - await drained(); - expect(events().filter((event) => event.type === "inbox_update")).toEqual([ - { type: "inbox_update", items: [{ id: followUp.id, mode: "followUp" }] }, - ]); - release.resolve(); - await followUp.wait(context); - await drained(); - const types = events().map((event) => event.type); - expect(types).toContain("auto_retry_start"); - expect(types).toContain("auto_retry_end"); - expect(types.filter((type) => type === "run_start")).toHaveLength(2); - expect( - events() - .filter((event) => event.type === "submission") - .map((event) => event.record.status), - ).toEqual(["placed", "queued", "done", "placed", "done"]); - await stream.stop(); - await harness.close(context); - }); - - it("delivers each of 100 pending batches in its own call, in order, unlike state watches", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream = await watchEvents(harness, root.id, context); - for (let index = 0; index < 100; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: { index } }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async (events) => { - batches.push([...events]); - }); - await drained(); - expect(batches).toHaveLength(100); - expect( - batches.map((events) => events.map((event) => (event.type === "entry_appended" ? event.entry.data : event))), - ).toEqual(Array.from({ length: 100 }, (_, index) => [{ index }])); - await stream.stop(); - await harness.close(context); - }); - - it("replaces undelivered batches with one snapshot after 100 pending batches", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream = await watchEvents(harness, root.id, context); - for (let index = 0; index < 101; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async (events) => { - batches.push([...events]); - }); - await drained(); - expect(batches).toHaveLength(1); - expect(batches[0]).toHaveLength(1); - expect(batches[0]![0]).toMatchObject({ type: "snapshot" }); - expect((batches[0]![0] as Extract).entries).toHaveLength(101); - await stream.stop(); - await harness.close(context); - }); - - it("rebuilds every committed partial of thinking, text, and tool-call arguments from message changes", async () => { - const setup = chatSetup({ tokensPerSecond: 150, tokenSize: { min: 1, max: 1 } }); - const message = fauxAssistantMessage( - [ - fauxThinking("thinking about it ".repeat(10)), - fauxText("some text ".repeat(10)), - fauxToolCall("missing", { path: "a/long/path/".repeat(10), note: "x".repeat(60) }, { id: "c1" }), - ], - { stopReason: "toolUse" }, - ); - setup.faux.setResponses([message, fauxAssistantMessage([fauxText("done")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const partials = partialsOf(harness); - const { stream, events } = await listen(harness, root); - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - await drained(); - const rebuilt: AssistantMessage[] = []; - let current: AssistantMessage | undefined; - // The streamed tool-calling message, up to its end; the short final answer commits no partial. - for (const event of events()) { - if (event.type === "message_end" && event.entry?.kind === "pi.assistant") break; - if (event.type === "message_start" && event.message.role === "assistant") current = event.message; - else if (event.type === "message_update") current = applyChanges(current!, event.changes); - else continue; - rebuilt.push(current!); - } - expect(partials.length).toBeGreaterThan(2); - expect(rebuilt.map((partial) => partial.content)).toEqual(partials.map((partial) => partial.content)); - const types = new Set( - events() - .flatMap((event) => (event.type === "message_update" ? event.changes : [])) - .map((c) => c.type), - ); - expect(types.has("thinking_delta") || types.has("text_delta")).toBe(true); - await stream.stop(); - await harness.close(context); - }); - - it("rebuilds a sliding tail window from output trims and appends", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "tail", - description: "Prints lines", - parameters: Type.Object({}), - outputLimits: { maxLines: 3, retain: "tail" }, - execute: async (_args, api) => { - for (let line = 0; line < 6; line++) { - api.output(`line ${line}\n`); - await new Promise((resolve) => setTimeout(resolve, 120)); - } - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("tail", {}, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const outputs: string[] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - const output = (change.value as LiveState | null)?.tools?.[0]?.output; - if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { - outputs.push(output); - } - } - }); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - await waitFor(() => outputs.at(-1)?.endsWith("line 5\n") === true); - gate.resolve(); - await submission.wait(context); - await drained(); - const rebuilt: string[] = []; - let text = ""; - for (const event of events()) { - if (event.type !== "tool_execution_update" || event.output === undefined) continue; - if ("set" in event.output) text = event.output.set; - else text = text.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - rebuilt.push(text); - } - expect(rebuilt).toEqual(outputs); - expect( - events().some((event) => event.type === "tool_execution_update" && "trimStart" in (event.output ?? {})), - ).toBe(true); - await stream.stop(); - await harness.close(context); - }); - - it("emits one exact batch when a run ends and a queued follow-up starts the next", async () => { - const setup = chatSetup(); - const release = deferred(); - const held: FauxResponseStep = async (_request, options) => { - await Promise.race([release.promise, aborted(options!.signal!)]); - return fauxAssistantMessage([fauxText("first")]); - }; - setup.faux.setResponses([held, fauxAssistantMessage([fauxText("second")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const input = await root.submit({ type: "input", content: "a" }, context); - const followUp = await root.submit({ type: "input", content: "f" }, context); - const { stream, batches } = await listen(harness, root); - release.resolve(); - await followUp.wait(context); - await drained(); - const boundary = batches.find((batch) => batch.some((event) => event.type === "run_end"))!; - expect(boundary.map((event) => event.type)).toEqual([ - "message_start", - "message_end", - "message_start", - "message_end", - "turn_end", - "run_end", - "submission", - "submission", - "inbox_update", - "usage_changed", - "run_start", - "turn_start", - ]); - expect(boundary.filter((event) => event.type === "submission").map((event) => event.record.id)).toEqual([ - input.id, - followUp.id, - ]); - await stream.stop(); - await harness.close(context); - }); - - it("ends a call that never runs and a tool aborted with its generation", async () => { - const setup = chatSetup(); - addTool( - setup.registry, - defineTool({ - name: "wait", - description: "Waits until aborted", - parameters: Type.Object({}), - execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("ghost", {}, { id: "c1" }), fauxToolCall("wait", {}, { id: "c2" })], { - stopReason: "toolUse", - }), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - await waitFor(() => events().some((event) => event.type === "tool_execution_start")); - // Aborting the generation aborts its round: the tool ends with its aborted result first (spec §8.5). - await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); - await submission.wait(context); - await drained(); - // The call not offered ends after its calling message and directly before its result message. - const round = events().map((event) => - event.type === "message_end" - ? `end:${event.entry.kind}` - : event.type === "message_start" - ? `start:${event.message.role}` - : event.type, - ); - const ghostEnd = round.indexOf("tool_execution_end"); - expect(round.slice(ghostEnd - 1, ghostEnd + 3)).toEqual([ - "end:pi.assistant", - "tool_execution_end", - "start:toolResult", - "end:pi.tool-result", - ]); - const tool = events().filter((event) => event.type.startsWith("tool_execution")); - expect(tool.map((event) => [event.type, "toolCallId" in event && event.toolCallId, "entry" in event])).toEqual([ - ["tool_execution_end", "c1", true], - ["tool_execution_start", "c2", false], - ["tool_execution_end", "c2", true], - ]); - await stream.stop(); - await harness.close(context); - }); - - it("reports a steer without run events, a reset as an appended entry, and nothing for other conversations", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "hold", - description: "Waits", - parameters: Type.Object({}), - execute: async () => { - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); - const { stream, events, batches } = await listen(harness, root); - const input = await root.submit({ type: "input", content: "a" }, context); - await waitFor(() => events().some((event) => event.type === "tool_execution_start")); - await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); - const before = batches.length; - await other.commit((tx) => tx.appendEntry(other.id, { kind: "note" }), context); - await drained(); - expect(batches.length).toBe(before); - gate.resolve(); - await input.wait(context); - await root.reset(undefined, context); - await drained(); - expect(events().filter((event) => event.type === "run_start")).toHaveLength(1); - expect(events().filter((event) => event.type === "run_end")).toHaveLength(1); - expect(batches.at(-1)!.map((event) => event.type)).toEqual(["entry_appended", "submission"]); - await stream.stop(); - await harness.close(context); - }); - - it("rejects an attachment cancelled while it waits for the Session line", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const release = deferred(); - const blocking = root.commit(async () => { - await release.promise; - }, context); - const controller = new AbortController(); - const attaching = watchEvents(harness, root.id, { ...context, abortSignal: controller.signal }); - controller.abort(new Error("cancelled")); - release.resolve(); - await blocking; - await expect(attaching).rejects.toThrow("cancelled"); - await harness.close(context); - }); - - it("applies deltas after an overflow snapshot that holds an in-flight partial", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const partial = fauxAssistantMessage([fauxText("hel")]); - const stream = await watchEvents(harness, root.id, context); - await root.commit(async (tx) => { - (await tx.doc(LiveDoc, root.id)).generation = { - attempt: 1, - message: JSON.parse(JSON.stringify(partial)), - }; - }, context); - for (let index = 0; index < 101; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async (events) => { - batches.push([...events]); - }); - await drained(); - const snapshot = batches[0]![0] as SnapshotEvent; - expect(snapshot.type).toBe("snapshot"); - await root.commit(async (tx) => { - const block = (await tx.doc(LiveDoc, root.id)).generation!.message!.content[0] as { text: string }; - block.text += "lo"; - }, context); - await drained(); - const update = batches.at(-1)![0]!; - if (update.type !== "message_update") throw new Error(`Unexpected ${update.type}`); - expect(update.changes).toEqual([{ type: "text_delta", contentIndex: 0, delta: "lo" }]); - const rebuilt = applyChanges(snapshot.generation!.message!, update.changes); - expect(rebuilt.content).toEqual([{ type: "text", text: "hello" }]); - await stream.stop(); - await harness.close(context); - }); - - it("reports usage-only updates, cleared tool progress, tools ending without entries, and deferred polls", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const { stream, batches } = await listen(harness, root); - const change = (edit: (live: Draft) => void) => - root.commit(async (tx) => edit(await tx.doc(LiveDoc, root.id)), context); - const partial = JSON.parse(JSON.stringify(fauxAssistantMessage([fauxText("partial")]))); - await change((live) => { - live.generation = { attempt: 1, message: partial }; - }); - // A usage-only change sends the usage and no changes. - await change((live) => { - live.generation!.message!.usage.input = 42; - }); - await change((live) => { - live.generation = { attempt: 1, deferred: { pollAt: 1 } }; - }); - await change((live) => { - live.generation!.deferred!.pollAt = 2; - }); - await change((live) => { - live.tools = [{ callId: "c1", name: "t", status: "running", details: { n: 1 }, diagnostics: [] }]; - }); - // A safe replay clears the running slot's progress. - await change((live) => { - delete live.tools![0]!.details; - delete live.tools![0]!.diagnostics; - }); - // A fault marks the slot done without an entry. - await change((live) => { - live.tools![0]!.status = "done"; - }); - await root.commit((tx) => tx.retireDoc(UsageDoc, root.id), context); - await root.commit((tx) => tx.retireDoc(AgentDoc, root.id), context); - await drained(); - expect(batches.map((batch) => batch.filter((event) => event.type !== "task_failed"))).toEqual([ - [{ type: "message_start", message: partial }], - [{ type: "message_update", usage: { ...partial.usage, input: 42 }, changes: [] }], - [{ type: "deferred_poll", pollAt: 1 }], - [{ type: "deferred_poll", pollAt: 2 }], - [{ type: "tool_execution_start", toolCallId: "c1", toolName: "t", args: {} }], - [{ type: "tool_execution_update", toolCallId: "c1", toolName: "t", details: null, diagnostics: [] }], - [{ type: "tool_execution_end", toolCallId: "c1", toolName: "t" }], - // Retired documents read as their initial values. - [{ type: "usage_changed", usage: { models: {}, tools: {} } }], - [{ type: "agent_changed", agent: {} }], - ]); - await stream.stop(); - await harness.close(context); - }); - - it("ends the stream with the Harness", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream: AgentEventStream = await watchEvents(harness, root.id, context); - await harness.close(context); - expect(await stream.closed).toEqual({ reason: "ended" }); - }); - - it("starts a message at the first committed partial and ends it with the converted entry on abort", async () => { - const setup = chatSetup({ tokensPerSecond: 100, tokenSize: { min: 1, max: 1 } }); - setup.faux.setResponses([fauxAssistantMessage([fauxText("x".repeat(400))])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "hi" }, context); - await waitFor(() => - events().some((event) => event.type === "message_start" && event.message.role === "assistant"), - ); - const live = await harness.snapshot(LiveDoc, root.id, context); - await harness.abortTask(live!.run!.taskId, context); - await submission.wait(context); - await drained(); - const assistantEnds = events().filter( - (event) => event.type === "message_end" && event.entry?.kind === "pi.assistant", - ); - expect(assistantEnds).toHaveLength(1); - expect( - events().filter((event) => event.type === "message_start" && event.message.role === "assistant"), - ).toHaveLength(1); - await stream.stop(); - await harness.close(context); - }); -}); - -const gc = (() => { - setFlagsFromString("--expose-gc"); - return runInNewContext("gc") as () => void; -})(); - -/** Bytes `run` allocates, counting what collections during it freed. */ -async function allocatedBytes(run: () => Promise): Promise { - gc(); - const profiler = new GCProfiler(); - profiler.start(); - const before = getHeapStatistics().used_heap_size; - await run(); - const after = getHeapStatistics().used_heap_size; - let freed = 0; - for (const collection of profiler.stop().statistics) { - freed += collection.beforeGC.heapStatistics.usedHeapSize - collection.afterGC.heapStatistics.usedHeapSize; - } - return after - before + freed; -} - -/** Heap bytes in use after a full collection. */ -async function heapInUse(): Promise { - for (let round = 0; round < 3; round++) { - await drained(); - gc(); - } - return getHeapStatistics().used_heap_size; -} - -// #9265 -describe("streamed partials", () => { - it("streams a tool-call partial as toolcall_delta appends at its argument paths", async () => { - const setup = chatSetup({ tokensPerSecond: 500, tokenSize: { min: 1, max: 1 } }); - setup.settings.progress = { partialIntervalMs: 20, outputIntervalMs: 100 }; - const args = { path: "src/file.ts", content: "const a = 1;\n".repeat(150) }; - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("missing", args, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const partials = partialsOf(harness); - const { stream, events } = await listen(harness, root); - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - await drained(); - const changes = events().flatMap((event) => (event.type === "message_update" ? event.changes : [])); - const deltas = changes.filter((change) => change.type === "toolcall_delta"); - expect(deltas.length).toBeGreaterThan(10); - expect(deltas.every((change) => ["path", "content"].includes(String(change.path)))).toBe(true); - // The call is resent whole only when a member appears (at most once per argument); growing strings are appends. - // With the raw argument text on the block, every commit resent the whole call. - expect(changes.filter((change) => change.type === "message")).toEqual([]); - expect(changes.filter((change) => change.type === "block").length).toBeLessThanOrEqual(2); - // The committed partials carry only the parsed arguments, never the raw argument text. - for (const partial of partials) { - for (const block of partial.content) { - expect(Object.keys(block).sort()).toEqual(["arguments", "id", "name", "type"]); - } - } - await stream.stop(); - await harness.close(context); - }); - - it("commits a growing text and tool call at a cost that follows the new text, not the string lengths", async () => { - /** Bytes allocated while streaming one answer of `size` characters of text and as much tool-call content. */ - const stream = async (size: number) => { - const setup = chatSetup({ tokensPerSecond: 64_000, tokenSize: { min: 64, max: 64 } }); - setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; - // Keep `pi.live` cached between commits, as the default retention does. Evicted after each use (the suite's - // evicting run), every commit replays the document's delta chain: a cost of eviction, not of appending. - setup.settings.documentRetentionMs = 60_000; - const text = 'streamed text with "quotes" and \\ escapes\n'.repeat(size / 40); - setup.faux.setResponses([ - fauxAssistantMessage( - [fauxText(text), fauxToolCall("missing", { path: "a.ts", content: text }, { id: "c1" })], - { - stopReason: "toolUse", - }, - ), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - let commits = 0; - harness.subscribeCommits(() => commits++); - const bytes = await allocatedBytes(async () => { - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - }); - await harness.close(context); - return { bytes, commits }; - }; - const small = await stream(40_000); - const large = await stream(400_000); - expect(large.commits).toBeGreaterThan(small.commits * 5); - // Per commit, the cost stays flat: comparing the whole strings on every commit made it grow with their length. - expect(large.bytes / large.commits / (small.bytes / small.commits)).toBeLessThan(2); - }, 60_000); - - it("keeps a watcher's memory proportional to a long streamed text and tool call, not to their commits", async () => { - const setup = chatSetup({ tokensPerSecond: 16_000, tokenSize: { min: 16, max: 16 } }); - setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; - const line = (index: number) => `line ${index}: some streamed source text with "quotes" and \\ escapes\n`; - const text = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); - const content = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); - setup.faux.setResponses([ - fauxAssistantMessage([fauxText(text), fauxToolCall("missing", { path: "a.ts", content }, { id: "c1" })], { - stopReason: "toolUse", - }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, batches } = await listen(harness, root); - const before = await heapInUse(); - const submission = await root.submit({ type: "input", content: "go" }, context); - const partialContent = async () => - (await harness.snapshot(LiveDoc, root.id, context))?.generation?.message?.content; - const shown = async (index: number): Promise => { - const block = (await partialContent())?.[index]; - if (block?.type === "text") return block.text.length; - return block?.type === "toolCall" ? String(block.arguments.content ?? "").length : 0; - }; - const retained: number[] = []; - // Measure while the text, then the tool call, is almost complete: the watcher's view holds the most then. - await waitFor(async () => (await shown(0)) > text.length * 0.95, 30_000); - retained.push((await heapInUse()) - before); - await waitFor(async () => (await shown(1)) > content.length * 0.95, 30_000); - retained.push((await heapInUse()) - before); - await submission.wait(context); - const commits = batches.length; - expect(commits).toBeGreaterThan(500); - // The two 140 KB strings, the watcher's recorded batches and the rest of the stream: about 3 MB. Sliced append ops - // made the watcher's view hold a copy of the text per commit: well over 100 MB. - for (const bytes of retained) expect(bytes).toBeLessThan(20_000_000); - await stream.stop(); - await harness.close(context); - }, 60_000); -}); diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index ef104ac4c..d88c34ea7 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -8,7 +8,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection } from "./harness-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; const directories = new Set(); @@ -130,9 +130,11 @@ describe("generation recovery", () => { opened.harness.resume(); const watch = (await opened.harness.watchDoc(LiveDoc, opened.root.id, context))!; let watched: string | undefined; - watch.start(async (value) => { - watched = textOf(value?.generation?.message as Message) ?? watched; - }); + watch.start( + onValue(async ({ value }) => { + watched = textOf(value.generation?.message as Message) ?? watched; + }), + ); const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; await waitFor(() => watched !== undefined); await opened.harness.close(context); @@ -195,7 +197,7 @@ describe("generation recovery", () => { await new Promise((resolve) => setTimeout(resolve, 20)); expect(setup.faux.state.callCount).toBe(1); expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({ - run: { taskId, inputs: [id] }, + run: { taskId, inputs: [id], turns: 0, first: expect.any(Number) }, generation: { attempt: 1, retry: { at: 61_000, error: "503 Service Unavailable" } }, }); now = 61_000; @@ -287,6 +289,43 @@ describe("generation recovery", () => { await opened.harness.close(context); }); + it("polls a checkpoint stored without a thinking level and records none", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); + let now = 1_000; + setup.now = () => now; + setup.faux.setResponses([fauxAssistantMessage("deferred answer")]); + let opened = await open(path, setup); + await opened.root.configure({ thinkingLevel: "high" }, context); + opened.harness.resume(); + setup.settings.stream = { deferred: true }; + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => { + const live = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + return live?.generation?.deferred !== undefined; + }); + const taskId = await runTaskId(opened.harness); + expect(await checkpoint(opened.harness, taskId)).toMatchObject({ phase: "poll", thinkingLevel: "high" }); + await opened.harness.close(context); + // Earlier versions stored poll checkpoints without the field. + const database = new DatabaseSync(path); + database + .prepare("UPDATE tasks SET record = json_remove(record, '$.state.checkpoint.thinkingLevel') WHERE id = ?") + .run(taskId); + database.close(); + + opened = await open(path, setup); + expect(await checkpoint(opened.harness, taskId)).not.toHaveProperty("thinkingLevel"); + now = 61_000; + opened.harness.resume(); + const settled = await (await opened.harness.submission(id, context))!.wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + const answer = (await opened.root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context))?.model?.[0]; + expect(textOf(answer)).toBe("deferred answer"); + expect(answer).not.toHaveProperty("thinkingLevel"); + await opened.harness.close(context); + }); + it("fails no_model when the pinned model is gone after reopen, in request and in poll", async () => { const path = await sqlitePath(); const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 50bae0a58..7a46a4a9f 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -9,7 +9,6 @@ import { } from "@earendil-works/pi-ai"; import { AssistantEntry, - type CommitPublication, type Conversation, createRegistry, defineDoc, @@ -31,7 +30,7 @@ import { resolveSettings } from "../src/harness/agent.ts"; import type { SessionImpl } from "../src/session/session.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, followCommits } from "./session-support.ts"; const ERROR_503 = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); @@ -76,9 +75,9 @@ function invalidFinalStream(): ReturnType { function livePublications(harness: Harness): LiveState[] { const values: LiveState[] = []; - harness.subscribeCommits((publication: CommitPublication) => { + followCommits(harness, (publication) => { for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === "pi.live" && change.value !== null) { + if (change.type === "document" && change.record.kind === "pi.live" && change.kind === "change") { values.push(change.value as LiveState); } } @@ -120,6 +119,8 @@ describe("generation", () => { it("stores partials as deltas and a complete base once nothing is in flight", async () => { const setup = chatSetup({ tokensPerSecond: 200, tokenSize: { min: 1, max: 1 } }); setup.faux.setResponses([fauxAssistantMessage("w".repeat(200))]); + // Store deferred partials well within the stream, so some reach Storage before the answer. + setup.settings.progress = { persistWithinMs: 100 }; const storage = new ControlledStorage(); const { harness, root } = await openChat(storage, setup); const record = await storage.findDocument( @@ -156,7 +157,10 @@ describe("generation", () => { expect((await harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "aborted" }); // The earlier settlement stays; the run's own settlement leaves it unchanged. expect(await submission.status(context)).toMatchObject({ status: "unanswered", reason: "withdrawn" }); - expect(await live(harness, root)).toEqual({}); + // A run ends without an answer even when its inputs were settled already. + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -180,8 +184,19 @@ describe("generation", () => { error: { message: "Model faux/missing is not available", detail: { reason: "no_model" } }, }, }); - expect(await live(harness, root)).toEqual({}); - expect(await live(harness, plain)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { + taskId: tasks.items[0]!.id, + inputs: [expect.any(Number)], + reason: "no_model", + detail: "Model faux/missing is not available", + first: expect.any(Number), + }, + }); + expect((await live(harness, plain))?.runFailure).toMatchObject({ + reason: "no_model", + detail: "No model is configured", + }); await harness.close(context); }); @@ -305,7 +320,9 @@ describe("generation", () => { const converted = entries[1]!.model![0] as AssistantMessage; expect(converted.stopReason).toBe("aborted"); expect(textOf(converted)!.startsWith(textOf(partial as Message)!)).toBe(true); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); expect((await harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "aborted" }); await harness.close(context); }); @@ -322,7 +339,9 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.faux.state.cancelledDeferred).toHaveLength(1); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -346,7 +365,9 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.reports).toContainEqual(new Error("cancel failed")); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -386,6 +407,24 @@ describe("generation", () => { await harness.close(context); }); + it("records the requested thinking level on answers, streamed and polled", async () => { + const setup = chatSetup({ deferred: { pendingFetches: 1, pollAfterMs: 1 } }); + setup.faux.setResponses([fauxAssistantMessage("streamed"), fauxAssistantMessage("polled")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + harness.resume(); + const answer = async (content: string) => { + const settled = await (await root.submit({ type: "input", content }, context)).wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + return (await root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context))?.model?.[0]; + }; + await root.configure({ thinkingLevel: "high" }, context); + expect(await answer("one")).toMatchObject({ thinkingLevel: "high" }); + setup.settings.stream = { deferred: true }; + expect(await answer("two")).toMatchObject({ thinkingLevel: "high" }); + expect(setup.faux.state.deferredFetchCount).toBe(2); + await harness.close(context); + }); + it("attributes model requests to pi by default and merges the Harness identity over it", async () => { const seen: (SimpleStreamOptions["identity"] | undefined)[] = []; const capture = (_context: unknown, options?: SimpleStreamOptions) => { @@ -509,7 +548,7 @@ describe("generation", () => { stream: {}, retry: { enabled: true, maxRetries: 3, baseDelayMs: 2000, maxAgentDelayMs: 60000 }, compaction: { enabled: true, reserveTokens: 16384, keepRecentTokens: 20000, backgroundTokens: 32768 }, - progress: { partialIntervalMs: 100, outputIntervalMs: 100 }, + progress: { partialIntervalMs: 100, outputIntervalMs: 100, persistWithinMs: 1000 }, toolExecution: "parallel", steeringMode: "one-at-a-time", followUpMode: "one-at-a-time", @@ -527,7 +566,11 @@ describe("generation", () => { expect(resolveSettings({ progress: { outputIntervalMs: 500 } }).progress).toEqual({ partialIntervalMs: 100, outputIntervalMs: 500, + persistWithinMs: 1000, }); + // Not a number of milliseconds, 0 or more: the default. + expect(resolveSettings({ progress: { persistWithinMs: -1 } }).progress.persistWithinMs).toBe(1000); + expect(resolveSettings({ progress: { persistWithinMs: 0 } }).progress.persistWithinMs).toBe(0); }); it("commits partials no more often than progress.partialIntervalMs", async () => { @@ -653,7 +696,15 @@ describe("generation", () => { expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); expect(entries[1]!.model![0]).toMatchObject({ role: "assistant", stopReason: "aborted" }); expect(textOf(entries[1]!.model![0])).toBe("partial"); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [submission.id], + reason: "faulted", + detail: expect.stringContaining("non-JSON function"), + first: expect.any(Number), + }, + }); const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 10), context); expect(tasks.items[0]!.state).toMatchObject({ status: "terminal", outcome: { status: "faulted" } }); await harness.close(context); @@ -674,11 +725,11 @@ describe("generation", () => { entry: entry.id, }); const taskId = await tx.createTask( - { definition: { ...GenerationTask.definition, version: 2 } }, + { definition: { ...GenerationTask.definition, version: GenerationTask.definition.version + 1 } }, {}, { ownership: { kind: "conversation" }, conversationId: root.id }, ); - (await tx.doc(LiveDoc, root.id)).run = { taskId, inputs: [submission.id] }; + (await tx.doc(LiveDoc, root.id)).run = { taskId, inputs: [submission.id], turns: 0 }; return { taskId, submissionId: submission.id }; }, context); harness.resume(); @@ -694,7 +745,9 @@ describe("generation", () => { status: "unanswered", reason: "task_too_old", }); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submissionId], reason: "orphaned", detail: "task_too_old" }, + }); await harness.close(context); }); diff --git a/packages/durable/test/harness-group-commit.test.ts b/packages/durable/test/harness-group-commit.test.ts new file mode 100644 index 000000000..46563015c --- /dev/null +++ b/packages/durable/test/harness-group-commit.test.ts @@ -0,0 +1,796 @@ +// Group commit and the scheduler (docs/group-commit.md §6): decisions read the scheduler's memory in regions that +// wait for the unpublished commits whose keys they read, memberships written at admission, outcomes, and close. +import type { JsonValue } from "@earendil-works/chord"; +import { + type ConversationId, + createRegistry, + defineExtension, + defineTask, + type Harness, + type TaskId, + type TaskRecord, + type TaskRuntime, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { SCHEDULER_INDEX_SIZES } from "../src/harness/harness.ts"; +import { ownedKey, ROOTS_KEY } from "../src/session/keys.ts"; +import type { SessionImpl } from "../src/session/session.ts"; +import type { Transaction } from "../src/session/transaction.ts"; +import { ControlledStorage, context, flush, followCommits } from "./session-support.ts"; +import { aborted, type Deferred, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +const gates = new Map>(); +function gate(name: string): Deferred { + let found = gates.get(name); + if (found === undefined) { + found = deferred(); + gates.set(name, found); + } + return found; +} + +/** How often each named phase ran. */ +const runs = new Map(); +function ran(name: string): number { + return runs.get(name) ?? 0; +} +function count(name: string): void { + runs.set(name, ran(name) + 1); +} + +afterEach(() => { + gates.clear(); + runs.clear(); +}); + +const COMPLETED = { status: "terminal", outcome: { status: "completed", result: null } } as const; + +function completedWith(result: R) { + return { status: "terminal", outcome: { status: "completed", result } } as const; +} + +/** Holds until its named gate opens, then completes; its abort handler ends it `aborted`. */ +const Hold = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.hold", + version: 1, + initial: () => ({ phase: "hold" }), + phases: { + hold: async (task, runtime, ctx) => { + count(task.input.name); + await Promise.race([gate(task.input.name).promise, aborted(runtime.signal)]); + await runtime.commit(() => COMPLETED, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +/** Commits its next phase, returns once `.return` opens; the next phase counts its run and completes. */ +const TwoPhases = defineTask<{ name: string }, { phase: "one" } | { phase: "two" }, null>({ + name: "gc.two-phases", + version: 1, + initial: () => ({ phase: "one" }), + phases: { + one: async (task, runtime, ctx) => { + await runtime.commit(() => ({ status: "running", checkpoint: { phase: "two" } }), ctx); + await gate(`${task.input.name}.return`).promise; + }, + two: async (task, runtime, ctx) => { + count(`${task.input.name}.two`); + await runtime.commit(() => COMPLETED, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +/** Once its gate opens, waits on the task in its input; records why a refused wait was refused, then completes. */ +const Waits = defineTask<{ name: string; on?: TaskId }, { phase: "wait" } | { phase: "done" }, null>({ + name: "gc.waits", + version: 1, + initial: () => ({ phase: "wait" }), + phases: { + wait: async (task, runtime, ctx) => { + await gate(task.input.name).promise; + const on = await runtime.memo("on", ctx); + try { + await runtime.commit( + () => ({ status: "waiting", checkpoint: { phase: "done" }, on: [on as TaskId], policy: "allSettled" }), + ctx, + ); + } catch (error) { + refusals.set(task.input.name, error); + await runtime.commit(() => COMPLETED, ctx); + } + }, + done: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx), + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); +const refusals = new Map(); + +async function status(harness: Harness, id: TaskId): Promise { + return (await harness.getTask(id, context))?.state.status; +} + +/** Stage a task record as is, the way only the scheduler and recovery write them. */ +function setTask(tx: unknown, record: TaskRecord): void { + (tx as Transaction).setTask(record); +} + +describe("group commit: scheduler regions", () => { + it("holds a live owner whose end races a creation below its ended tasks: live P, ended B and A, empty C", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const a = await tx.createTask(Hold, { name: "a" }, { ownership: { kind: "task", taskId: b } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: a } }); + return { p, b, a, c: c.id }; + }, context); + gate("a").resolve(); + await harness.waitForTask(tree.a, context); + gate("b").resolve(); + await harness.waitForTask(tree.b, context); + // The creation of D in C is admitted and in flight when P commits its end. + const held = storage.holdCommits(); + const c = (await harness.conversation(tree.c, context))!; + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await held.entered; + // P's decision reads what the creation writes, so it waits for its publication, holding the line. + gate("p").resolve(); + await flush(); + held.release(); + const d = await creating; + await eventually(async () => (await status(harness, tree.p)) === "completing"); + gate("d").resolve(); + await harness.waitForTask(d, context); + expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("holds an owner whose end follows a creation below its child's pending end: live P, ending B, conversation C", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: b } }); + return { p, b, c: c.id }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + await eventually(async () => (await status(harness, tree.b)) === "running"); + // A call in flight, so what follows is admitted and stays unpublished, in this order. + const held = storage.holdCommits(); + const blocker = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + // B ends; then D is created in C while B still shows live; then P ends. + gate("b").resolve(); + await flush(); + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await flush(); + gate("p").resolve(); + await flush(); + held.release(); + await blocker; + const d = await creating; + // In line order D's creation precedes P's decision: P holds while D is live. + await eventually(async () => (await status(harness, tree.p)) === "completing"); + expect(await status(harness, d)).toBe("running"); + gate("d").resolve(); + expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("ends a child before its next phase when its owner's abort mark is unpublished at its step", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold, TwoPhases]); + const root = await harness.root(context); + harness.resume(); + const { owner, child } = await root.commit(async (tx) => { + const owner = await tx.createTask(Hold, { name: "o" }, { ownership: { kind: "conversation" } }); + const child = await tx.createTask(TwoPhases, { name: "t" }, { ownership: { kind: "task", taskId: owner } }); + return { owner, child }; + }, context); + await eventually(async () => { + const record = await harness.getTask(child, context); + return record?.state.status === "running" && (record.state.checkpoint as { phase: string }).phase === "two"; + }); + const held = storage.holdCommits(); + const marking = harness.abortTask(owner, context); + await held.entered; + // The child's phase returns: its step reads the intent set, which the unpublished mark changes. + gate("t.return").resolve(); + await flush(); + expect(ran("t.two")).toBe(0); + held.release(); + expect(await marking).toBe("marked"); + expect((await harness.waitForTask(child, context)).state.outcome.status).toBe("aborted"); + expect((await harness.waitForTask(owner, context)).state.outcome.status).toBe("aborted"); + expect(ran("t.two")).toBe(0); + await harness.close(context); + }); + + it("refuses the second of two opposite waits admitted behind each other", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Waits]); + const root = await harness.root(context); + const { x, y } = await root.commit(async (tx) => { + const x = await tx.createTask(Waits, { name: "x" }, { ownership: { kind: "conversation" } }); + const y = await tx.createTask(Waits, { name: "y" }, { ownership: { kind: "conversation" } }); + return { x, y }; + }, context); + // Each waits on the other, named in a memo. + await root.commit(async (tx) => { + // Every table read before the first write. + const records = [(await tx.task(x))!, (await tx.task(y))!]; + setTask(tx, { ...records[0]!, memos: { on: y } } as TaskRecord); + setTask(tx, { ...records[1]!, memos: { on: x } } as TaskRecord); + }, context); + harness.resume(); + await eventually( + async () => (await status(harness, x)) === "running" && (await status(harness, y)) === "running", + ); + const held = storage.holdCommits(); + gate("x").resolve(); + await held.entered; + gate("y").resolve(); + await flush(); + held.release(); + await harness.waitForTask(y, context); + expect(String(refusals.get("y"))).toContain("wait cycle"); + expect(refusals.has("x")).toBe(false); + expect((await harness.waitForTask(x, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("starts no invocation on a reservation before the reservation is published", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Hold, { name: "r" }, { ownership: { kind: "conversation" } }), + context, + ); + // The reservation's write is in flight: a scheduler commit with writes is never delivered before it is published. + const held = storage.holdCommits(); + harness.resume(); + await held.entered; + await flush(); + expect(ran("r")).toBe(0); + held.release(); + await eventually(() => ran("r") === 1); + expect(await status(harness, id)).toBe("running"); + gate("r").resolve(); + await harness.waitForTask(id, context); + await harness.close(context); + }); + + it("runs a migration once although its reservation waits for an unpublished write of the task", async () => { + const storage = new ControlledStorage(); + let migrations = 0; + const V2 = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.migrated", + version: 2, + initial: () => ({ phase: "hold" }), + migrate: (input, checkpoint) => { + migrations++; + return { input: input as { name: string }, checkpoint: checkpoint as { phase: "hold" } }; + }, + phases: { hold: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx) }, + abort: async () => {}, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "v2", tasks: [V2] })); + const { harness } = await openTasks(storage, [], { registry }); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(V2, { name: "m" }, { ownership: { kind: "conversation" } }), + context, + ); + // Stored at version 1, and written again by a commit that is in flight when the reservation reads it. + const held = storage.holdCommits(); + const writing = root.commit(async (tx) => { + const record = (await tx.task(id))!; + setTask(tx, { ...record, version: 1, memos: { touched: true } } as TaskRecord< + JsonValue, + JsonValue, + JsonValue + >); + }, context); + await held.entered; + harness.resume(); + await flush(); + expect(migrations).toBe(0); + held.release(); + await writing; + expect((await harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + expect(migrations).toBe(1); + await harness.close(context); + }); + + it("reports a failed migration once although its reservation waits for an unpublished write of the task", async () => { + const storage = new ControlledStorage(); + let migrations = 0; + const V2 = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.unmigratable", + version: 2, + initial: () => ({ phase: "hold" }), + migrate: () => { + migrations++; + throw new Error("cannot migrate"); + }, + phases: { hold: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx) }, + abort: async () => {}, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "v2", tasks: [V2] })); + const { harness, infos } = await openTasks(storage, [], { registry }); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(V2, { name: "m" }, { ownership: { kind: "conversation" } }), + context, + ); + const held = storage.holdCommits(); + const writing = root.commit(async (tx) => { + const record = (await tx.task(id))!; + setTask(tx, { ...record, version: 1 } as TaskRecord); + }, context); + await held.entered; + harness.resume(); + await flush(); + held.release(); + await writing; + await eventually(() => migrations > 0); + await flush(); + expect(migrations).toBe(1); + expect(infos.filter((info) => info.kind === "migration_failed")).toHaveLength(1); + await harness.close(context); + }); + + it("makes a reader of the roots wait for a creation under a root a pending end would unlist", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Root R is listed for its last live task, T. + const last = await root.commit( + (tx) => tx.createTask(Hold, { name: "t" }, { ownership: { kind: "conversation" } }), + context, + ); + await eventually(async () => (await status(harness, last)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // T's end is in flight; a creation under R is admitted after it, while R still shows listed. + const held = storage.holdCommits(); + gate("t").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = root.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + // A reader of the roots on the line (global idle's key) waits until the creation is published too. + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ROOTS_KEY); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); + + it("walks a creation's memberships past an ancestor whose pending end is unpublished", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Live P, in the root conversation, owns the empty conversation C. + const { p, c } = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + return { p, c: c.id }; + }, context); + await eventually(async () => (await status(harness, p)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // P's end is in flight; a creation in C is admitted after it. Its chain must not stop at P, which shows live. + const conversation = (await harness.conversation(c, context))!; + const held = storage.holdCommits(); + gate("p").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = conversation.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ROOTS_KEY); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); + + it("reserves the live work of the owner above an ancestor whose end is pending, for a creation below it", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Live G owns P, which owns the empty conversation C. + const { g, p, c } = await root.commit(async (tx) => { + const g = await tx.createTask(Hold, { name: "g" }, { ownership: { kind: "conversation" } }); + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "task", taskId: g } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + return { g, p, c: c.id }; + }, context); + await eventually(async () => (await status(harness, p)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // P's end is in flight; a creation in C is admitted after it, while P still shows live. + const conversation = (await harness.conversation(c, context))!; + const held = storage.holdCommits(); + gate("p").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = conversation.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + // A reader of G's live work (as G's end decision is) waits until the creation is published too. + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ownedKey(g)); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + gate("g").resolve(); + await harness.waitForTask(g, context); + await harness.close(context); + }); + + it("keeps the nodes a chain load installed for an unpublished creation through the sweep", async () => { + const storage = new ControlledStorage(); + let opened = await openTasks(storage, [Hold]); + const root = await opened.harness.root(context); + opened.harness.resume(); + const tree = await root.commit(async (tx) => { + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: b } }); + return { b, c: c.id }; + }, context); + gate("b").resolve(); + await opened.harness.waitForTask(tree.b, context); + await opened.harness.close(context); + // After reopen, B and C's edge are known only once a chain load reads them. + opened = await openTasks(storage.reopen(), [Hold]); + const harness = opened.harness; + const sizes = () => + (harness as unknown as Record)[ + SCHEDULER_INDEX_SIZES + ]!; + expect(sizes().settled).toBe(0); + const c = (await harness.conversation(tree.c, context))!; + const held = storage.holdCommits(); + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await held.entered; + // The admission loaded B; the sweep that load scheduled has run on the line since, and kept it. + await harness.waitForIdle(context); + await flush(); + expect(sizes().settled).toBe(1); + expect(sizes().pins).toBeGreaterThan(0); + held.release(); + await creating; + // Published below a known chain: nothing is left to load, and no pin is left. + expect(sizes().unloaded).toBe(0); + expect(sizes().pins).toBe(0); + expect(await settled(harness.waitForIdle(context))).toBe(false); + harness.resume(); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); +}); + +describe("group commit: records a decision copies", () => { + type Probe = { readonly name: string }; + /** Runtimes of running probes, by name, so a test commits for them. */ + const runtimes = new Map>(); + const untilAborted = (signal: AbortSignal) => + new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); + /** Holds until aborted, its runtime exposed; fails once its gate opens. Its abort handler holds too. */ + const Probe = defineTask({ + name: "gc.probe", + version: 1, + initial: () => ({ phase: "hold", n: 0 }), + phases: { + hold: async (task, runtime, ctx) => { + runtimes.set(task.input.name, runtime); + const failing = gate(task.input.name).promise.then(() => "fail" as const); + if ((await Promise.race([failing, untilAborted(runtime.signal)])) !== "fail") return; + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "failed", error: { message: "gate" } } }), + ctx, + ); + }, + }, + abort: async (_task, runtime) => untilAborted(runtime.signal), + }); + /** Creates probes `x` and `y` and waits on them `failFast`. */ + const Waiter = defineTask({ + name: "gc.fail-fast", + version: 1, + initial: () => ({ phase: "spawn" }), + phases: { + spawn: async (task, runtime, ctx) => + runtime.commit(async (tx) => { + const on: TaskId[] = []; + for (const name of ["x", "y"]) { + on.push(await tx.createTask(Probe, { name }, { ownership: { kind: "task", taskId: task.id } })); + } + return { status: "waiting", checkpoint: { phase: "done" }, on, policy: "failFast" }; + }, ctx), + done: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx), + }, + abort: async (_task, runtime) => untilAborted(runtime.signal), + }); + + afterEach(() => { + runtimes.clear(); + }); + + /** Advance probe `name` to `n = 1` while the mark-writing commit is in flight, and run a reconcile pass meanwhile. */ + async function raceMark(storage: ControlledStorage, harness: Harness, name: string, mark: () => Promise) { + const heldMark = storage.holdCommits(); + const marking = mark(); + await heldMark.entered; + let prepared = false; + const advancing = runtimes.get(name)!.commit(() => { + prepared = true; + return { status: "running", checkpoint: { phase: "hold", n: 1 } }; + }, context); + await eventually(() => prepared); + await flush(); + // Global idle schedules a reconcile pass, which reaches the line while both are unpublished. + harness.waitForIdle(context).catch(() => {}); + await flush(); + const heldAdvance = storage.holdCommits(); + heldMark.release(); + await heldAdvance.entered; + await flush(); + heldAdvance.release(); + await Promise.all([marking, advancing]); + await flush(); + } + + it("marks a child in a cascade from its current record, not one its pending commit replaces", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Probe]); + const root = await harness.root(context); + const { parent, child } = await root.commit(async (tx) => { + const parent = await tx.createTask(Probe, { name: "p" }, { ownership: { kind: "conversation" } }); + const child = await tx.createTask(Probe, { name: "c" }, { ownership: { kind: "task", taskId: parent } }); + return { parent, child }; + }, context); + harness.resume(); + await eventually(() => runtimes.has("c")); + await flush(); + await raceMark(storage, harness, "c", () => harness.abortTask(parent, context)); + await eventually(async () => (await harness.getTask(child, context))?.abortRequested === true); + expect((await harness.getTask(child, context))?.state).toMatchObject({ checkpoint: { n: 1 } }); + await harness.close(context); + }); + + it("marks the other tasks of a failFast wait from their current records", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Probe, Waiter]); + const root = await harness.root(context); + await root.commit((tx) => tx.createTask(Waiter, null, { ownership: { kind: "conversation" } }), context); + harness.resume(); + await eventually(() => runtimes.has("x") && runtimes.has("y")); + await flush(); + const y = runtimes.get("y")!.taskId; + // X fails while its commit is in flight; the pass that marks Y runs while Y's own commit is pending. + await raceMark(storage, harness, "y", async () => { + gate("x").resolve(); + }); + await eventually(async () => (await harness.getTask(y, context))?.abortRequested === true); + expect((await harness.getTask(y, context))?.state).toMatchObject({ checkpoint: { n: 1 } }); + await harness.close(context); + }); +}); + +describe("group commit: reads on the line", () => { + it("reads no task record from Storage for more live and awaited tasks than the record cache's count", async () => { + class Counting extends ControlledStorage { + taskReads = 0; + override task(...args: Parameters) { + this.taskReads++; + return super.task(...args); + } + } + const Child = defineTask({ + name: "gc.child", + version: 1, + initial: () => ({ phase: "end" }), + phases: { end: async (task, runtime, ctx) => runtime.commit(() => completedWith(task.id), ctx) }, + abort: async () => {}, + }); + const Parent = defineTask({ + name: "gc.parent", + version: 1, + initial: () => ({ phase: "spawn" }), + phases: { + spawn: async (task, runtime, ctx) => + runtime.commit(async (tx) => { + const on: TaskId[] = []; + for (let n = 0; n < 300; n++) { + on.push(await tx.createTask(Child, null, { ownership: { kind: "task", taskId: task.id } })); + } + return { status: "waiting", checkpoint: { phase: "read", on }, on, policy: "allSettled" }; + }, ctx), + read: async (task, runtime, ctx) => { + const outcomes = await runtime.outcomes(task.state.checkpoint.on, ctx); + const sum = outcomes.reduce((total, outcome) => total + (outcome.status === "completed" ? 1 : 0), 0); + await runtime.commit(() => completedWith(sum), ctx); + }, + }, + abort: async () => {}, + }); + const storage = new Counting(); + const { harness } = await openTasks(storage, [Parent, Child]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Parent, null, { ownership: { kind: "conversation" } }), + context, + ); + storage.taskReads = 0; + harness.resume(); + expect((await harness.waitForTask(id, context)).state.outcome).toEqual({ status: "completed", result: 300 }); + // Each change of a live task, and the parent's read of the outcomes it waited on, finds the record in memory. + expect(storage.taskReads).toBe(0); + await harness.close(context); + }); +}); + +describe("group commit: outcomes and publication through the Harness", () => { + it("finds the submission an unpublished commit admitted with the same request ID", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const draft = { type: "write", entry: { kind: "note", data: {} }, requestId: "r1" } as const; + const held = storage.holdCommits(); + const first = root.submit(draft, context); + await held.entered; + const second = root.submit(draft, context); + await flush(); + held.release(); + expect((await second).id).toBe((await first).id); + await harness.close(context); + }); + + it("runs a wait continuation of one commit before the next commit of the same call is published", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + const one = await root.commit( + (tx) => tx.createTask(Hold, { name: "one" }, { ownership: { kind: "conversation" } }), + context, + ); + const two = await other.commit( + (tx) => tx.createTask(Hold, { name: "two" }, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await eventually( + async () => (await status(harness, one)) === "running" && (await status(harness, two)) === "running", + ); + const ended = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") ended.add(change.value.id); + } + }); + // Both end in one call, behind a commit in flight. + const held = storage.holdCommits(); + const blocker = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + let seen: boolean | undefined; + const waited = harness.waitForTask(one, context).then(() => { + seen = ended.has(two); + }); + gate("one").resolve(); + await flush(); + gate("two").resolve(); + await flush(); + held.release(); + await blocker; + await waited; + await harness.waitForTask(two, context); + expect(seen).toBe(false); + await harness.close(context); + }); + + it("signals an invocation when it closes during a call in flight, and stores that call before Storage closes", async () => { + const storage = new ControlledStorage(); + const signalled = deferred(); + const Listens = defineTask({ + name: "gc.listens", + version: 1, + initial: () => ({ phase: "wait" }), + phases: { + wait: async (_task, runtime) => { + await aborted(runtime.signal).catch(() => signalled.resolve()); + }, + }, + abort: async () => {}, + }); + const { harness } = await openTasks(storage, [Listens]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Listens, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await eventually(async () => (await status(harness, id)) === "running"); + const held = storage.holdCommits(); + const entry = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + const closing = harness.close(context); + await signalled.promise; + expect(await settled(closing)).toBe(false); + held.release(); + await entry; + await closing; + const reopened = await openTasks(storage.reopen(), [Listens]); + const conversation = (await reopened.harness.conversation(root.id as ConversationId, context))!; + expect((await conversation.entries({}, 10, undefined, context)).items.map((item) => item.kind)).toContain("note"); + // Close wrote the task it stopped back as pending. + expect(await status(reopened.harness, id)).toBe("pending"); + await reopened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-hook-reads.test.ts b/packages/durable/test/harness-hook-reads.test.ts new file mode 100644 index 000000000..2681d244d --- /dev/null +++ b/packages/durable/test/harness-hook-reads.test.ts @@ -0,0 +1,94 @@ +import { fauxAssistantMessage } from "@earendil-works/pi-ai"; +import { + defineExtension, + defineTask, + type EntryRecord, + GenerationTask, + hook, + MemoryStorage, + UserEntry, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { allEntries, chatSetup, openChat } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { completed } from "./task-support.ts"; + +describe("entry reads for hooks and tasks", () => { + // #11141 + it("reads raw history and single entries from a hook", async () => { + const setup = chatSetup(); + const seen: { kinds: string[]; user?: string }[] = []; + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + onYield: async (_answer, api, ctx) => { + const page = await api.entries({ order: "ascending" }, 10, undefined, ctx); + const first = page.items[0]!; + const user = await api.entry(UserEntry, first.id, ctx); + seen.push({ + kinds: page.items.map((entry) => entry.kind), + ...(user === undefined ? {} : { user: user.kind }), + }); + // Another kind: undefined. + expect(await api.entry(GenerationTask as never, first.id, ctx)).toBeUndefined(); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + // The answer is not committed yet when `onYield` asks. + expect(seen).toEqual([{ kinds: ["pi.user"], user: "pi.user" }]); + await harness.close(context); + }); + + // #11141 + it("pages a task's conversation history by entry ID range", async () => { + const setup = chatSetup(); + const pages: (readonly EntryRecord[])[] = []; + const Reader = defineTask({ + name: "test.reader", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, ctx) => { + const all = await runtime.entries({ order: "ascending" }, 1, undefined, ctx); + pages.push(all.items); + const rest = await runtime.entries({ order: "ascending" }, 1, all.next, ctx); + pages.push(rest.items); + const bounded = await runtime.entries( + { minEntryId: rest.items[0]!.id, maxEntryId: rest.items[0]!.id }, + 10, + undefined, + ctx, + ); + pages.push(bounded.items); + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); + setup.registry.install(defineExtension({ name: "reader", tasks: [Reader] })); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const id = await root.commit( + (tx) => tx.createTask(Reader, null, { ownership: { kind: "conversation" } }), + context, + ); + await harness.waitForTask(id, context); + const entries = await allEntries(root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); + expect(pages.map((page) => page.map((entry) => entry.id))).toEqual([ + [entries[0]!.id], + [entries[1]!.id], + [entries[1]!.id], + ]); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-host-inputs.test.ts b/packages/durable/test/harness-host-inputs.test.ts index 4719cea94..4922cd0d4 100644 --- a/packages/durable/test/harness-host-inputs.test.ts +++ b/packages/durable/test/harness-host-inputs.test.ts @@ -6,7 +6,6 @@ import { fauxToolCall, type Models, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -19,6 +18,7 @@ import { MemoryStorage, type Registry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { allEntries, textOf } from "./chat-support.ts"; import { context } from "./session-support.ts"; @@ -86,7 +86,7 @@ describe("HarnessOptions.models and settings as functions", () => { defineTool({ name: "probe", description: "Records api.models before and after the host swaps it", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { seen.push(api.models); current = second; diff --git a/packages/durable/test/harness-idle-scopes.test.ts b/packages/durable/test/harness-idle-scopes.test.ts new file mode 100644 index 000000000..b034150b0 --- /dev/null +++ b/packages/durable/test/harness-idle-scopes.test.ts @@ -0,0 +1,591 @@ +// Idle checks without per-commit scans: a task change settles only the conversations above it (idle waiters and kept +// context ranges), expiry runs oldest first from an ordered index, and `wake()` reads only sleeping tasks while work +// is active. Every case checks the incremental idle state against a full recompute (`SCHEDULER_IDLE_STATE`). +import { createModels } from "@earendil-works/pi-ai"; +import { + type Conversation, + type ConversationId, + createRegistry, + defineExtension, + defineTask, + Harness, + MemoryStorage, + type SetTimer, + type Storage, + type TaskId, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { SCHEDULER_IDLE_STATE } from "../src/harness/harness.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { ROOT_CONVERSATION_ID, type StorageWrite } from "../src/types.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { aborted, type Deferred, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +const gates = new Map>(); +/** Resolves once the named task's context read returned. */ +const reads = new Map>(); + +function named(map: Map>, name: string): Deferred { + let found = map.get(name); + if (found === undefined) { + found = deferred(); + map.set(name, found); + } + return found; +} + +afterEach(() => { + gates.clear(); + reads.clear(); +}); + +/** + * Reads the context of conversation `read` (when given), then holds until its gate opens and completes; its abort + * handler ends it `aborted`. + */ +const Work = defineTask<{ name: string; read?: number }, { phase: "run" }, null>({ + name: "test.idle.work", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + if (task.input.read !== undefined) { + await runtime.context(task.input.read as ConversationId, ctx); + named(reads, task.input.name).resolve(); + } + const opened = await Promise.race([ + named(gates, task.input.name).promise.then(() => true), + aborted(runtime.signal), + ]); + if (opened !== true) return; + await runtime.commit(() => ({ status: "terminal", outcome: { status: "completed", result: null } }), ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +type IdleState = Record<"contexts" | "idleContexts" | "idleMismatches" | "fullIdleChecks", number>; + +function idleState(harness: Harness): IdleState { + return (harness as unknown as Record)[SCHEDULER_IDLE_STATE]!; +} + +/** Background timers declined, as Cloudflare Workers do; foreground ones on `setTimeout`. */ +const declineBackground: SetTimer = (callback, delayMs, { background }) => { + if (background) return undefined; + const timer = setTimeout(callback, delayMs); + return () => clearTimeout(timer); +}; + +/** + * A resumed Harness running `Work`, over MemoryStorage counting the entry rows each scan returns, with `clock` as its + * time, the retention `retention` returns, and background timers declined unless `timers`. + */ +async function open(options: { clock?: () => number; retention?: () => number; timers?: boolean } = {}) { + const scanned = { rows: 0 }; + const memory = new MemoryStorage(); + const storage = new Proxy(memory, { + get(target, key) { + const value: unknown = Reflect.get(target, key, target); + if (key === "scanEntries") { + return async (...args: Parameters) => { + const page = await target.scanEntries(...args); + scanned.rows += page.items.length; + return page; + }; + } + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "tasks", tasks: [Work] })); + const retention = options.retention ?? (() => 600_000); + const harness = await Harness.open( + storage, + { + models: createModels(), + registry, + settings: () => ({ contextRetentionMs: retention() }), + ...(options.clock === undefined ? {} : { now: options.clock }), + ...(options.timers === true ? {} : { setTimer: declineBackground }), + }, + context, + ); + harness.resume(); + const root = await harness.root(context); + /** A new ownerless conversation with `entries` entries. */ + const conversation = async (entries = 2): Promise => { + const created = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + for (let index = 0; index < entries; index++) { + await created.commit((tx) => tx.appendEntry(created.id, { kind: "note", data: { index } }), context); + } + return created; + }; + /** Start `Work` named `name` in `into`, reading conversation `read`, and wait for its read. */ + const work = async (name: string, into: Conversation, read?: ConversationId): Promise => { + const input = read === undefined ? { name } : { name, read: read as number }; + const id = await into.commit( + (tx) => tx.createTask(Work, input, { ownership: { kind: "conversation" }, conversationId: into.id }), + context, + ); + if (read !== undefined) await named(reads, name).promise; + return id; + }; + /** Open the gate of `name` and wait for its task, `id`, to end. */ + const end = async (name: string, id: TaskId): Promise => { + named(gates, name).resolve(); + await harness.waitForTask(id, context); + }; + /** The entry rows a read of `read` from a new task in `into` scans. */ + const rowsOf = async (name: string, into: Conversation, read: ConversationId): Promise => { + const before = scanned.rows; + const id = await work(name, into, read); + const rows = scanned.rows - before; + await end(name, id); + return rows; + }; + return { harness, root, conversation, work, end, rowsOf }; +} + +/** Flush until scheduler passes started by the last changes settled. */ +async function quiet(): Promise { + for (let round = 0; round < 4; round++) await flush(); +} + +describe("idle waiters by scope", () => { + it("resolve through owner chains, owned conversations and ended tasks between, ignoring background work", async () => { + const { harness, root } = await open(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Work, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const a = await tx.createTask(Work, { name: "a" }, { ownership: { kind: "task", taskId: b } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: a } }); + const d = await tx.createTask( + Work, + { name: "d" }, + { ownership: { kind: "conversation" }, conversationId: c.id }, + ); + const g = await tx.createTask( + Work, + { name: "g" }, + { ownership: { kind: "conversation" }, conversationId: c.id, background: true }, + ); + return { p, b, a, c: c.id, d, g }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + const idleC = c.waitForIdle(context); + const idleRoot = root.waitForIdle(context); + const idleAll = harness.waitForIdle(context); + // The owners A and B hold for D: live, though their handlers returned. + named(gates, "a").resolve(); + named(gates, "b").resolve(); + await eventually(async () => (await harness.getTask(tree.b, context))!.state.status === "completing"); + expect([await settled(idleC), await settled(idleRoot), await settled(idleAll)]).toEqual([false, false, false]); + // D ends: C is idle (G is background); A and B finalize; P still holds the root. + named(gates, "d").resolve(); + await idleC; + await harness.waitForTask(tree.b, context); + expect([await settled(idleRoot), await settled(idleAll)]).toEqual([false, false]); + named(gates, "p").resolve(); + await idleRoot; + await idleAll; + expect((await harness.getTask(tree.g, context))!.state.status).not.toBe("terminal"); + await harness.close(context); + }); + + it("count work whose chain is not loaded yet as inside, until the open reconcile loads it", async () => { + const storage = new ControlledStorage(); + const first = await openTasks(storage, [Work]); + const firstRoot = await first.harness.root(context); + const tree = await firstRoot.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + const d = await tx.createTask( + Work, + { name: "d" }, + { ownership: { kind: "conversation" }, conversationId: c.id }, + ); + const other = await tx.createConversation({ ownership: { kind: "ownerless" } }); + return { p, c: c.id, d, other: other.id }; + }, context); + await first.harness.close(context); + // Reopened, no chain is loaded: P and D count as inside every scope until the reconcile pass loads theirs. + const { harness } = await openTasks(storage.reopen(), [Work]); + const idleOther = (await harness.conversation(tree.other, context))!.waitForIdle(context); + const idleRoot = (await harness.root(context)).waitForIdle(context); + await idleOther; + expect(await settled(idleRoot)).toBe(false); + harness.resume(); + named(gates, "d").resolve(); + await harness.waitForTask(tree.d, context); + expect(await settled(idleRoot)).toBe(false); + named(gates, "p").resolve(); + await idleRoot; + await harness.close(context); + }); + + it("resolve a waiter on an unrelated conversation without settling the busy one", async () => { + const { harness, conversation, work, end } = await open(); + const [busy, other] = [await conversation(0), await conversation(0)]; + const held = await work("held", busy); + const quick = await work("quick", other); + const idleBusy = busy.waitForIdle(context); + const idleOther = other.waitForIdle(context); + await end("quick", quick); + await idleOther; + expect(await settled(idleBusy)).toBe(false); + await end("held", held); + await idleBusy; + await harness.close(context); + }); +}); + +describe("kept context ranges", () => { + it("settles a run end whose sweep drops the chain without a check of every scope", async () => { + const { harness, root } = await open(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const t = await tx.createTask(Work, { name: "t" }, { ownership: { kind: "task", taskId: p } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: t } }); + // A context range keeps entries: an empty conversation has none to keep. + await tx.appendEntry(c.id, { kind: "note" }); + return { p, t, c: c.id }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + const d = await c.commit( + (tx) => + tx.createTask( + Work, + { name: "d", read: tree.c as number }, + { ownership: { kind: "conversation" }, conversationId: tree.c }, + ), + context, + ); + await named(reads, "d").promise; + named(gates, "t").resolve(); + await eventually(async () => (await harness.getTask(tree.t, context))!.state.status === "completing"); + await quiet(); + const before = idleState(harness); + expect(before).toMatchObject({ contexts: 1, idleContexts: 0, idleMismatches: 0 }); + // D's end makes C idle, T finalizes, and the sweep forgets C's chain: none of it checks every scope. + named(gates, "d").resolve(); + await harness.waitForTask(d, context); + await harness.waitForTask(tree.t, context); + await quiet(); + expect(idleState(harness)).toEqual({ ...before, idleContexts: 1 }); + named(gates, "p").resolve(); + await harness.waitForTask(tree.p, context); + await harness.close(context); + }); + + it("drops a range past its retention when a task change makes it busy again, with background timers declined", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [x, z] = [await conversation(), await conversation(0)]; + await end("reader", await work("reader", z, x.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1, idleMismatches: 0 }); + clock = 2_500; + // No timer expired it; the task change that makes X busy checks expiry first. + const busy = await work("busy", x); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await end("busy", busy); + await harness.close(context); + }); + + it("drops a range past its retention at a reconcile pass with nothing to do, with background timers declined (review)", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [x, z] = [await conversation(), await conversation(0)]; + await end("reader", await work("reader", z, x.id)); + const held = await work("held", z); + // Held runs: no task changes from here. + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1 }); + clock = 2_500; + // A wait on busy Z asks for a reconcile pass, which has no work and changes no task: it checks expiry. + const idle = z.waitForIdle(context); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await end("held", held); + await idle; + await harness.close(context); + }); + + it("expires in idle order when the clock goes back", async () => { + let clock = 1_000; + const { harness, conversation, work, end, rowsOf } = await open({ + clock: () => clock, + retention: () => 1_000, + }); + const [a, b, z] = [await conversation(3), await conversation(5), await conversation(0)]; + await end("read a", await work("read a", z, a.id)); + clock = 500; + await end("read b", await work("read b", z, b.id)); + expect(idleState(harness)).toMatchObject({ contexts: 2, idleContexts: 2, idleMismatches: 0 }); + // B went idle at 500 and expires first, though it went idle last. + clock = 1_600; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleMismatches: 0 }); + expect(await rowsOf("again b", z, b.id)).toBe(5); + expect(await rowsOf("again a", z, a.id)).toBe(0); + await harness.close(context); + }); + + it("drops every idle range when retention becomes zero, also one that went idle after the clock's now (review)", async () => { + let clock = 2_000; + let retention = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => retention }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1 }); + // The clock goes back below A's idle start; then retention 0 and an unrelated task change. + clock = 1_000; + retention = 0; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it("keeps an idle range's expiry when another task reads it again", async () => { + let clock = 1_000; + const { harness, conversation, work, end, rowsOf } = await open({ + clock: () => clock, + retention: () => 1_000, + }); + const [a, b, z] = [await conversation(3), await conversation(5), await conversation(0)]; + await end("read a", await work("read a", z, a.id)); + clock = 1_500; + await end("read b", await work("read b", z, b.id)); + clock = 1_800; + expect(await rowsOf("again a", z, a.id)).toBe(0); + clock = 2_100; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleMismatches: 0 }); + expect(await rowsOf("late b", z, b.id)).toBe(0); + await harness.close(context); + }); + + it("drops a range from the expiry order when an entry commit finds it expired", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + clock = 2_500; + // An entry-only commit settles no idleness: the range is dropped where it would be extended. + await a.commit((tx) => tx.appendEntry(a.id, { kind: "note", data: { late: true } }), context); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it("drops a range from the expiry order when an entry commit cannot extend it", async () => { + const { harness, conversation, work, end } = await open(); + const [a, z] = [await conversation(0), await conversation(0)]; + const first = await a.commit((tx) => tx.appendEntry(a.id, { kind: "note" }), context); + const second = await a.commit((tx) => tx.appendEntry(a.id, { kind: "note" }), context); + await a.commit((tx) => tx.appendEntry(a.id, { kind: "summary", head: second.id }), context); + await end("read", await work("read", z, a.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1, idleMismatches: 0 }); + // A marker whose head is older than the range's: the range cannot be extended and is dropped. + await a.commit((tx) => tx.appendEntry(a.id, { kind: "summary", head: first.id }), context); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it.each([ + ["shorter", 400, 0], + ["longer", 10_000, 1], + ["zero", 0, 0], + ] as const)("applies a %s retention to idle ranges at an unrelated commit", async (_name, changed, kept) => { + let clock = 1_000; + let retention = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => retention }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + clock = 1_500; + retention = changed; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: kept, idleMismatches: 0 }); + await harness.close(context); + }); + + it("drops a range that goes idle with zero retention", async () => { + const { harness, conversation, work, end } = await open({ retention: () => 0 }); + const a = await conversation(); + const own = await work("own", a, a.id); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 0, idleMismatches: 0 }); + await end("own", own); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); +}); + +/** A random source with a fixed seed. */ +function seeded(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1103515245 + 12345) % 2147483648; + return state / 2147483648; + }; +} + +describe("incremental idle state", () => { + it("equals a full recompute after every step of random nested work", async () => { + const random = seeded(42); + const pick = (items: readonly T[]): T => items[Math.floor(random() * items.length)]!; + const { harness, root } = await open(); + // A context range keeps entries: every conversation gets one. + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + const conversations: ConversationId[] = [root.id]; + const live = new Map(); + let count = 0; + let fullChecks = idleState(harness).fullIdleChecks; + const handle = async (id: ConversationId) => (await harness.conversation(id, context))!; + for (let step = 0; step < 200; step++) { + const action = random(); + const name = `t${count++}`; + if (action < 0.1) { + const created = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + await created.commit((tx) => tx.appendEntry(created.id, { kind: "note" }), context); + conversations.push(created.id); + } else if (action < 0.45 || live.size === 0) { + // A task in any conversation, also one whose owner chain ended and was swept; it may read a context. + const into = pick(conversations); + const read = random() < 0.6 ? pick(conversations) : undefined; + const background = random() < 0.15; + const input = read === undefined ? { name } : { name, read: read as number }; + const target = await handle(into); + const id = await target.commit( + (tx) => + tx.createTask(Work, input, { ownership: { kind: "conversation" }, conversationId: into, background }), + context, + ); + live.set(name, { id, conversation: into }); + if (read !== undefined) await named(reads, name).promise; + } else if (action < 0.6) { + // Work a live task owns: a child task, or an owned conversation with a task that reads it. + const [ownerName, owner] = pick([...live]); + const state = (await harness.getTask(owner.id, context))?.state.status; + if (state === undefined || state === "terminal" || state === "completing") { + live.delete(ownerName); + continue; + } + const conversation = random() < 0.5; + const created = await (await handle(owner.conversation)).commit(async (tx) => { + if (!conversation) { + const id = await tx.createTask(Work, { name }, { ownership: { kind: "task", taskId: owner.id } }); + return { id, into: owner.conversation, read: false }; + } + const child = await tx.createConversation({ ownership: { kind: "task", taskId: owner.id } }); + await tx.appendEntry(child.id, { kind: "note" }); + const id = await tx.createTask( + Work, + { name, read: child.id as number }, + { ownership: { kind: "conversation" }, conversationId: child.id }, + ); + return { id, into: child.id, read: true }; + }, context); + if (created.read) conversations.push(created.into); + live.set(name, { id: created.id, conversation: created.into }); + if (created.read) await named(reads, name).promise; + } else { + const [ended] = pick([...live]); + live.delete(ended); + named(gates, ended).resolve(); + } + await quiet(); + const state = idleState(harness); + expect(state.idleMismatches, `step ${step}`).toBe(0); + fullChecks = state.fullIdleChecks; + } + const state = idleState(harness); + expect(state.contexts).toBeGreaterThan(5); + expect(state.idleContexts).toBeGreaterThan(0); + expect(state.idleContexts).toBeLessThan(state.contexts); + expect(fullChecks).toBe(0); + for (const gate of gates.values()) gate.resolve(); + await harness.close(context); + }); +}); + +describe("wake()", () => { + const kind = Work.definition.name; + const record = (id: number, state: unknown, extra: Record = {}): StorageWrite => ({ + type: "task", + value: { + id, + conversationId: ROOT_CONVERSATION_ID, + kind, + version: 1, + input: { name: `seeded${id}` }, + state, + background: false, + abortRequested: false, + ...extra, + } as never, + }); + const sleeping = (until: number) => ({ status: "sleeping", checkpoint: { phase: "run" }, until }); + const SLEEPS = record(10, sleeping(5_000)); + + it.each([ + ["a sleeping task", [SLEEPS], { active: false, at: 5_000 }], + [ + "an abort-marked sleeping task, which runs its handler", + [SLEEPS, record(11, sleeping(3_000), { abortRequested: true })], + { active: true, at: 5_000 }, + ], + ["a sleeping task that is due", [record(11, sleeping(500))], { active: true }], + [ + "a task waiting on a live one", + [SLEEPS, record(11, { status: "waiting", checkpoint: { phase: "run" }, on: [10], policy: "allSettled" })], + { active: false, at: 5_000 }, + ], + [ + "a completing task holding for a sleeping child", + [ + record(11, { status: "completing", outcome: { status: "completed", result: null } }), + record(10, sleeping(5_000), { owner: 11 }), + ], + { active: false, at: 5_000 }, + ], + [ + "a task no definition can take", + [SLEEPS, record(11, { status: "pending", checkpoint: { phase: "run" } }, { kind: "test.idle.missing" })], + { active: false, at: 5_000 }, + ], + [ + "a pending task", + [SLEEPS, record(11, { status: "pending", checkpoint: { phase: "run" } })], + { active: true, at: 5_000 }, + ], + ] as const)("with %s", async (_name, writes, expected) => { + const storage = new MemoryStorage(); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, ...writes], context); + const { harness } = await openTasks(storage, [Work], { now: () => 1_000 }); + await quiet(); + expect(harness.wake()).toEqual(expected); + await harness.close(context); + }); + + it("with an invocation running: active, and the earliest sleep still found", async () => { + const storage = new MemoryStorage(); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, SLEEPS], context); + const { harness } = await openTasks(storage, [Work], { now: () => 1_000 }); + harness.resume(); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Work, { name: "running" }, { ownership: { kind: "conversation" } }), + context, + ); + await eventually(async () => (await harness.getTask(id, context))!.state.status === "running"); + expect(harness.wake()).toEqual({ active: true, at: 5_000 }); + named(gates, "running").resolve(); + await harness.waitForTask(id, context); + await quiet(); + expect(harness.wake()).toEqual({ active: false, at: 5_000 }); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index 19a719286..a4cb91837 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -9,12 +9,10 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { type Conversation, - defineDoc, defineTool, type EntryRecord, GenerationTask, @@ -30,12 +28,13 @@ import { ToolTask, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { recordUsage } from "../src/harness/usage.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; -import { ControlledStorage, context, documentChanges, flush } from "./session-support.ts"; +import { ControlledStorage, context, documentChanges, documentValues, flush } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; const directories = new Set(); @@ -68,7 +67,7 @@ function holdTool(setup: ChatSetup, gate: Deferred, result: ToolExecutionResult defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return result; @@ -82,6 +81,14 @@ function queuedNote(text: string) { return { kind: "note", model: [{ role: "user", content: text, timestamp: 0 }] } as const; } +/** A faux response that records the request's messages as `role:text` and answers `text`. */ +function recordRequest(requests: string[][], text: string): FauxResponseStep { + return (request) => { + requests.push(request.messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)); + return answer(text); + }; +} + const HOLD = fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }); async function status(submission: Submission): Promise { @@ -183,22 +190,20 @@ describe("inbox", () => { await first.reached; const f1 = await root.submit({ type: "input", content: "f1" }, context); const f2 = await root.submit({ type: "input", content: "f2" }, context); - // Occupy the line, let the answer queue its boundary commit behind it, then change the mode. - const held = storage.holdCommits(); - const Marker = defineDoc<{ n: number }>({ - checkpointWhen: () => false, - kind: "test.marker", - version: 1, - scope: "session", - initial: () => ({ n: 0 }), - }); - const occupying = root.commit(async (tx) => void (await tx.doc(Marker)).n++, context); - await held.entered; + // Occupy the line with a commit whose callback waits, let the answer queue its boundary commit behind it, then + // change the mode. + const entered = deferred(); + const release = deferred(); + const occupying = root.commit(async () => { + entered.resolve(); + await release.promise; + }, context); + await entered.promise; first.release(); await yielded.promise; await flush(); setup.settings.followUpMode = "all"; - held.release(); + release.resolve(); await occupying; const settled = await f2.wait(context); expect(await status(f1)).toEqual({ ...settled, id: f1.id, entry: expect.any(Number) }); @@ -239,11 +244,7 @@ describe("inbox", () => { const gate = deferred(); holdTool(setup, gate); const requests: string[][] = []; - const record: FauxResponseStep = (request) => { - requests.push(request.messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)); - return answer("fresh"); - }; - setup.faux.setResponses([HOLD, record]); + setup.faux.setResponses([HOLD, recordRequest(requests, "fresh")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); @@ -348,11 +349,164 @@ describe("inbox", () => { await harness.close(context); }); - it("ends the run with a pi.reset entry when a tool requests a handoff", async () => { + it("continues the run in a new context when a tool requests a handoff", async () => { const setup = chatSetup(); const gate = deferred(); gate.resolve(); holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "after handoff")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); + const entries = await allEntries(root); + expect(settled).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.assistant:after handoff", + ]); + const reset = entries.find((entry) => ResetEntry.is(entry))!; + expect(reset.head).toBe(reset.id); + expect(requests).toEqual([["system:", "user:continue here"]]); + expect(setup.faux.state.callCount).toBe(2); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("places queued writes and steers after the handoff text and adds the steers to the continuing run", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "after handoff")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const steer = await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); + const write = await root.submit({ type: "write", entry: queuedNote("w") }, context); + gate.resolve(); + await input.wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "note:w", + "pi.user:s", + "pi.assistant:after handoff", + ]); + expect(requests).toEqual([["system:", "user:continue here", "user:w", "user:s"]]); + for (const submission of [input, steer]) { + expect(await status(submission)).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + } + expect(await status(write)).toMatchObject({ status: "done" }); + await harness.close(context); + }); + + it("holds a follow-up queued across a handoff until the continuing run's final boundary", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const after = gated(answer("after handoff")); + setup.faux.setResponses([HOLD, after.step, answer("follow-up")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const followUp = await root.submit({ type: "input", content: "f" }, context); + gate.resolve(); + await after.reached; + expect((await status(input)).status).toBe("placed"); + expect(await inbox(harness, root)).toEqual([[followUp.id, "followUp"]]); + after.release(); + await followUp.wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.assistant:after handoff", + "pi.user:f", + "pi.assistant:follow-up", + ]); + const answers = entries.filter((entry) => entry.kind === "pi.assistant"); + expect(await status(input)).toMatchObject({ status: "done", answer: answers[1]!.id }); + expect(await status(followUp)).toMatchObject({ status: "done", answer: answers[2]!.id }); + await harness.close(context); + }); + + it("ends a handed-off run at a queued reset and runs earlier follow-ups after it", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "fresh")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const followUp = await root.submit({ type: "input", content: "f" }, context); + await root.reset(undefined, context); + gate.resolve(); + await followUp.wait(context); + expect(await status(input)).toMatchObject({ status: "unanswered", reason: "reset" }); + expect(transcript(await allEntries(root))).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.reset", + "pi.user:f", + "pi.assistant:fresh", + ]); + expect(requests).toEqual([["system:", "user:f"]]); + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + it("hands off again in a later round of the same run", async () => { + const setup = chatSetup(); + let handoffs = 0; + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Hands off", + parameters: obj({}), + execute: async () => ({ output: [], control: { handoff: `h${++handoffs}` } }), + }), + ); + setup.faux.setResponses([HOLD, HOLD, answer("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:h1", + "pi.assistant", + "pi.tool-result", + "pi.reset:h2", + "pi.assistant:done", + ]); + expect(settled).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + const messages = (await root.context(context)).messages; + expect(messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)).toEqual([ + "system:", + "user:h2", + "assistant:done", + ]); + await harness.close(context); + }); + + it("ends the run after the handoff when the round also terminates", async () => { + const setup = chatSetup(); + const gate = deferred(); + gate.resolve(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here", terminate: true } }); setup.faux.setResponses([HOLD]); const { harness, root } = await openChat(new MemoryStorage(), setup); const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); @@ -360,7 +514,6 @@ describe("inbox", () => { const calling = entries.find((entry) => entry.kind === "pi.assistant")!; expect(settled).toMatchObject({ status: "done", answer: calling.id }); expect(transcript(entries)).toEqual(["pi.user:a", "pi.assistant", "pi.tool-result", "pi.reset:continue here"]); - expect(entries.at(-1)!.head).toBe(entries.at(-1)!.id); expect(setup.faux.state.callCount).toBe(1); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); @@ -460,10 +613,13 @@ describe("inbox", () => { setup.faux.setResponses([first.step, answer("second")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const ops: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); + } + }, + end: () => {}, }); const input = await root.submit({ type: "input", content: "a" }, context); await first.reached; @@ -630,7 +786,7 @@ describe("inbox", () => { await harness.close(context); }); - it("writes the last handoff in call order and then runs queued follow-ups in the new context", async () => { + it("writes only the last handoff in call order and runs queued follow-ups after the continuing run", async () => { const setup = chatSetup(); const firstGate = deferred(); const secondGate = deferred(); @@ -640,7 +796,7 @@ describe("inbox", () => { defineTool({ name: "later", description: "Finishes first", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await secondGate.promise; return { output: [], control: { handoff: "two" } }; @@ -651,7 +807,7 @@ describe("inbox", () => { [fauxToolCall("hold", {}, { id: "c1" }), fauxToolCall("later", {}, { id: "c2" })], { stopReason: "toolUse" }, ); - setup.faux.setResponses([round, answer("follow-up")]); + setup.faux.setResponses([round, answer("after handoff"), answer("follow-up")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); @@ -660,13 +816,19 @@ describe("inbox", () => { await waitFor(async () => (await harness.snapshot(LiveDoc, root.id, context))?.tools?.[1]?.status === "done"); firstGate.resolve(); await f.wait(context); - expect(await status(input)).toMatchObject({ status: "done" }); - expect(transcript(await allEntries(root)).slice(-4)).toEqual([ + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", "pi.tool-result", "pi.reset:two", + "pi.assistant:after handoff", "pi.user:f", "pi.assistant:follow-up", ]); + const answers = entries.filter((entry) => entry.kind === "pi.assistant"); + expect(await status(input)).toMatchObject({ status: "done", answer: answers[1]!.id }); await harness.close(context); }); @@ -711,10 +873,13 @@ describe("inbox", () => { const items = []; for (const text of ["x", "y", "z"]) items.push(await root.submit({ type: "input", content: text }, context)); const ops: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); + } + }, + end: () => {}, }); await items[1]!.abort(context); expect(ops).toEqual([[["p", ["items"], 1, 1, []]]]); @@ -730,26 +895,29 @@ describe("inbox", () => { let inboxId: number | undefined; class RecordingStorage extends MemoryStorage { override async commit( - writes: Parameters[0], - adopt: Parameters[1], + batches: Parameters[0], + receive: Parameters[1], callContext: Parameters[2], ) { - for (const write of writes) { + for (const write of batches.flat()) { if (write.type !== "document.change" || write.id !== inboxId) continue; const content = write.content; const empty = content.kind === "base" && (content.value as InboxState).items.length === 0; written.push({ kind: content.kind, empty }); } - return super.commit(writes, adopt, callContext); + return super.commit(batches, receive, callContext); } } const setup = chatSetup(); const first = gated(answer("first")); setup.faux.setResponses([first.step, answer("second")]); const { harness, root } = await openChat(new RecordingStorage(), setup); - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) - if (change.record.kind === "pi.inbox") inboxId = change.record.id; + harness.observeCommits({ + advance: (publication) => { + for (const change of documentChanges(publication)) + if (change.record.kind === "pi.inbox") inboxId = change.record.id; + }, + end: () => {}, }); const input = await root.submit({ type: "input", content: "a" }, context); await first.reached; @@ -912,12 +1080,15 @@ describe("usage", () => { setup.faux.setResponses([answer("one"), answer("two")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: { entries: number; ops: readonly Op[] }[] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind !== "pi.usage") continue; - const entries = publication.changes.filter((other) => other.type === "entry").length; - commits.push({ entries, ops: change.ops }); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind !== "pi.usage") continue; + const entries = publication.changes.filter((other) => other.type === "entry").length; + commits.push({ entries, ops: change.ops }); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "a" }, context)).wait(context); await (await root.submit({ type: "input", content: "b" }, context)).wait(context); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index 019172d03..ce6baa468 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -1,4 +1,5 @@ // Inputs placed as application entries, and data-only writes appended at once (#10407). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,10 +10,8 @@ import { fauxText, fauxToolCall, type Context as ModelContext, - Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type Conversation, ConversationBusy, defineTool, @@ -20,14 +19,16 @@ import { type Harness, InboxDoc, LiveDoc, + type LiveState, MemoryStorage, type SubmissionDraft, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addSection, addTool } from "./harness-support.ts"; +import { viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; @@ -54,7 +55,7 @@ function holdTool(setup: ChatSetup, gate: Deferred): void { defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return { output: [{ type: "text", text: "held" }] }; @@ -263,11 +264,7 @@ describe("data-only writes", () => { const requests: string[][] = []; setup.faux.setResponses([requestLog(requests, HOLD), requestLog(requests, answer("done"))]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const events: AgentEvent[] = []; - const stream = await watchEvents(harness, root.id, context); - stream.start(async (batch) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); const chat = { kind: "chat.message", data: { author: "bob", text: "lunch?" } }; @@ -281,12 +278,13 @@ describe("data-only writes", () => { ["pi.assistant"], ["chat.message", { author: "bob", text: "lunch?" }], ]); - await waitFor(() => events.some((event) => event.type === "entry_appended")); - const appended = events.find((event) => event.type === "entry_appended"); - expect(appended).toMatchObject({ - entry: { kind: "chat.message", id: settled.status === "done" && settled.entry }, + // The view shows it while the run goes on. + const shown = history.views.find((view) => view.entries.some((entry) => entry.kind === "chat.message")); + expect(shown?.entries.at(-1)).toMatchObject({ + kind: "chat.message", + id: settled.status === "done" && settled.entry, }); - expect(events.some((event) => event.type === "run_end")).toBe(false); + expect((shown?.docs["pi.live"] as LiveState).run?.inputs).toEqual([input.id]); // A repeated request ID returns the same write without a second entry. expect((await root.submit({ type: "write", entry: chat, requestId: "bob:1" }, context)).id).toBe(write.id); @@ -303,7 +301,7 @@ describe("data-only writes", () => { ["system:", "user:a"], ["system:", "user:a", "assistant:", "toolResult:held"], ]); - await stream.stop(); + history.detach(); await harness.close(context); }); @@ -374,7 +372,15 @@ describe("data-only writes", () => { await harness.waitForIdle(context); expect((await f.status(context)).status).toBe("queued"); expect((await inboxItems(harness, root)).map((item) => item.id)).toEqual([f.id]); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [input.id], + reason: "model_error", + detail: "invalid request", + first: expect.any(Number), + }, + }); expect(setup.faux.state.callCount).toBe(1); // The next input places the waiting mention first. const g = await root.submit({ type: "input", content: "g" }, context); diff --git a/packages/durable/test/harness-inspect.test.ts b/packages/durable/test/harness-inspect.test.ts index 78aa8f20d..bec664694 100644 --- a/packages/durable/test/harness-inspect.test.ts +++ b/packages/durable/test/harness-inspect.test.ts @@ -94,11 +94,11 @@ describe("Harness.inspect()", () => { const paused = await harness.inspect(context); expect(paused.scheduling).toBe("paused"); expect(paused.tasks.map((entry) => entry.record.id)).toEqual(Object.values(ids)); - expect(stateOf(paused.tasks, ids.gate)).toEqual({ kind: "ready", migrates: false }); - expect(stateOf(paused.tasks, ids.dependent)).toEqual({ kind: "ready", migrates: false }); - expect(stateOf(paused.tasks, ids.migrating)).toEqual({ kind: "ready", migrates: true }); + expect(stateOf(paused.tasks, ids.gate)).toEqual({ kind: "pending", migrates: false }); + expect(stateOf(paused.tasks, ids.dependent)).toEqual({ kind: "pending", migrates: false }); + expect(stateOf(paused.tasks, ids.migrating)).toEqual({ kind: "pending", migrates: true }); // A migration that was never tried is not run to find out. - expect(stateOf(paused.tasks, ids.failing)).toEqual({ kind: "ready", migrates: true }); + expect(stateOf(paused.tasks, ids.failing)).toEqual({ kind: "pending", migrates: true }); expect(stateOf(paused.tasks, ids.noMigration)).toMatchObject({ kind: "blocked", reason: "migration_failed", diff --git a/packages/durable/test/harness-json.test.ts b/packages/durable/test/harness-json.test.ts new file mode 100644 index 000000000..963699770 --- /dev/null +++ b/packages/durable/test/harness-json.test.ts @@ -0,0 +1,23 @@ +import { type JsonValue, track } from "@earendil-works/chord/delta"; +import { describe, expect, it } from "vitest"; +import { assignJson } from "../src/harness/json.ts"; + +type Doc = { message?: JsonValue }; + +function assign(tracker: ReturnType>, value: JsonValue): void { + const change = tracker.beginChange(); + assignJson(change.state as Doc, "message", value); + tracker.adopt(change.prepare()); +} + +describe("assignJson", () => { + it("does not merge a new __proto__ member into Object.prototype", () => { + const tracker = track({}); + assign(tracker, { args: JSON.parse('{"path":"x"}') }); + assign(tracker, { args: JSON.parse('{"path":"xy","__proto__":{"polluted":true}}') }); + + expect(({} as Record).polluted).toBeUndefined(); + const args = (tracker.value.message as { args: object }).args; + expect(Object.getOwnPropertyDescriptor(args, "__proto__")?.value).toEqual({ polluted: true }); + }); +}); diff --git a/packages/durable/test/harness-lifecycle.test.ts b/packages/durable/test/harness-lifecycle.test.ts index be422e526..bf044120b 100644 --- a/packages/durable/test/harness-lifecycle.test.ts +++ b/packages/durable/test/harness-lifecycle.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; -import { createModels, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationId, @@ -17,9 +17,11 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; import { ControlledStorage, context, flush } from "./session-support.ts"; @@ -73,12 +75,12 @@ async function seedRunningTask(storage: Storage): Promise { const id = await storage.mintId>(); const state = { status: "running", checkpoint: { phase: "run" } } as const; const task = { id, conversationId, kind: "test.seeded", version: 1, input: null, background: false, state }; - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId } }, { type: "task", value: { ...task, abortRequested: false } }, ], - () => {}, context, ); } @@ -117,7 +119,7 @@ describe("Harness open", () => { const reports: unknown[] = []; const opening = Harness.open( storage, - { models: createModels(), registry: createRegistry(), onReport: (error) => reports.push(error) }, + { models: createModels(), registry: createRegistry(), onReport: ({ error }) => reports.push(error) }, context, ); await expect(opening).rejects.toThrow("disk full"); @@ -167,7 +169,7 @@ describe("Harness close", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { if (where === "tool") await stubborn(); return { output: [] }; @@ -318,9 +320,9 @@ describe("Harness close", () => { expect(docWatch.value).toBe(docValue); expect(viewWatch.value).toBe(viewValue); expect(graphWatch.value).toEqual({ tasks: {} }); - expect(await graphWatch.closed).toEqual({ reason: "ended" }); - expect(await docWatch.closed).toEqual({ reason: "ended" }); - expect(await viewWatch.closed).toEqual({ reason: "ended" }); + expect(await graphWatch.closed).toEqual({ kind: "ended" }); + expect(await docWatch.closed).toEqual({ kind: "ended" }); + expect(await viewWatch.closed).toEqual({ kind: "ended" }); // The commit itself settled. expect(storage.commits.at(-1)).toContainEqual(expect.objectContaining({ type: "entry" })); }); @@ -341,8 +343,9 @@ describe("Harness close", () => { }, context); // Count commit subscriptions of document observers. let subscriptions = 0; - const subscribe = harness.subscribeCommits.bind(harness); - harness.subscribeCommits = (listener) => { + const session = harness as unknown as SessionImpl; + const subscribe = session.followCommits.bind(session); + session.followCommits = (listener) => { subscriptions++; const unsubscribe = subscribe(listener); let active = true; @@ -586,7 +589,7 @@ describe("registry changes before resume", () => { { state: { kind: "blocked", reason: "missing_task" } }, ]); const installed = addTask(first.registry, define("v1")); - expect((await first.harness.inspect(context)).tasks).toMatchObject([{ state: { kind: "ready" } }]); + expect((await first.harness.inspect(context)).tasks).toMatchObject([{ state: { kind: "pending" } }]); // The same extension name replaces the definition in place, still before resume. installed.dispose(); addTask(first.registry, define("v2")); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index df5593384..93ac8187c 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -1,5 +1,5 @@ import type { Op } from "@earendil-works/chord/delta"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CommitPublication, defineTool, @@ -10,10 +10,11 @@ import { type ToolExecutionApi, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; -import { context, documentChanges } from "./session-support.ts"; +import { context, documentChanges, documentValues, followCommits } from "./session-support.ts"; type Action = (api: ToolExecutionApi) => void | Promise; @@ -30,7 +31,7 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { defineTool({ name: "drive", description: "Driven by the test", - parameters: Type.Object({}), + parameters: obj({}), outputLimits, execute: async (_args, api, callContext) => { for (;;) { @@ -52,8 +53,8 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } }); @@ -104,7 +105,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -114,10 +115,13 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); @@ -128,36 +132,39 @@ describe("pi.live deltas", () => { const result = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; expect(commits).toEqual([ // submission - [["s", ["run"], { taskId: firstGeneration, inputs: [expect.any(Number)] }]], - // request - [["s", ["generation"], { attempt: 1 }]], - // the generation starts its tool round and keeps the run - expect.arrayContaining([ - ["d", ["generation"]], + [ + [ + "s", + ["run"], + { taskId: firstGeneration, inputs: [expect.any(Number)], turns: 0, first: expect.any(Number) }, + ], + ], + // the generation starts its tool round and keeps the run, counting the round as one turn (#11043); the request's + // attempt is deferred, so it is stored here, its held operation first + [ + ["s", ["generation"], { attempt: 1 }], ["s", ["tools"], [{ callId: "c1", name: "noop", taskId: tool, status: "pending" }]], - ]), - // intent - [["s", ["tools", 0, "status"], "running"]], - // result - expect.arrayContaining([ + ["d", ["generation"]], + ["s", ["run", "turns"], 1], + ], + // result: the running status is deferred like progress, so it is stored with the result, folded into its write + [ ["s", ["tools", 0, "status"], "done"], ["s", ["tools", 0, "entry"], result], - ]), + ], // the generation's tools phase hands the run to the next generation expect.arrayContaining([ ["d", ["tools"]], ["s", ["run", "taskId"], secondGeneration], ]), - [["s", ["generation"], { attempt: 1 }]], - // the answer ends the run - expect.arrayContaining([ + // the answer ends the run, with the second request's deferred attempt first + [ + ["s", ["generation"], { attempt: 1 }], ["d", ["run"]], ["d", ["generation"]], - ]), + ], ]); - expect(commits[2]).toHaveLength(2); - expect(commits[4]).toHaveLength(2); - expect(commits[5]).toHaveLength(2); + expect(commits[3]).toHaveLength(2); await harness.close(context); }); @@ -261,8 +268,8 @@ describe("pi.live deltas", () => { setup.faux.setResponses([fauxAssistantMessage([fauxText("word ".repeat(250))])]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } }); @@ -282,17 +289,26 @@ describe("pi.live deltas", () => { let liveId: number | undefined; class RecordingStorage extends MemoryStorage { override async commit( - writes: Parameters[0], - adopt: Parameters[1], + batches: Parameters[0], + receive: Parameters[1], context: Parameters[2], ) { - let kind: "base" | "delta" | undefined; - for (const write of writes) { - if (write.type === "document.change" && write.id === liveId) kind = write.content.kind; - } - const seq = await super.commit(writes, adopt, context); - if (kind !== undefined) written.set(seq, kind); - return seq; + const kinds = batches.map((writes) => { + let kind: "base" | "delta" | undefined; + for (const write of writes) { + if (write.type === "document.change" && write.id === liveId) kind = write.content.kind; + } + return kind; + }); + return super.commit( + batches, + (index, seq) => { + const kind = kinds[index]; + if (kind !== undefined) written.set(seq, kind); + receive(index, seq); + }, + context, + ); } } const setup = chatSetup(); @@ -302,7 +318,7 @@ describe("pi.live deltas", () => { defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { api.output(`${name} output\n`); await new Promise((resolve) => setTimeout(resolve, 150)); @@ -319,14 +335,19 @@ describe("pi.live deltas", () => { fauxAssistantMessage([fauxText("done")]), ]); setup.settings.toolExecution = "sequential"; + // Store deferred progress within each tool's run, so running slots reach Storage as deltas. + setup.settings.progress = { persistWithinMs: 50 }; const { harness, root } = await openChat(new RecordingStorage(), setup); const values = new Map(); - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind !== "pi.live") continue; - liveId = change.record.id; - if (change.value !== null) values.set(publication.seq, change.value as LiveState); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind !== "pi.live") continue; + liveId = change.record.id; + if (change.kind === "change") values.set(publication.seq, change.value as LiveState); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const nothingRuns = (value: LiveState) => @@ -341,14 +362,14 @@ describe("pi.live deltas", () => { await harness.close(context); }); - it("starts calls the request did not offer as done and marks a faulted tool's slot done without an entry", async () => { + it("starts calls the request did not offer as flagged tasks and marks a faulted tool's slot done without an entry", async () => { const setup = chatSetup(); addTool( setup.registry, defineTool({ name: "bad", description: "bad", - parameters: Type.Object({}), + parameters: obj({}), // Not strict JSON: the result commit throws and the scheduler faults the task. execute: async () => ({ output: [], details: { fn: (() => 1) as never } }), }), @@ -361,20 +382,22 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - const entries = await root.entries({}, 10, undefined, context); - const ghostResult = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; const handover = commits.find((ops) => ops.some((op) => op[0] === "s" && op[1][0] === "tools"))!; + // #11043: an unoffered call gets a task too, flagged on its slot. expect(handover).toContainEqual([ "s", ["tools"], [ - { callId: "g", name: "ghost", status: "done", entry: ghostResult }, + { callId: "g", name: "ghost", taskId: expect.any(Number), status: "pending", unoffered: true }, { callId: "b", name: "bad", taskId: expect.any(Number), status: "pending" }, ], ]); @@ -390,7 +413,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -402,12 +425,15 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); let handover: CommitPublication | undefined; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && (change.value as LiveState | null)?.tools?.length === 2) { - handover ??= publication; + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && (change.value as LiveState)?.tools?.length === 2) { + handover ??= publication; + } } - } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const changes = handover!.changes; @@ -445,7 +471,7 @@ describe("pi.live deltas", () => { const base = (value: LiveState) => LiveDoc.definition.checkpointWhen!(value, [], { deltasSinceBase: 1000 } as never); const slot = (status: "pending" | "running" | "done") => ({ callId: "c", name: "n", status }); - const run = { taskId: 1 as never, inputs: [] }; + const run = { taskId: 1 as never, inputs: [], turns: 0 }; expect(base({})).toBe(true); expect(base({ run, generation: { attempt: 1 } })).toBe(false); expect(base({ run, tools: [slot("pending"), slot("done")] })).toBe(true); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 342e103c0..8d80aa903 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -8,10 +8,8 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, defineExtension, defineTask, defineTool, @@ -23,16 +21,17 @@ import { ToolResultEntry, ToolTask, type ToolTaskInput, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; +import { nestedResultOf, nestedResults, viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); function tool( name: string, @@ -75,16 +74,6 @@ function results(entries: Awaited>): ToolResultMes return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); } -/** Collect tool end events from now on. */ -async function collectEnds(harness: Harness, conversationId: Parameters[0]) { - const ends: Extract[] = []; - const stream = await watchEvents(harness, conversationId, context); - stream.start(async (batch) => { - for (const event of batch) if (event.type === "tool_execution_end") ends.push(event); - }); - return { ends, stop: () => stream.stop() }; -} - function codes(result: NestedToolExecutionResult | undefined): string[] { return (result?.diagnostics ?? []).flatMap((diagnostic) => (diagnostic.code === undefined ? [] : [diagnostic.code])); } @@ -142,18 +131,19 @@ describe("nested tool calls across a restart", () => { await opened.harness.close(context); opened = await open(path, setup); - const { ends, stop } = await collectEnds(opened.harness, opened.root.id); + const history = await viewHistory(opened.harness, opened.root.id); + const nested = nestedResults(opened.harness); opened.harness.resume(); expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); await opened.harness.waitForIdle(context); - await stop(); + history.detach(); expect(runs).toEqual({ started: 1, gated: 0 }); const [started, gate] = await nestedTasks(opened.harness); for (const task of [started, gate]) { expect(task).toMatchObject({ abandonOnRestart: true, abortRequested: true, abortReason: "restart" }); expect(task!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); } - const end = (callId: string) => ends.find((event) => event.toolCallId === callId)?.result; + const end = (callId: string) => nestedResultOf(history.live(), nested, callId); expect(codes(end("c1/1"))).toEqual(["interrupted"]); expect(end("c1/1")).toMatchObject({ isError: true, details: { step: 1 } }); expect(end("c1/1")).not.toHaveProperty("structuredOutput"); @@ -497,15 +487,23 @@ describe("nested tool calls across a restart", () => { opened = await open(path, setup); let slotsAfterCaller: number | undefined; - opened.harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "task" || change.value.kind !== ToolTask.definition.name) continue; - if ((change.value.input as ToolTaskInput).kind !== "model" || change.value.state.status !== "terminal") - continue; - const live = publication.changes.find((each) => each.type === "document" && each.record.kind === "pi.live"); - const value = live?.type === "document" ? (live.value as { nestedTools?: unknown[] } | null) : undefined; - slotsAfterCaller = value?.nestedTools?.length ?? 0; - } + opened.harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type !== "task" || change.value.kind !== ToolTask.definition.name) continue; + if ((change.value.input as ToolTaskInput).kind !== "model" || change.value.state.status !== "terminal") + continue; + const live = publication.changes.find( + (each) => each.type === "document" && each.record.kind === "pi.live", + ); + const value = + live?.type === "document" && live.kind === "change" + ? (live.value as { nestedTools?: unknown[] }) + : undefined; + slotsAfterCaller = value?.nestedTools?.length ?? 0; + } + }, + end: () => {}, }); opened.harness.resume(); expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index 5d68eb34a..66de32766 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -3,22 +3,23 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { JsonValue } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; +import { applyImmutable } from "@earendil-works/chord/delta"; import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { - type AgentEvent, AssistantEntry, + type ConversationView, defineTool, type EntryRecord, type Harness, LiveDoc, + type LiveState, MemoryStorage, NestedCallDoc, type NestedToolExecutionResult, @@ -30,16 +31,18 @@ import { type ToolTaskInput, type ToolTaskResult, UsageDoc, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; -import { context } from "./session-support.ts"; +import { endedAt, nestedResultOf, nestedResults, slotsOf, startOrder, viewHistory } from "./live-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); type Tool = ToolRegistration; function tool(name: string, execute: Tool["execute"], extra: Partial = {}): Tool { @@ -115,11 +118,11 @@ describe("nested tool calls", () => { // Per document written, the keys of the nested calls it holds. const written: { kind: string; key: string | undefined }[] = []; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - for (const write of writes) { + storage.commit = (batches, receive, commitContext) => { + for (const write of batches.flat()) { if (write.type === "document.create") written.push({ kind: write.record.kind, key: write.record.key }); } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const { harness, root } = await openChat(storage, setup); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -326,7 +329,7 @@ describe("nested tool calls", () => { defineTool({ name: "pair", description: "pair", - parameters: Type.Object({ a: Type.String(), b: Type.String() }), + parameters: obj({ a: str(), b: str() }), execute: async (args) => { runs++; return { output: [{ type: "text", text: `${args.a}${args.b}` }] }; @@ -372,15 +375,12 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const ends: AgentEvent[] = []; - stream.start(async (batch) => { - ends.push(...batch.filter((event) => event.type === "tool_execution_end")); - }); + const history = await viewHistory(harness, root.id); + const nested = nestedResults(harness); const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(settled.status).toBe("done"); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); const [result] = results(await allEntries(root)); expect(text(result)).toBe("returned early"); const [, child] = await toolTasks(harness); @@ -389,16 +389,13 @@ describe("nested tool calls", () => { outcome: { status: "aborted", result: { kind: "nested" } }, }); // The aborted call's result keeps its details; programs get no partial output, which only the model read. - const childEnd = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - expect(childEnd).toMatchObject({ result: { isError: true, details: { step: 1 } } }); - expect((childEnd as { result?: NestedToolExecutionResult }).result).not.toHaveProperty("structuredOutput"); - // The nested call ends, with its result, before its caller. - expect(ends.map((event) => [event.type === "tool_execution_end" && event.toolCallId, "result" in event])).toEqual( - [ - ["c1/1", true], - ["c1", false], - ], - ); + const childResult = nestedResultOf(history.live(), nested, "c1/1"); + expect(childResult).toMatchObject({ isError: true, details: { step: 1 } }); + expect(childResult).not.toHaveProperty("structuredOutput"); + // The nested call ends before its caller. + const live = history.live(); + expect(endedAt(live, "c1/1")).toBeGreaterThan(0); + expect(endedAt(live, "c1/1")).toBeLessThan(endedAt(live, "c1")); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -429,7 +426,7 @@ describe("nested tool calls", () => { await harness.close(context); }); - it("reports nested calls in pi.live.nestedTools and as tool events with their parent", async () => { + it("reports nested calls in pi.live.nestedTools with their parent", async () => { const setup = chatSetup(); const release = deferred(); const running = deferred(); @@ -452,12 +449,9 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - expect(stream.snapshot.nestedTools).toEqual([]); - const events: AgentEvent[] = []; - stream.start(async (batch) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); + expect(history.live()[0]!.nestedTools).toBeUndefined(); + const nested = nestedResults(harness); const submission = await root.submit({ type: "input", content: "go" }, context); await running.promise; const live = await harness.snapshot(LiveDoc, root.id, context); @@ -477,32 +471,19 @@ describe("nested tool calls", () => { release.resolve(); await submission.wait(context); await harness.waitForIdle(context); - await stream.stop(); - const tools = events.flatMap((event) => - event.type === "tool_execution_start" || event.type === "tool_execution_end" - ? [[event.type, event.toolCallId, event.parentToolCallId]] - : [], - ); - expect(tools).toEqual([ - ["tool_execution_start", "c1", undefined], - ["tool_execution_start", "c1/1", "c1"], - ["tool_execution_end", "c1/1", "c1"], - ["tool_execution_end", "c1", undefined], - ]); - const nestedEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - expect(nestedEnd).toMatchObject({ result: { structuredOutput: "slow done", isError: false } }); - // Events name the call's task and, for a nested call, the calling one. + history.detach(); + const states = history.live(); + expect(startOrder(states)).toEqual(["c1", "c1/1"]); + expect(endedAt(states, "c1/1")).toBeLessThan(endedAt(states, "c1")); + expect(nestedResultOf(states, nested, "c1/1")).toMatchObject({ structuredOutput: "slow done", isError: false }); + // Slots name the call's task and, for a nested call, the calling one. const [parent, child] = await toolTasks(harness); - const parentEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1"); - expect(nestedEnd).toMatchObject({ taskId: child!.id, parentTaskId: parent!.id }); - expect(parentEnd).toMatchObject({ taskId: parent!.id }); - expect(parentEnd).not.toHaveProperty("parentTaskId"); - expect( - events.some( - (event) => - event.type === "tool_execution_update" && event.toolCallId === "c1/1" && event.parentToolCallId === "c1", - ), - ).toBe(true); + expect(slotsOf(states, "c1/1").find((slot) => slot !== undefined)).toMatchObject({ + taskId: child!.id, + parentCallId: "c1", + parentTaskId: parent!.id, + }); + expect(slotsOf(states, "c1").find((slot) => slot?.taskId !== undefined)).toMatchObject({ taskId: parent!.id }); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -535,7 +516,9 @@ describe("nested tool calls", () => { record!.state.status === "terminal" ? record!.state.outcome : undefined; expect(outcome(child)).toMatchObject({ status: "aborted", result: { kind: "nested" } }); expect(outcome(parent)).toMatchObject({ status: "aborted", result: { kind: "model" } }); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: expect.objectContaining({ reason: "aborted" }), + }); await harness.close(context); }); @@ -831,29 +814,19 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("outer"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const tools: (string | undefined)[][] = []; - stream.start(async (batch) => { - for (const event of batch) { - if (event.type === "tool_execution_start" || event.type === "tool_execution_end") { - tools.push([event.type, event.toolCallId, event.parentToolCallId]); - } - } - }); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); expect(text(results(await allEntries(root))[0])).toBe("echo deep"); const [outer, inner, deep] = await toolTasks(harness); expect([inner!.owner, deep!.owner]).toEqual([outer!.id, inner!.id]); - expect(tools).toEqual([ - ["tool_execution_start", "c1", undefined], - ["tool_execution_start", "c1/1", "c1"], - ["tool_execution_start", "c1/1/x", "c1/1"], - ["tool_execution_end", "c1/1/x", "c1/1"], - ["tool_execution_end", "c1/1", "c1"], - ["tool_execution_end", "c1", undefined], - ]); + const states = history.live(); + expect(startOrder(states)).toEqual(["c1", "c1/1", "c1/1/x"]); + expect(slotsOf(states, "c1/1/x").find((slot) => slot !== undefined)).toMatchObject({ parentCallId: "c1/1" }); + // Children end first. + expect(endedAt(states, "c1/1/x")).toBeLessThan(endedAt(states, "c1/1")); + expect(endedAt(states, "c1/1")).toBeLessThan(endedAt(states, "c1")); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -889,31 +862,33 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const submission = await root.submit({ type: "input", content: "go" }, context); await Promise.all([running.one.promise, running.two.promise]); - const stream = await watchEvents(harness, root.id, context); - expect(stream.snapshot.nestedTools.map((slot) => [slot.callId, slot.status])).toEqual([ + const watch = await root.watch(context); + const nestedOf = (view: ConversationView) => (view.docs["pi.live"] as LiveState).nestedTools ?? []; + expect(nestedOf(watch.value).map((slot) => [slot.callId, slot.status])).toEqual([ ["c1/1", "running"], ["c1/2", "running"], ]); - const updates: [string, unknown][] = []; - stream.start(async (batch) => { - for (const event of batch) { - if (event.type === "tool_execution_update" && event.output !== undefined) { - updates.push([event.toolCallId, event.output]); - } - } - }); + // Each value's operations rebuild the next one, the late subscriber's starting value included. + let replayed = watch.value; + const outputs: [string | undefined, string | undefined][] = []; + watch.start( + onValue(async ({ value, ops }) => { + replayed = applyImmutable(replayed, ops); + expect(replayed).toEqual(value); + const [one, two] = nestedOf(value); + outputs.push([one?.output, two?.output]); + }), + ); // The second slot writes first, so a wrong slot index in the update path would show. wrote.two.resolve(); - await eventually(() => updates.length === 1); + await eventually(() => outputs.at(-1)?.[1] === "two\n"); + expect(outputs.at(-1)).toEqual([undefined, "two\n"]); wrote.one.resolve(); - await eventually(() => updates.length === 2); + await eventually(() => outputs.at(-1)?.[0] === "one\n"); + expect(outputs.at(-1)).toEqual(["one\n", "two\n"]); release.resolve(); await submission.wait(context); - await stream.stop(); - expect(updates).toEqual([ - ["c1/2", { set: "two\n" }], - ["c1/1", { set: "one\n" }], - ]); + await watch.stop(); await harness.close(context); }); @@ -988,17 +963,13 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const ends: AgentEvent[] = []; - stream.start(async (batch) => { - ends.push(...batch.filter((event) => event.type === "tool_execution_end")); - }); + const history = await viewHistory(harness, root.id); + const nested = nestedResults(harness); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); // The abort's result is built from the slot, which never got the output. - const end = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - const result = (end as { result?: NestedToolExecutionResult }).result!; + const result = nestedResultOf(history.live(), nested, "c1/1")!; expect(result).toMatchObject({ isError: true }); expect(result).not.toHaveProperty("structuredOutput"); expect(result).not.toHaveProperty("details"); @@ -1038,11 +1009,6 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const starts: AgentEvent[] = []; - stream.start(async (batch) => { - starts.push(...batch.filter((event) => event.type === "tool_execution_start")); - }); const submission = await root.submit({ type: "input", content: "go" }, context); const slot = async () => (await harness.snapshot(LiveDoc, root.id, context))!.nestedTools![0]!; await hookEntered.promise; @@ -1053,10 +1019,7 @@ describe("nested tool calls", () => { release.resolve(); await submission.wait(context); await harness.waitForIdle(context); - await stream.stop(); - // The start event carries the arguments the call runs with; the task input keeps the ones it was made with. - const start = starts.find((event) => event.type === "tool_execution_start" && event.toolCallId === "c1/1"); - expect(start).toMatchObject({ args: { text: large } }); + // The slot showed the arguments the call runs with; the task input keeps the ones it was made with. const nested = (await toolTasks(harness)).find((task) => (task.input as ToolTaskInput).kind === "nested"); expect(nested!.input as ToolTaskInput).toMatchObject({ call: { arguments: { text: "asked" } } }); await harness.close(context); @@ -1271,14 +1234,34 @@ describe("nested tool calls", () => { checkpoint: { phase: "call" }, }); }); + + // #11043 + it("keeps version 2 tool task input as it is", () => { + const migrate = ToolTask.definition.migrate!; + const model = { kind: "model", assistant: 3, callId: "c1" }; + const nested = { + kind: "nested", + parent: 7, + parentCallId: "c1", + key: "k", + call: { type: "toolCall", id: "c1/k", name: "echo", arguments: {} }, + }; + expect(migrate(model, { phase: "call" }, 2).input).toEqual(model); + expect(migrate(nested, { phase: "call" }, 2).input).toEqual(nested); + }); }); -describe("tool task version 1 records", () => { +describe("tool task version 1 and 2 records", () => { /** - * Store a live tool task for call `c1` of an assistant entry, close, and rewrite its record as version 1 stored it: - * `{ assistant, callId }` input at version 1, with `checkpoint`. Returns the storage path and the task ID. + * Store a live tool task for call `c1` of an assistant entry, close, and rewrite its record as `version` stored it, + * with `checkpoint`: version 1 had `{ assistant, callId }` input, version 2 added `kind`. Returns the storage path + * and the task ID. */ - async function storeVersion1(setup: ChatSetup, checkpoint: JsonValue): Promise<{ path: string; id: TaskId }> { + async function storeVersion( + setup: ChatSetup, + checkpoint: JsonValue, + version: 1 | 2 = 1, + ): Promise<{ path: string; id: TaskId }> { const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-v1-")); directories.add(directory); const path = join(directory, "session.sqlite"); @@ -1295,11 +1278,12 @@ describe("tool task version 1 records", () => { await harness.close(context); const storage = await openNodeSqliteStorage(path); const record = (await storage.task(id, context))!; - const { kind: _kind, ...input } = record.input as Extract; + const { kind: _kind, ...version1 } = record.input as Extract; + const input = version === 1 ? version1 : record.input; const state = { ...record.state, checkpoint } as typeof record.state; - await storage.commit( - [{ type: "task", value: { ...record, version: 1, input, state } as typeof record }], - () => {}, + await commitWrites( + storage, + [{ type: "task", value: { ...record, version, input, state } as typeof record }], context, ); await storage.close(context); @@ -1310,12 +1294,12 @@ describe("tool task version 1 records", () => { const setup = chatSetup(); const echo = echoTool(); addTool(setup.registry, echo.registration); - const { path, id } = await storeVersion1(setup, { phase: "call" }); + const { path, id } = await storeVersion(setup, { phase: "call" }); const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); expect((await harness.getTask(id, context))?.version).toBe(1); harness.resume(); const settled = await harness.waitForTask(id, context); - expect(settled.version).toBe(2); + expect(settled.version).toBe(3); expect(settled.input).toMatchObject({ kind: "model", callId: "c1" }); expect(settled.state.outcome).toMatchObject({ status: "completed", result: { kind: "model" } }); expect(echo.state.runs).toBe(1); @@ -1329,11 +1313,11 @@ describe("tool task version 1 records", () => { const echo = echoTool(); addTool(setup.registry, echo.registration); const intent = { phase: "execute", arguments: { text: "old" }, replay: "unsafe" }; - const { path, id } = await storeVersion1(setup, intent); + const { path, id } = await storeVersion(setup, intent); const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); harness.resume(); const settled = await harness.waitForTask(id, context); - expect(settled.version).toBe(2); + expect(settled.version).toBe(3); expect(settled.state.outcome).toMatchObject({ status: "failed", result: { kind: "model" } }); expect(echo.state.runs).toBe(0); const [result] = results(await allEntries(root)); @@ -1341,4 +1325,23 @@ describe("tool task version 1 records", () => { expect(text(result)).toContain("Tool echo was interrupted"); await harness.close(context); }); + + // #11043 + it("reserves a stored version 2 task under version 3 and runs it as a model-issued call", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const { path, id } = await storeVersion(setup, { phase: "call" }, 2); + const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); + expect((await harness.getTask(id, context))?.version).toBe(2); + harness.resume(); + const settled = await harness.waitForTask(id, context); + expect(settled.version).toBe(3); + expect(settled.input).toEqual({ kind: "model", assistant: expect.any(Number), callId: "c1" }); + expect(settled.state.outcome).toMatchObject({ status: "completed", result: { kind: "model" } }); + expect(echo.state.runs).toBe(1); + const [result] = results(await allEntries(root)); + expect([result!.toolCallId, text(result)]).toEqual(["c1", "echo old"]); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts new file mode 100644 index 000000000..13b8cbf14 --- /dev/null +++ b/packages/durable/test/harness-observers.test.ts @@ -0,0 +1,469 @@ +// Tests for the Harness's synchronous observers (`observeConversation()`, `observeTaskGraph()`, `observeDoc()` of a task +// document) and for task document watches, which end with their task. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchCall } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type ConversationView, + defineDoc, + defineExtension, + defineTask, + type Observer, + type TaskGraph, + type TaskId, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import type { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; +import { completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +type Step = { phase: "run" }; + +const Progress = defineDoc<{ lines: string[] }>({ + checkpointWhen: () => false, + kind: "test.observers.progress", + version: 1, + scope: "task", + initial: () => ({ lines: [] }), +}); + +/** A task that writes its progress document, waits for `gate`, then completes, which retires the document. */ +function writer(name: string, gate: Promise, written: () => void) { + return defineTask({ + name, + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + (await tx.doc(Progress, task.id)).lines.push("started"); + }, ctx); + written(); + await gate; + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); +} + +type WatchValue = Extract, { readonly kind: "value" }>; + +/** An observer recording each call and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array<{ readonly advance: WatchValue; readonly adopting: boolean } | { readonly end: unknown }> = []; + const observer: Observer = { + advance: (call) => { + // Durable never sends a failure call. + if (call.kind !== "value") throw new Error("Unexpected failure call"); + calls.push({ advance: call, adopting: storage.adopting }); + }, + end: (end) => { + calls.push({ end }); + }, + }; + return { calls, observer }; +} + +/** A Harness with a writer task started in the root conversation and its progress document written. */ +async function openWriter() { + const storage = new ControlledStorage(); + const gate = deferred(); + const written = deferred(); + const Writer = writer("test.observers.writer", gate.promise, written.resolve); + const opened = await openTasks(storage, [Writer]); + const root = await opened.harness.root(context); + const id: TaskId = await root.commit( + (tx) => tx.createTask(Writer, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + await written.promise; + return { ...opened, storage, root, id, finish: gate.resolve }; +} + +const collect = (() => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + return async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; +})(); + +describe("observeConversation and observeTaskGraph", () => { + it("start at the current value and advance at the publication with the same change a watch gets", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const viewObservation = await harness.observeConversation(root.id, view.observer, context); + const graph = recorder(storage); + const graphObservation = await harness.observeTaskGraph(graph.observer, context); + expect(viewObservation.value.conversation.id).toBe(root.id); + expect(graphObservation.value).toEqual({ tasks: {} }); + const watch = await root.watch(context); + const watched: WatchCall[] = []; + watch.start(async (change) => { + watched.push(change); + }); + const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + const [call] = view.calls as [{ advance: WatchValue; adopting: boolean }]; + // Published once Storage stored it, outside its call. + expect(call.adopting).toBe(false); + expect(call.advance.retired).toBe(false); + expect(call.advance.value.entries.at(-1)?.id).toBe(entry.id); + await flush(); + expect(watched).toEqual([call.advance]); + // No task changed: the graph got nothing. + expect(graph.calls).toEqual([]); + viewObservation.detach(); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + await watch.stop(); + const graphWatch = await harness.watchTaskGraph(context); + await harness.close(context); + // S1: a Session close fails an observer (it may follow the next Session); a watch ends `ended`. + expect(graph.calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await graphWatch.closed).toEqual({ kind: "ended" }); + }); + + it("end with the error that failed the Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const graph = recorder(storage); + await harness.observeTaskGraph(graph.observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + await harness.closed; + expect(graph.calls).toEqual([{ end: { kind: "failed", error: failure } }]); + }); + + it("report a throwing observer, detach it and end it with the error, and keep the view going", async () => { + const storage = new ControlledStorage(); + const { harness, reports, infos } = await openTasks(storage, []); + const root = await harness.root(context); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + await harness.observeConversation( + root.id, + { + advance: () => { + throw bug; + }, + end: (end) => ends.push(end), + }, + context, + ); + const healthy = recorder(storage); + await harness.observeConversation(root.id, healthy.observer, context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ kind: "listener_error" }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(healthy.calls).toHaveLength(2); + await harness.close(context); + expect(ends).toHaveLength(1); + expect(await harness.closed).toEqual({ kind: "closed" }); + }); + + it("reject, never throw, on a closed Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + await harness.close(context); + const view = harness.observeConversation(root.id, recorder(storage).observer, context); + const graph = harness.observeTaskGraph(recorder(storage).observer, context); + await expect(view).rejects.toThrow("Session is closed"); + await expect(graph).rejects.toThrow("Session is closed"); + }); + + it("reject an attachment cancelled while it waits on the line, and attach nothing", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const graph = recorder(storage); + // A commit holds the line, so both attachments wait behind it. + const held = storage.holdCommits(); + const committing = root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await held.entered; + const controller = new AbortController(); + const cancelled = withAbortSignal(controller.signal, context); + const attachingView = harness.observeConversation(root.id, view.observer, cancelled); + const attachingGraph = harness.observeTaskGraph(graph.observer, cancelled); + controller.abort(new Error("gone")); + held.release(); + await committing; + await expect(attachingView).rejects.toThrow("gone"); + await expect(attachingGraph).rejects.toThrow("gone"); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await harness.close(context); + expect(view.calls).toEqual([]); + expect(graph.calls).toEqual([]); + }); + + it("advance a graph observer with a blocked refresh, outside any commit", async () => { + const Missing = defineTask({ + name: "test.observers.missing", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: (_task, runtime, ctx) => runtime.commit(() => completed(null), ctx) }, + abort: async () => {}, + }); + const storage = new ControlledStorage(); + const { harness, registry } = await openTasks(storage, []); + const graph = recorder(storage); + const observation = await harness.observeTaskGraph(graph.observer, context); + const root = await harness.root(context); + // Scheduling is not resumed: the task stays pending, blocked while its definition is missing. + const id = await root.commit( + (tx) => tx.createTask(Missing, null, { ownership: { kind: "conversation" } }), + context, + ); + const key = String(id); + expect(graph.calls).toEqual([ + { + advance: { + kind: "value", + value: { tasks: { [key]: expect.objectContaining({ blocked: "missing_task" }) } }, + ops: [["s", ["tasks", key], expect.objectContaining({ blocked: "missing_task" })]], + retired: false, + }, + adopting: false, + }, + ]); + registry.install(defineExtension({ name: "late", tasks: [Missing] })); + await eventually(() => graph.calls.length === 2); + // The registry change re-derives the reason on the line, with no commit and outside the storage's `adopt`. + expect(graph.calls[1]).toEqual({ + advance: { kind: "value", value: expect.anything(), ops: [["d", ["tasks", key, "blocked"]]], retired: false }, + adopting: false, + }); + expect((graph.calls[1] as { advance: WatchValue }).advance.value.tasks[key]?.blocked).toBeUndefined(); + harness.resume(); + await harness.waitForTask(id, context); + observation.detach(); + await harness.close(context); + }); + + it("keep no detached observer alive", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + /** Attach observers only the Harness references; `detach` says whether to detach them before returning. */ + const attach = async (detach: boolean) => { + const view: Observer = { advance: () => {}, end: () => {} }; + const graph: Observer = { advance: () => {}, end: () => {} }; + const observations = [ + await harness.observeConversation(root.id, view, context), + await harness.observeTaskGraph(graph, context), + ]; + if (detach) for (const observation of observations) observation.detach(); + return { observations, view: new WeakRef(view), graph: new WeakRef(graph) }; + }; + // Control: attached observers stay, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.view.deref()).toBeDefined(); + expect(attached.graph.deref()).toBeDefined(); + for (const observation of attached.observations) observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.view.deref()).toBeUndefined(); + expect(detached.graph.deref()).toBeUndefined(); + await harness.close(context); + }); +}); + +describe("task documents", () => { + it("a watch delivers the retirement of the commit that ends its task, then ends `ended`", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + expect(watch.value).toEqual({ lines: ["started"] }); + const changes: WatchCall>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(changes).toEqual([{ kind: "value", value: { lines: ["started"] }, ops: [], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(watch.value); + // The task is terminal: its address is absent for good. + expect(await harness.watchDoc(Progress, id, context)).toBeUndefined(); + await harness.close(context); + }); + + it("an observer gets the retirement, then its end, at the publication of the commit that ends the task", async () => { + const { harness, storage, id, finish } = await openWriter(); + const progress = recorder>(storage); + let endedAdopting: boolean | undefined; + const observer: Observer> = { + advance: progress.observer.advance, + end: (end) => { + endedAdopting = storage.adopting; + progress.observer.end(end); + }, + }; + await harness.observeDoc(Progress, id, observer, context); + finish(); + await harness.waitForTask(id, context); + expect(progress.calls).toEqual([ + { advance: { kind: "value", value: { lines: ["started"] }, ops: [], retired: true }, adopting: false }, + { end: { kind: "ended" } }, + ]); + expect(endedAdopting).toBe(false); + await harness.close(context); + expect(progress.calls).toHaveLength(2); + }); + + it("keeps no task document watch alive once its task ended", async () => { + const { harness, id, finish } = await openWriter(); + /** A started watch of the task's document, which only the Harness references. */ + const acquire = async (): Promise> => { + const watch = (await harness.watchDoc(Progress, id, context))!; + watch.start(async () => {}); + return new WeakRef(watch); + }; + const watch = await acquire(); + // Control: the watch stays while its task runs, so the collection below is real. + await collect(); + expect(watch.deref()).toBeDefined(); + finish(); + await harness.waitForTask(id, context); + await collect(); + expect(watch.deref()).toBeUndefined(); + // The Harness is the Session kernel; nothing stays pinned for the ended watch. + expect((harness as unknown as SessionImpl).documentCacheSizes.attached).toBe(0); + await harness.close(context); + }); + + it("a watch never started when its task ends still ends with the Harness's close, keeping nothing of it", async () => { + /** Acquire a task document's watch, never start it, end its task, and close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + // Its retirement is pending until a start that never comes: the watch is open. + await flush(); + expect(await settled(watch.closed)).toBe(false); + await harness.close(context); + return { watch, harness: new WeakRef(harness) }; + }; + const { watch, harness } = await acquire(); + // Closed at the Harness's close, the pending retirement dropped, as for every watch. + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + await collect(); + expect(harness.deref()).toBeUndefined(); + }); + + it("a watch never started when its task ends fails with the Harness", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); + }); + + it("a watch busy in an earlier call when its task ends fails with the Harness before its retirement call", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + const values: unknown[] = []; + watch.start( + onValue(async ({ value }) => { + values.push(value); + entered.resolve(); + await release.promise; + }), + ); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await entered.promise; + // The task ends while the listener is busy: its retirement waits behind the running call. + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); + release.resolve(); + await flush(); + expect(values).toEqual([{ lines: ["started", "more"] }]); + }); + + it("a watch whose retirement call runs when its task ends ends only after that call returns", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + let returned = false; + watch.start(async () => { + entered.resolve(); + await release.promise; + returned = true; + }); + // An explicit retirement first: its call runs while the task ends, and nothing more is pending. + await harness.commit((tx) => tx.retireDoc(Progress, id), context); + await entered.promise; + finish(); + await harness.waitForTask(id, context); + await flush(); + expect(await settled(watch.closed)).toBe(false); + release.resolve(); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(returned).toBe(true); + await harness.close(context); + }); + + it("hands listeners changes that keep nothing of the Harness", async () => { + /** Watch a task document until its task ends, keeping every change, then close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const changes: WatchCall>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await flush(); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ kind: "ended" }); + await harness.close(context); + return { changes, harness: new WeakRef(harness) }; + }; + const { changes, harness } = await acquire(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["kind", "value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + } + await collect(); + expect(harness.deref()).toBeUndefined(); + }); +}); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index d363f9c76..d0251e24c 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationHandle, @@ -16,14 +16,16 @@ import { LiveDoc, MemoryStorage, type Storage, + type StorageWrite, type Submission, type TaskId, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, refusingAt } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; /** Outcome a held task commits once its gate opens. */ @@ -174,7 +176,7 @@ async function waitUntil(check: () => Promise): Promise { /** Mark a conversation busy with `task` standing in for its run, so submissions queue. */ async function busy(conversation: Conversation, task: TaskId): Promise { await conversation.commit(async (tx) => { - (await tx.doc(LiveDoc, conversation.id)).run = { taskId: task, inputs: [] }; + (await tx.doc(LiveDoc, conversation.id)).run = { taskId: task, inputs: [], turns: 0 }; }, context); } @@ -937,18 +939,25 @@ describe("ownership", () => { await opened.harness.abortTask(tree.owner, context); await opened.harness.close(context); - const storage = await openNodeSqliteStorage(path); + const storage: Storage = await openNodeSqliteStorage(path); let rejectMark: TaskId | undefined; const commit = storage.commit.bind(storage); - storage.commit = async (writes, adopt, ctx) => { - if ( - writes.some((write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested) - ) { - rejectMark = undefined; - throw new Error("disk gone"); - } - return commit(writes, adopt, ctx); - }; + // The batch that marks `rejectMark` is refused. + storage.commit = (batches, receive, ctx) => + refusingAt( + commit, + batches, + receive, + ctx, + (writes) => { + const marks = writes.some( + (write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested, + ); + if (marks) rejectMark = undefined; + return marks; + }, + new Error("disk gone"), + ); opened = await openHarness(storage); const child = (await opened.harness.conversation(tree.child, context))!; const late = await child.commit(async (tx) => { @@ -959,7 +968,7 @@ describe("ownership", () => { // The failed cascade commit fails the Harness. `closed` settles once its invocations have ended, and the owner's // abort handler ignores its signal: let it return. open("abort.owner", "completed"); - expect(await opened.harness.closed).toMatchObject({ reason: "failed" }); + expect(await opened.harness.closed).toMatchObject({ kind: "failed" }); await opened.harness.close(context); // Reopening derives the mark again. opened = await openHarness(await openNodeSqliteStorage(path)); @@ -971,19 +980,19 @@ describe("ownership", () => { const failure = new Error("disk gone"); let failMark: TaskId | undefined; class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const marks = writes.some( - (write) => write.type === "task" && write.value.id === failMark && write.value.abortRequested, - ); - if (marks) { - failMark = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const marks = writes.some( + (write) => write.type === "task" && write.value.id === failMark && write.value.abortRequested, + ); + if (marks) failMark = undefined; + return marks; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -991,7 +1000,7 @@ describe("ownership", () => { const tree = await ownedChild(root, "owner"); failMark = tree.inner; open("owner", "failed"); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); const reopened = await openHarness(storage.reopen()); @@ -1011,7 +1020,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1056,7 +1065,7 @@ describe("owned conversations from tools and supervisors", () => { await harness.close(context); }); - it("rejects a handle operation queued on the line when the invocation ends before it runs", async () => { + it("runs a handle operation queued behind the abort mark before the mark's publication, and the cascade aborts it", async () => { const setup = chatSetup(); const ready = deferred<{ child: ConversationId; taskId: TaskId }>(); const go = deferred(); @@ -1066,7 +1075,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates late", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1087,9 +1096,29 @@ describe("owned conversations from tools and supervisors", () => { setup.faux.setResponses([ fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }), ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); + // Holds the storage call that writes the abort mark, so the mark is published only once the test lets it. + let holdMark: TaskId | undefined; + const stored = deferred(); + class HoldMark extends MemoryStorage { + override async commit( + batches: Parameters[0], + receive: Parameters[1], + ctx: Parameters[2], + ) { + const marks = batches + .flat() + .some((write) => write.type === "task" && write.value.id === holdMark && write.value.abortRequested); + if (marks) { + holdMark = undefined; + await stored.promise; + } + return super.commit(batches, receive, ctx); + } + } + const { harness, root } = await openChat(new HoldMark(), setup); await root.submit({ type: "input", content: "go" }, context); const { child, taskId } = await ready.promise; + holdMark = taskId; // Hold the Session line, queue the abort mark behind it, then let the tool queue its submit. const release = deferred(); const holding = root.commit(async () => { @@ -1100,10 +1129,22 @@ describe("owned conversations from tools and supervisors", () => { const { submitting } = await queued.promise; release.resolve(); await holding; + // The submit starts while the mark's storage call is in flight. + await new Promise((resolve) => setTimeout(resolve, 0)); + stored.resolve(); await aborting; - await expect(submitting).rejects.toThrow(); + // Cancellation is checked when a job starts: the submit touches nothing the mark writes, so it starts before the + // mark is published and lands. The mark's cascade then aborts the run it started in the owned conversation. + const submission = (await submitting) as Submission; const childConversation = (await harness.conversation(child, context))!; - expect((await childConversation.entries({}, 10, undefined, context)).items).toEqual([]); + await childConversation.waitForIdle(context); + expect(await (await harness.submission(submission.id, context))!.status(context)).toMatchObject({ + status: "unanswered", + reason: "aborted", + }); + expect((await childConversation.entries({}, 10, undefined, context)).items.map((entry) => entry.kind)).toEqual([ + "pi.user", + ]); expect((await harness.inspect(context)).submissions.filter((record) => record.conversationId === child)).toEqual( [], ); @@ -1119,7 +1160,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1153,7 +1194,7 @@ describe("owned conversations from tools and supervisors", () => { const childConversation = (await harness.conversation(child!, context))!; await childConversation.waitForIdle(context); const childLive = await harness.snapshot(LiveDoc, child!, context); - expect(childLive).toEqual({}); + expect(childLive).toEqual({ runFailure: expect.objectContaining({ reason: "aborted" }) }); const childInputs = (await harness.inspect(context)).submissions.filter((s) => s.conversationId === child); expect(childInputs).toEqual([]); await harness.waitForIdle(context); @@ -1168,7 +1209,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates and cancels", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1207,7 +1248,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "spawn", description: "Starts owned work, then throws or hangs", - parameters: Type.Object({ mode: Type.String() }), + parameters: obj({ mode: str() }), execute: async (args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1265,7 +1306,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "subagent", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { diff --git a/packages/durable/test/harness-partial-changes.test.ts b/packages/durable/test/harness-partial-changes.test.ts index 3b61133b7..4e2ebf78d 100644 --- a/packages/durable/test/harness-partial-changes.test.ts +++ b/packages/durable/test/harness-partial-changes.test.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "@earendil-works/chord"; +import { copyJson } from "@earendil-works/chord"; import { type Op, track } from "@earendil-works/chord/delta"; import { type AssistantMessage, @@ -7,11 +7,12 @@ import { ToolCallArgumentsStream, } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; -import { PartialChanges } from "../src/harness/partial-changes.ts"; +import type { LiveState } from "../src/harness/live.ts"; +import { PartialChanges, type PartialMessage } from "../src/harness/partial-changes.ts"; // #9265: partial commits reassigned the growing strings of a streamed answer, and Chord compared each one whole. -type Live = { generation: Record }; +type Live = { generation: NonNullable }; /** * A response streaming one text block (started empty) and then one tool call, as a provider produces it: the partial @@ -63,10 +64,11 @@ function setup() { }, /** One partial commit, as the generation's throttle makes it; `fail` aborts it like a rejected commit. */ commit(fail = false) { - const message = structuredClone(partial) as unknown as { content: JsonValue[] }; + // As the generation copies it: `copyJson()` returns untyped JSON, here a message's JSON representation. + const message = copyJson(partial) as PartialMessage; const taken = changes.take(message); const change = tracker.beginChange(); - changes.commit(change.state.generation, message as unknown as JsonValue, taken); + changes.commit(change.state.generation, message, taken); if (fail) { change.abort(); changes.restore(taken); diff --git a/packages/durable/test/harness-prompt.test.ts b/packages/durable/test/harness-prompt.test.ts index 40ef2e09e..39affbc58 100644 --- a/packages/durable/test/harness-prompt.test.ts +++ b/packages/durable/test/harness-prompt.test.ts @@ -1,4 +1,4 @@ -import { type SystemMessage, type Tool, Type } from "@earendil-works/pi-ai"; +import type { SystemMessage, Tool } from "@earendil-works/pi-ai"; import { getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { type Conversation, @@ -12,6 +12,7 @@ import { type ToolRegistration, wrapSection, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { planSystemEntries, renderSections, replaySections } from "../src/harness/prompt.ts"; @@ -207,7 +208,7 @@ describe("tool loadout preparation", () => { const declaration = (name: string, description = name): Tool => ({ name, description, - parameters: Type.Object({}), + parameters: obj({}), }); /** Plan tools only, append the plan, check that replay offers `tools` in order, and return each message's changes. */ diff --git a/packages/durable/test/harness-read-only.test.ts b/packages/durable/test/harness-read-only.test.ts index da6539789..b45b14b0f 100644 --- a/packages/durable/test/harness-read-only.test.ts +++ b/packages/durable/test/harness-read-only.test.ts @@ -30,7 +30,7 @@ async function openReadOnly(path: string, setup: ChatSetup, onWake?: (wake: Wake models: setup.models, registry: setup.registry, readOnly: true, - onReport: (error) => setup.reports.push(error), + onReport: ({ error }) => setup.reports.push(error), ...(onWake === undefined ? {} : { onWake }), }, context, diff --git a/packages/durable/test/harness-registry-reader.test.ts b/packages/durable/test/harness-registry-reader.test.ts new file mode 100644 index 000000000..b2a82da35 --- /dev/null +++ b/packages/durable/test/harness-registry-reader.test.ts @@ -0,0 +1,198 @@ +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + type FacetHost, + local, + method, + type Native, + type RegistryTable, + str, +} from "@earendil-works/chord/next"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { describe, expect, it } from "vitest"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + MemoryStorage, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../src/index.ts"; + +/** p3's pattern: the extensions registry composes Durable's snapshot, leaving out an extension Durable refuses. */ +const Extensions = defineRegistry("app.harness.extensions", { version: 1 }); + +function compose(table: RegistryTable, report: (conflict: { message: string; plugins: string[] }) => void) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Done = defineTask({ + name: "app.done", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "completed", result: null } }), + taskContext, + ); + }, + }, + abort: async () => {}, +}); + +/** A host-lifetime object contributing an extension it replaces with `install` (one with the task `app.done`). */ +const Plugin = local("app.plugin", { version: 1, params: {}, members: { install: method() } }); + +function serve(host: FacetHost): void { + host.serve(Extensions, { compose: (table, { report }) => compose(table, report) }); + host.serve(Plugin, (_, self) => { + const extension = self.writable(defineExtension({ name: "app.plugin" })); + self.contribute(Extensions, extension); + return { + install: async () => { + extension.set(defineExtension({ name: "app.plugin", tasks: [Done] })); + return null; + }, + }; + }); +} + +describe("registryReader", () => { + it("reads the composed snapshot; a listener is called synchronously, inside the batch that composed", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const value = await host.registry(Extensions, context); + const reader = registryReader(value, context); + expect(reader.snapshot().extension("app.plugin")?.tasks).toBeUndefined(); + const calls: string[] = []; + const unsubscribe = reader.subscribe(() => { + // Inside Chord's batch: a Chord call here is refused, so a listener only wakes its scheduler. + try { + value.observe({ advance: () => {}, end: () => {} }, context); + calls.push("outside"); + } catch (error) { + calls.push((error as { reason?: string }).reason ?? "other"); + } + }); + const plugin = await host.get(Plugin, {}, context); + await plugin.install(context); + expect(calls).toEqual(["reentrant"]); + expect(reader.snapshot().task("app.done")).toBe(Done); + unsubscribe(); + await host.close(context); + }); + + it("wakes a Harness's scheduler: a task blocked on its missing definition runs once a contribution brings it", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const harness = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: registryReader(await host.registry(Extensions, context), context) }, + context, + ); + harness.resume(); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Done, undefined, { ownership: { kind: "conversation" } }), + context, + ); + const done = harness.waitForTask(id, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await (await host.get(Plugin, {}, context)).install(context); + expect((await done).state.outcome).toEqual({ status: "completed", result: null }); + await harness.close(context); + await host.close(context); + }); +}); + +/** p3's pattern for Durable's views from other objects: the object that opens Durable contributes its Harness. */ +const Harnesses = defineRegistry, Harness | undefined>("app.harnesses", { version: 1 }); + +/** Opens a Harness (a new one per attempt), contributes it, and follows its task graph: its Session closing fails it. */ +const Durable = local("app.durable", { version: 1, params: {}, members: { close: method(), write: method(str()) } }); + +/** Follows the root conversation's entry kinds through the contributed Harness, from another object. */ +const Viewer = local("app.viewer", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +function serveViews(host: FacetHost): void { + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Durable, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); + host.serve(Viewer, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value === undefined + ? undefined + : harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +} + +describe("Durable's views from other objects (slice 8 §5, S1)", () => { + it("a follower of a contributed Harness fails when that Harness's Session closes, and follows the next one", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serveViews) }], context); + const durable = host.hold(Durable, {}, context); + const viewer = host.hold(Viewer, {}, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await durable.write("app.first", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.first"); + // The Session closes: both followers fail `transient` (S1); a retry opens the next Harness, whose contribution + // wakes the viewer (`retryOn`), which follows it. + // The call is cut off by its object's failure: the Session's close fails the object that opened it. + await expect(durable.close(context)).rejects.toThrow("session closed"); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + durable.retry(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(viewer.state.value).toEqual({ kind: "live" }); + expect(viewer.kinds.value).not.toContain("app.first"); + await durable.write("app.second", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.second"); + await host.close(context); + }); +}); diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index bf2edf002..b51002576 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -1,4 +1,3 @@ -import { Type } from "@earendil-works/pi-ai"; import { type AgentState, CompactionTask, @@ -15,6 +14,7 @@ import { wrapSection, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentHooks, resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { context } from "./session-support.ts"; @@ -25,7 +25,7 @@ function tool(name: string, extra: Partial = {}): AppTool { return { name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), ...extra, }; @@ -142,6 +142,7 @@ describe("settings", () => { expect(resolveSettings({ progress: { partialIntervalMs: undefined, outputIntervalMs: 250 } }).progress).toEqual({ partialIntervalMs: 100, outputIntervalMs: 250, + persistWithinMs: 1000, }); }); }); diff --git a/packages/durable/test/harness-reports.test.ts b/packages/durable/test/harness-reports.test.ts index 0bc2304cd..97a7acf51 100644 --- a/packages/durable/test/harness-reports.test.ts +++ b/packages/durable/test/harness-reports.test.ts @@ -1,4 +1,4 @@ -import { createModels, fauxAssistantMessage, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -14,6 +14,7 @@ import { section, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup } from "./chat-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; @@ -34,7 +35,7 @@ async function open( { models: createModels(), registry, - onReport: (error, info) => reports.push({ message: (error as Error).message, info }), + onReport: ({ error, ...info }) => reports.push({ message: (error as Error).message, info }), ...(options.onWake === undefined ? {} : { onWake: options.onWake }), ...(options.setTimer === undefined ? {} : { setTimer: options.setTimer }), }, @@ -69,8 +70,8 @@ describe("reports", () => { // The task's own failures name the extension that installed it. const task = { task: { id, kind: "test.faulty", conversationId: root.id }, extension: "test" }; expect(reports).toEqual([ - { message: "noted", info: { type: "task_error", ...task } }, - { message: "broken", info: { type: "task_faulted", ...task, reason: "threw" } }, + { message: "noted", info: { kind: "task_error", ...task } }, + { message: "broken", info: { kind: "task_faulted", ...task, reason: "threw" } }, ]); await harness.close(context); }); @@ -104,7 +105,7 @@ describe("reports", () => { registry.install(defineExtension({ name: "new-extension", tasks: [definition("new")] })); gate.resolve(); await harness.waitForTask(id, context); - const faulted = (report: Report) => report.info.type === "task_faulted"; + const faulted = (report: Report) => report.info.kind === "task_faulted"; await eventually(() => reports.some(faulted)); expect(reports.find(faulted)).toMatchObject({ message: "broken in old", @@ -147,7 +148,7 @@ describe("reports", () => { env: () => { throw new Error("env"); }, - onReport: (error, info) => reports.push({ message: (error as Error).message, info }), + onReport: ({ error, ...info }) => reports.push({ message: (error as Error).message, info }), }, context, ); @@ -155,9 +156,9 @@ describe("reports", () => { await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); const infoOf = (message: string) => reports.find((report) => report.message === message)?.info; const generation = { kind: "pi.generation", conversationId: root.id }; - expect(infoOf("env")).toMatchObject({ type: "host_error", callback: "env", task: generation }); - expect(infoOf("section")).toMatchObject({ type: "extension_error", extension: "sections", task: generation }); - expect(infoOf("hook")).toMatchObject({ type: "extension_error", extension: "hooks", task: generation }); + expect(infoOf("env")).toMatchObject({ kind: "host_error", callback: "env", task: generation }); + expect(infoOf("section")).toMatchObject({ kind: "extension_error", extension: "sections", task: generation }); + expect(infoOf("hook")).toMatchObject({ kind: "extension_error", extension: "hooks", task: generation }); await harness.close(context); }); @@ -189,15 +190,18 @@ describe("reports", () => { const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }); registry.install(defineExtension({ name: "tools", tools: [echo] })); registry.install( defineExtension({ name: "wraps", wraps: [wrapTool(echo, (tool) => ({ ...tool, name: "renamed" }))] }), ); - harness.subscribeCommits(() => { - throw new Error("listener"); + harness.observeCommits({ + advance: () => { + throw new Error("listener"); + }, + end: () => {}, }); const root = await harness.root(context); await root.agent(context); @@ -208,18 +212,18 @@ describe("reports", () => { ); const infoOf = (message: string) => reports.find((report) => report.message === message)?.info; expect(infoOf("Wrapper renamed echo to renamed")).toEqual({ - type: "extension_error", + kind: "extension_error", conversationId: root.id, extension: "wraps", }); - expect(infoOf("listener")).toEqual({ type: "listener_error" }); - expect(infoOf("alarm")).toEqual({ type: "host_error", callback: "onWake" }); - expect(infoOf("no timers")).toEqual({ type: "host_error", callback: "setTimer" }); + expect(infoOf("listener")).toEqual({ kind: "listener_error" }); + expect(infoOf("alarm")).toEqual({ kind: "host_error", callback: "onWake" }); + expect(infoOf("no timers")).toEqual({ kind: "host_error", callback: "setTimer" }); const failure = new Error("disk gone"); storage.failNextCommit(failure); await root.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context).catch(() => {}); await harness.closed; - expect(infoOf("disk gone")).toEqual({ type: "session_failed" }); + expect(infoOf("disk gone")).toEqual({ kind: "storage_failed" }); }); }); diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts new file mode 100644 index 000000000..c8ece30c7 --- /dev/null +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -0,0 +1,780 @@ +// What a crash (a host that dies, is evicted, or is reset) does to work that was running: a generation's interrupted +// request is a failed attempt under the retry policy, and every other task ends visibly once it was cut off +// `settings.restartLimit` times in a row (#11139). + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type Message, + type Models, + type ToolResultMessage, +} from "@earendil-works/pi-ai"; +import { + defineTool, + type GenerationCheckpoint, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, + ROOT_CONVERSATION_ID, + type Storage, + type SubmissionId, + type TaskId, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; +import { addSection, addTool } from "./harness-support.ts"; +import { context, followCommits } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-restart-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** + * Open a chat over the SQLite file at `path`. `kill()` simulates a crash: every later storage call hangs, so the Harness + * writes nothing more, not even the `pending` records a close writes for the tasks it stops. + */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +async function live(harness: Harness): Promise> { + return (await harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context)) ?? {}; +} + +/** Every `pi.live` value from now on, as memory adopts it. */ +function liveValues(harness: Harness): LiveState[] { + const values: LiveState[] = []; + followCommits(harness, (publication) => { + for (const change of publication.changes) { + if (change.type === "document" && change.record.kind === "pi.live" && change.kind === "change") { + values.push(change.value as LiveState); + } + } + }); + return values; +} + +/** Let deferred progress commits (`persistWithinMs: 0`) reach Storage before a crash. */ +function stored(): Promise { + return new Promise((resolve) => setTimeout(resolve, 50)); +} + +/** A setup whose model streams a long answer slowly; partials are stored as soon as they are committed. */ +function streaming(): ChatSetup { + const setup = chatSetup({ tokensPerSecond: 20, tokenSize: { min: 1, max: 1 } }); + setup.faux.setResponses([fauxAssistantMessage("z".repeat(400))]); + setup.settings.progress = { persistWithinMs: 0 }; + return setup; +} + +/** Submit an input to a fresh chat at `path` and crash once its streamed partial is stored. */ +async function crashWhileStreaming(path: string, setup: ChatSetup) { + const opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + const taskId = (await live(opened.harness)).run!.taskId; + await stored(); + opened.kill(); + return { id, taskId }; +} + +/** Reopen after a crash in the middle of a request, with another streaming answer, and crash again. */ +async function crashAgain(path: string, setup: ChatSetup, taskId: TaskId) { + const opened = await open(path, setup); + opened.harness.resume(); + // The first attempt was charged; the second one streams. + await waitFor(async () => { + const state = (await opened.harness.getTask(taskId, context))?.state; + if (state?.status !== "running") return false; + const checkpoint = state.checkpoint as GenerationCheckpoint; + return checkpoint.phase === "request" && checkpoint.attempt === 2; + }); + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + await stored(); + opened.kill(); +} + +async function settled(harness: Harness, id: SubmissionId) { + return (await harness.submission(id, context))!.wait(context); +} + +describe("a generation interrupted while the model answers (#11139)", () => { + it("charges the attempt: keeps the partial, waits for the retry backoff, and answers on the next attempt", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const sent: string[][] = []; + setup.faux.setResponses([ + (request) => { + sent.push(request.messages.map((message) => message.role)); + return fauxAssistantMessage("answer"); + }, + ]); + const opened = await open(path, setup); + const values = liveValues(opened.harness); + const partial = textOf((await live(opened.harness)).generation?.message as Message)!; + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + // The backoff showed in `pi.live` with why. + const retry = values.find((value) => value.generation?.retry !== undefined)?.generation; + expect(retry).toEqual({ + attempt: 1, + retry: { at: expect.any(Number), error: "The host stopped while the model was answering, on attempt 1" }, + }); + expect(values.some((value) => value.generation?.attempt === 2)).toBe(true); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant", "pi.assistant"]); + const converted = entries[1]!.model![0] as AssistantMessage; + expect(converted.stopReason).toBe("aborted"); + expect(textOf(converted)).toBe(partial); + // The aborted partial is excluded from the next request. + expect(sent).toEqual([["user"]]); + expect(await live(opened.harness)).toEqual({}); + await opened.harness.close(context); + }); + + it("sleeps the backoff of `settings.retry` before the next attempt", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; + const { taskId } = await crashWhileStreaming(path, first); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; + const now = 1_000_000; + setup.now = () => now; + const opened = await open(path, setup); + opened.harness.resume(); + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + const record = await opened.harness.getTask(taskId, context); + expect(record?.state).toMatchObject({ status: "sleeping", until: now + 60_000, checkpoint: { phase: "retry" } }); + expect(record?.restarts).toBeUndefined(); + expect((await live(opened.harness)).generation).toEqual({ + attempt: 1, + retry: { at: now + 60_000, error: "The host stopped while the model was answering, on attempt 1" }, + }); + expect(opened.harness.wake()).toEqual({ active: false, at: now + 60_000 }); + await opened.harness.close(context); + }); + + it("ends the run interrupted once no retry is left, keeping every partial", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + // Attempt 2 is interrupted too. + const second = streaming(); + second.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + await crashAgain(path, second, taskId); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const opened = await open(path, setup); + opened.harness.resume(); + const text = "The host stopped while the model was answering, on attempt 2"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "interrupted", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text, first: expect.any(Number) }, + }); + expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toEqual({ + status: "failed", + error: { message: text, detail: { reason: "interrupted" } }, + }); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant", "pi.assistant"]); + expect(entries.slice(1).map((entry) => (entry.model![0] as AssistantMessage).stopReason)).toEqual([ + "aborted", + "aborted", + ]); + // No request was sent after the second crash. + expect(setup.faux.state.callCount).toBe(0); + await opened.harness.close(context); + }); + + it("ends the run at once when retries are disabled", async () => { + const path = await sqlitePath(); + const first = streaming(); + const { id } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "interrupted" }); + await opened.harness.close(context); + }); + + it("resends after a clean close: a close is not a crash", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: false }; + const opened = await open(path, first); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + const taskId = (await live(opened.harness)).run!.taskId; + await opened.harness.close(context); + + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + setup.faux.setResponses([fauxAssistantMessage("answer")]); + const reopened = await open(path, setup); + expect((await reopened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + reopened.harness.resume(); + expect((await settled(reopened.harness, id)).status).toBe("done"); + await reopened.harness.close(context); + }); + + it("does not charge a generation that sleeps in a retry or waits for its tools", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 60_000 }; + const gate = deferred(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: obj({}), + replay: "safe", + execute: async () => { + started.resolve(); + await gate.promise; + return {}; + }, + }), + ); + const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); + setup.faux.setResponses([error]); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + const taskId = (await live(opened.harness)).run!.taskId; + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + opened.kill(); + + // Asleep in its retry: not running, so not cut off. + const later = chatSetup(); + later.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 60_000 }; + later.now = () => Date.now() + 120_000; + addTool(later.registry, setup.registry.snapshot().tools()[0]!.tool); + later.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + opened = await open(path, later); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + opened.harness.resume(); + // Waiting for its tool round: not running either. + await started.promise; + const waiting = await opened.harness.getTask(taskId, context); + expect(waiting?.state).toMatchObject({ status: "waiting", checkpoint: { phase: "tools" } }); + opened.kill(); + + opened = await open(path, later); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + gate.resolve(); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + await opened.harness.close(context); + }); + + it("ends a run whose generation is cut off restartLimit times outside a request", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const reached = deferred(); + // Preparation never finishes, as one that exhausts the host's memory would not. + addSection(setup.registry, "preamble", async (_input, ctx) => { + reached.resolve(); + return aborted(ctx.abortSignal!); + }); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await reached.promise; + const taskId = (await live(opened.harness)).run!.taskId; + opened.kill(); + + opened = await open(path, setup); + opened.harness.resume(); + const text = "The generation was interrupted once in its prepare phase"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text, first: expect.any(Number) }, + }); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toContainEqual( + expect.objectContaining({ kind: "task_interrupted", reason: "restart_limit", restarts: 1 }), + ); + await opened.harness.close(context); + }); +}); + +describe("other tasks cut off restartLimit times in a row (#11139)", () => { + it("ends a tool call with an error result instead of running it again", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + let runs = 0; + const running = { current: deferred() }; + addTool( + setup.registry, + defineTool({ + name: "heavy", + description: "Never finishes", + parameters: obj({}), + replay: "safe", + execute: async (_args, api, ctx) => { + runs++; + api.output("partial\n"); + running.current.resolve(); + return aborted(ctx.abortSignal!); + }, + }), + ); + const answers: FauxResponseStep[] = [ + () => fauxAssistantMessage([fauxToolCall("heavy", {}, { id: "c1" })], { stopReason: "toolUse" }), + (request) => { + const result = request.messages.at(-1) as ToolResultMessage; + return fauxAssistantMessage(`saw: ${textOf(result)}`); + }, + ]; + setup.faux.setResponses(answers); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + for (let crash = 1; crash <= 2; crash++) { + await running.current.promise; + running.current = deferred(); + await stored(); + opened.kill(); + opened = await open(path, setup); + opened.harness.resume(); + } + expect((await settled(opened.harness, id)).status).toBe("done"); + expect(runs).toBe(2); + const result = (await allEntries(opened.root)).find((entry) => entry.kind === "pi.tool-result")!; + const message = result.model![0] as ToolResultMessage; + expect(message.isError).toBe(true); + expect(result.data).toMatchObject({ + diagnostics: [ + { + severity: "error", + code: "restart_limit", + message: "Tool heavy was interrupted 2 times in a row; it is not run again", + }, + ], + }); + // The slot's partial output is kept, as for an interrupted call. + expect(textOf(message)).toContain("partial"); + const toolTask = (await opened.harness.commit((tx) => tx.scanTasks({ kind: "pi.tool" }, 10), context)).items[0]!; + expect(toolTask.state).toMatchObject({ + status: "terminal", + outcome: { status: "failed", error: { detail: { reason: "restart_limit" } } }, + }); + expect(setup.infos).toContainEqual({ + kind: "task_interrupted", + task: { id: toolTask.id, kind: "pi.tool", conversationId: ROOT_CONVERSATION_ID }, + reason: "restart_limit", + restarts: 2, + }); + await opened.harness.close(context); + }); + + it("fails a compaction into compactionFailure", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + // Little recent context is kept, so the first turn is summarized. + setup.settings.compaction = { enabled: false, keepRecentTokens: 10 }; + setup.faux.setResponses([fauxAssistantMessage("a1"), fauxAssistantMessage("a2")]); + let opened = await open(path, setup); + opened.harness.resume(); + await (await opened.root.submit({ type: "input", content: "u1 ".repeat(200) }, context)).wait(context); + await (await opened.root.submit({ type: "input", content: "u2 ".repeat(200) }, context)).wait(context); + // The summary request never returns. + const hanging = unanswered(); + setup.faux.setResponses([hanging.step]); + const compaction = await opened.root.compact(undefined, context); + await hanging.reached; + opened.kill(); + + opened = await open(path, setup); + opened.harness.resume(); + const error = "Compaction was interrupted once"; + expect((await opened.harness.waitForTask(compaction, context)).state.outcome).toEqual({ + status: "failed", + error: { message: error, detail: { reason: "restart_limit" } }, + }); + expect((await live(opened.harness)).compactionFailure).toEqual({ + taskId: compaction, + trigger: "manual", + error, + }); + expect((await live(opened.harness)).compactions).toBeUndefined(); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toContainEqual({ + kind: "task_interrupted", + task: { id: compaction, kind: "pi.compaction", conversationId: ROOT_CONVERSATION_ID }, + reason: "restart_limit", + restarts: 1, + }); + await opened.harness.close(context); + }); +}); + +/** + * A setup whose model answers deferred, and whose first fetch of each deferred response is in flight until the Harness + * stops; later fetches answer. `fetching` resolves when the first fetch began. + */ +function deferredSetup(pollAfterMs = 10) { + const base = chatSetup({ deferred: { pollAfterMs } }); + base.settings.stream = { deferred: true }; + base.faux.setResponses([fauxAssistantMessage("deferred answer")]); + const fetching = deferred(); + let fetches = 0; + const fetchDeferred: Models["fetchDeferred"] = (model, handle, options) => { + fetches++; + if (fetches > 1) return base.models.fetchDeferred(model, handle, options); + fetching.resolve(); + return aborted(options!.signal!); + }; + const models = new Proxy(base.models, { + get(target, property) { + if (property === "fetchDeferred") return fetchDeferred; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const setup: ChatSetup = { ...base, models }; + return { setup, fetching, fetches: () => fetches }; +} + +/** Submit to a fresh chat at `path`, let its deferred response be fetched, and crash during that fetch. */ +async function crashWhilePolling(path: string, setup: ChatSetup, fetching: Promise) { + const opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await fetching; + const taskId = (await live(opened.harness)).run!.taskId; + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "running", + checkpoint: { phase: "poll" }, + }); + opened.kill(); + return { id, taskId }; +} + +describe("crash edges (#11139)", () => { + it("charges a crash during a poll's fetch, then polls the same deferred response again", async () => { + const path = await sqlitePath(); + const { setup, fetching, fetches } = deferredSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const { id, taskId } = await crashWhilePolling(path, setup, fetching.promise); + + const opened = await open(path, setup); + const values = liveValues(opened.harness); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + opened.harness.resume(); + const answer = await settled(opened.harness, id); + expect(answer.status).toBe("done"); + expect(values.find((value) => value.generation?.retry !== undefined)?.generation).toEqual({ + attempt: 1, + retry: { + at: expect.any(Number), + error: "The host stopped while the deferred answer was fetched, on attempt 1", + }, + }); + expect(values.some((value) => value.generation?.attempt === 2 && value.generation.deferred !== undefined)).toBe( + true, + ); + // One request, polled twice: the provider's job is neither dropped nor submitted again. + expect(setup.faux.state.callCount).toBe(1); + expect(fetches()).toBe(2); + expect(setup.faux.state.cancelledDeferred).toEqual([]); + const entries = await allEntries(opened.root); + expect(textOf(entries.at(-1)!.model![0])).toBe("deferred answer"); + await opened.harness.close(context); + }); + + it("cancels the deferred response when a crash during its poll ends the run", async () => { + const path = await sqlitePath(); + const { setup, fetching } = deferredSetup(); + setup.settings.retry = { enabled: false }; + const { id, taskId } = await crashWhilePolling(path, setup, fetching.promise); + const opened = await open(path, setup); + opened.harness.resume(); + const text = "The host stopped while the deferred answer was fetched, on attempt 1"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "interrupted", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text, first: expect.any(Number) }, + }); + expect(setup.faux.state.cancelledDeferred).toHaveLength(1); + await opened.harness.close(context); + }); + + it("charges an interrupted poll even when the clock moved backwards", async () => { + const path = await sqlitePath(); + const { setup, fetching } = deferredSetup(10); + let now = 1000; + setup.now = () => now; + setup.settings.retry = { enabled: false }; + const first = await open(path, setup); + first.harness.resume(); + const id = (await first.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(first.harness)).generation?.deferred !== undefined); + now = 1010; + first.harness.resume(); + await fetching.promise; + first.kill(); + // A replacement host whose clock is behind: the poll looks not due yet, but its fetch was cut off. + now = 1000; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "interrupted" }); + await opened.harness.close(context); + }); + + it("charges a request once when a crash is followed by a clean close before the charge", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 3, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 3, baseDelayMs: 1 }; + setup.faux.setResponses([fauxAssistantMessage("answer")]); + let opened = await open(path, setup); + await opened.harness.close(context); + opened = await open(path, setup); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + const values = liveValues(opened.harness); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + const charged = values.filter((value) => value.generation?.retry !== undefined); + expect(charged.map((value) => value.generation?.attempt)).toEqual([1]); + expect((await allEntries(opened.root)).map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.assistant", + "pi.assistant", + ]); + await opened.harness.close(context); + }); + + it("ends a run restart_limit when charging the interrupted request keeps being cut off", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.restartLimit = 1; + const { id, taskId } = await crashWhileStreaming(path, first); + // At the limit: the partial is still kept. + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const opened = await open(path, setup); + const partial = textOf((await live(opened.harness)).generation?.message as Message)!; + opened.harness.resume(); + const text = "The generation was interrupted once in its request phase"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text, first: expect.any(Number) }, + }); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); + expect(textOf(entries[1]!.model![0])).toBe(partial); + expect(setup.faux.state.callCount).toBe(0); + await opened.harness.close(context); + }); + + it("drops the partial past the limit, where converting it may be what keeps failing", async () => { + const path = await sqlitePath(); + const first = streaming(); + const { id, taskId } = await crashWhileStreaming(path, first); + // As if the charge had been cut off twice before: the open after this crash counts 3, past the limit of 2. + const raw = await openNodeSqliteStorage(path); + const record = await raw.task(taskId, context); + await commitWrites(raw, [{ type: "task", value: { ...record!, restarts: 2 } }], context); + await raw.close(context); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: "The generation was interrupted 3 times in a row in its request phase", + }); + expect((await allEntries(opened.root)).map((entry) => entry.kind)).toEqual(["pi.user"]); + expect((await live(opened.harness)).generation).toBeUndefined(); + await opened.harness.close(context); + }); + + it("ends a run someone aborted as aborted when its abort handler is cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ deferred: { pollAfterMs: 3_600_000 } }); + setup.settings.restartLimit = 2; + setup.settings.stream = { deferred: true }; + setup.faux.setResponses([fauxAssistantMessage("never")]); + let cancels = 0; + // The abort handler of a polling generation cancels its response: make that hang. + setup.models.cancelDeferred = () => { + cancels++; + return new Promise(() => {}); + }; + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + const taskId = (await live(opened.harness)).run!.taskId; + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + void opened.root.abort(context).catch(() => {}); + for (let crash = 1; crash <= 2; crash++) { + await waitFor(() => cancels === crash); + opened.kill(); + opened = await open(path, setup); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(crash); + opened.harness.resume(); + } + const detail = `Abort handler of task ${taskId} (pi.generation) was cut off 2 times`; + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "aborted", detail }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "aborted", detail, first: expect.any(Number) }, + }); + expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toMatchObject({ + status: "faulted", + error: { detail: { reason: "restart_limit" } }, + }); + await waitFor(() => setup.infos.some((info) => info.kind === "task_faulted")); + expect(setup.infos).toContainEqual(expect.objectContaining({ kind: "task_faulted", reason: "restart_limit" })); + await opened.harness.close(context); + }); +}); + +describe("reports of work crashes ended (#11139)", () => { + it("reports task_interrupted for a run that ends interrupted and for a task that ends at the limit", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: false }; + const { taskId } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const opened = await open(path, setup); + opened.harness.resume(); + await opened.harness.waitForTask(taskId, context); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toEqual([ + { + kind: "task_interrupted", + task: { id: taskId, kind: "pi.generation", conversationId: ROOT_CONVERSATION_ID }, + reason: "interrupted", + restarts: 1, + }, + ]); + expect(String(setup.reports[0])).toContain("The host stopped while the model was answering, on attempt 1"); + await opened.harness.close(context); + }); +}); + +describe("the restart limit on the normal path (#11139)", () => { + it("resolves the settings no more often than before: only a task that was cut off reads the limit", async () => { + const setup = chatSetup(); + let resolutions = 0; + // Every resolution of the host settings reads `extensions` once. + (setup as { settings: object }).settings = new Proxy( + {}, + { + get(target, key) { + if (key === "extensions") resolutions++; + return Reflect.get(target, key); + }, + }, + ); + addTool( + setup.registry, + defineTool({ + name: "t", + description: "t", + parameters: obj({}), + replay: "safe", + execute: async () => ({}), + }), + ); + const rounds = 5; + const responses: FauxResponseStep[] = []; + for (let round = 0; round <= rounds; round++) { + responses.push(() => + fauxAssistantMessage([fauxToolCall("t", {}, { id: `c${round}` })], { stopReason: "toolUse" }), + ); + responses.push(() => fauxAssistantMessage([fauxText("done")])); + } + setup.faux.setResponses(responses); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "warm up" }, context)).wait(context); + resolutions = 0; + for (let round = 0; round < rounds; round++) { + await (await root.submit({ type: "input", content: `round ${round}` }, context)).wait(context); + } + // One generation round with one tool call, no more often than before the restart limit applied to every kind + // (#11327: 49; group commit coalesces some reconcile passes). + expect(resolutions / rounds).toBeLessThanOrEqual(49); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-run-failure.test.ts b/packages/durable/test/harness-run-failure.test.ts new file mode 100644 index 000000000..638a4715b --- /dev/null +++ b/packages/durable/test/harness-run-failure.test.ts @@ -0,0 +1,164 @@ +// `pi.live.runFailure`: why the newest run ended without an answer, as state a renderer shows (#11327). + +import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; +import { + type AfterFailedRun, + type Conversation, + defineTool, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, + type RunFailure, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { describe, expect, it } from "vitest"; +import { chatSetup, openChat, unanswered } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +async function live(harness: Harness, conversation: Conversation): Promise> { + return (await harness.snapshot(LiveDoc, conversation.id, context)) ?? {}; +} + +/** A response that fails for good once `gate` opens; `reached` resolves when it was requested. */ +function failingOnce() { + const gate = deferred(); + const reached = deferred(); + const step: FauxResponseStep = async () => { + reached.resolve(); + await gate.promise; + return fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + }; + return { step, gate, reached }; +} + +describe("pi.live.runFailure", () => { + it("records a model error with its text, and a later run clears it", async () => { + const setup = chatSetup(); + setup.faux.setResponses([ + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }), + fauxAssistantMessage([fauxText("fine")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "hi" }, context); + const settled = await submission.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "model_error", detail: "Invalid request" }); + const failure = (await live(harness, root)).runFailure; + expect(failure).toEqual({ + taskId: expect.any(Number), + inputs: [submission.id], + reason: "model_error", + detail: "Invalid request", + first: expect.any(Number), + } satisfies RunFailure); + const next = await root.submit({ type: "input", content: "again" }, context); + // The commit that starts the next run removes it. + expect((await live(harness, root)).runFailure).toBeUndefined(); + expect((await next.wait(context)).status).toBe("done"); + expect(await live(harness, root)).toEqual({}); + await harness.close(context); + }); + + it("records a missing model with which one", async () => { + const setup = chatSetup(); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ model: { provider: "faux", modelId: "gone" } }, context); + const submission = await root.submit({ type: "input", content: "hi" }, context); + const settled = await submission.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "no_model" }); + // The settlement is unchanged: no detail. + expect(settled).not.toHaveProperty("detail"); + expect((await live(harness, root)).runFailure).toMatchObject({ + inputs: [submission.id], + reason: "no_model", + detail: "Model faux/gone is not available", + }); + await harness.close(context); + }); + + it("records an abort", async () => { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await busy.reached; + const taskId = (await live(harness, root)).run!.taskId; + await root.abort(context); + expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); + expect((await live(harness, root)).runFailure).toEqual({ + taskId, + inputs: [submission.id], + reason: "aborted", + first: expect.any(Number), + }); + await harness.close(context); + }); + + it("records a reset that cut the run before an answer, kept by the run its queued input starts", async () => { + const setup = chatSetup(); + const gate = deferred(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: obj({}), + execute: async () => { + started.resolve(); + await gate.promise; + return {}; + }, + }), + ); + const second = unanswered(); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + second.step, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.reset(undefined, context); + const followUp = await root.submit({ type: "input", content: "next", whenBusy: "followUp" }, context); + gate.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "reset" }); + await second.reached; + const state = await live(harness, root); + // The follow-up's run started in the commit that ended the first; the failure stays until a later run starts. + expect(state.run?.inputs).toEqual([followUp.id]); + expect(state.runFailure).toMatchObject({ inputs: [first.id], reason: "reset" }); + await root.abort(context); + expect((await live(harness, root)).runFailure).toMatchObject({ inputs: [followUp.id], reason: "aborted" }); + await harness.close(context); + }); + + it("keeps the failure while a run continues the queue in the same commit, and replaces it with the next", async () => { + for (const policy of ["continue", "keep"] as const satisfies readonly AfterFailedRun[]) { + const setup = chatSetup(); + setup.settings.afterFailedRun = policy; + const failing = failingOnce(); + const busy = unanswered(); + setup.faux.setResponses([failing.step, busy.step, fauxAssistantMessage([fauxText("third")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await failing.reached.promise; + const second = await root.submit({ type: "input", content: "second", whenBusy: "followUp" }, context); + failing.gate.resolve(); + await first.wait(context); + if (policy === "continue") { + await busy.reached; + const state = await live(harness, root); + expect(state.run?.inputs).toEqual([second.id]); + expect(state.runFailure).toMatchObject({ inputs: [first.id], reason: "model_error" }); + } else { + await harness.waitForIdle(context); + expect((await live(harness, root)).runFailure).toMatchObject({ inputs: [first.id] }); + } + await harness.close(context); + } + }); +}); diff --git a/packages/durable/test/harness-scheduler-indexes.test.ts b/packages/durable/test/harness-scheduler-indexes.test.ts index c42efd96e..33fd9ef88 100644 --- a/packages/durable/test/harness-scheduler-indexes.test.ts +++ b/packages/durable/test/harness-scheduler-indexes.test.ts @@ -3,7 +3,14 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createSession, defineTask, type Harness, type TaskId, type TaskRuntime } from "@earendil-works/pi-durable"; +import { + createSession, + defineTask, + type Harness, + type Storage, + type TaskId, + type TaskRuntime, +} from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { context } from "./session-support.ts"; @@ -104,16 +111,16 @@ describe("scheduler indexes", () => { }, context); await session.close(context); // Record every mark of the leaf, in commit order. - const storage = await openNodeSqliteStorage(path); + const storage: Storage = await openNodeSqliteStorage(path); const reasons: (string | undefined)[] = []; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - for (const write of writes) { + storage.commit = (batches, receive, commitContext) => { + for (const write of batches.flat()) { if (write.type === "task" && write.value.id === ids.leaf && write.value.abortRequested) { reasons.push(write.value.abortReason); } } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const { harness } = await openTasks(storage, [Hold]); harness.resume(); @@ -123,17 +130,20 @@ describe("scheduler indexes", () => { }); it("finalizes a chain of held owners in the commit that frees the last one", async () => { - const storage = await openNodeSqliteStorage(await sqlitePath()); + const storage: Storage = await openNodeSqliteStorage(await sqlitePath()); // The commit, by count, that wrote each terminal record. const endedIn = new Map(); let commits = 0; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - for (const write of writes) { - if (write.type === "task" && write.value.state.status === "terminal") endedIn.set(write.value.id, commits); + storage.commit = (batches, receive, commitContext) => { + for (const writes of batches) { + commits++; + for (const write of writes) { + if (write.type === "task" && write.value.state.status === "terminal") + endedIn.set(write.value.id, commits); + } } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const opened = await openTasks(storage, [Hold]); const harness = opened.harness; diff --git a/packages/durable/test/harness-scheduler-retention.test.ts b/packages/durable/test/harness-scheduler-retention.test.ts index 8b12c4cb0..c50ba6fe6 100644 --- a/packages/durable/test/harness-scheduler-retention.test.ts +++ b/packages/durable/test/harness-scheduler-retention.test.ts @@ -125,21 +125,6 @@ async function sqlitePath(): Promise { } /** Count the task and conversation reads a storage serves. */ -function countReads(storage: Storage): { tasks: number; conversations: number } { - const reads = { tasks: 0, conversations: 0 }; - const task = storage.task.bind(storage); - const conversation = storage.conversation.bind(storage); - storage.task = (id, readContext) => { - reads.tasks++; - return task(id, readContext); - }; - storage.conversation = (id, readContext) => { - reads.conversations++; - return conversation(id, readContext); - }; - return reads; -} - async function open(storage?: Storage) { const opened = await openTasks(storage ?? (await openNodeSqliteStorage(await sqlitePath())), [ Hold, @@ -191,9 +176,7 @@ describe("scheduler retention", () => { }); it("holds a live owner for work later created in a forgotten conversation, loading its chain again", async () => { - const storage = await openNodeSqliteStorage(await sqlitePath()); - const reads = countReads(storage); - const { harness, root } = await open(storage); + const { harness, root } = await open(); // Live P -> ended B -> ended A -> conversation C, empty: forgotten. const tree = await root.commit(async (tx) => { const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); @@ -206,15 +189,14 @@ describe("scheduler retention", () => { gate("b").resolve(); await harness.waitForTask(tree.b, context); const before = sizes(harness); - const loaded = reads.tasks + reads.conversations; const c = (await harness.conversation(tree.c, context))!; const d = await c.commit( (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), context, ); gate("p").resolve(); + // P holds for D: the scheduler loaded the chain from C up to P again. await eventually(async () => (await status(harness, tree.p)).status === "completing"); - expect(reads.tasks + reads.conversations).toBeGreaterThan(loaded); gate("d").resolve(); expect((await harness.waitForTask(d, context)).state.outcome.status).toBe("completed"); expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); @@ -267,7 +249,7 @@ describe("scheduler retention", () => { const c = (await harness.conversation(tree.c, context))!; // Busy, so the input queues instead of starting a run. await c.commit(async (tx) => { - (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [] }; + (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [], turns: 0 }; }, context); const queued = (await c.submit({ type: "input", content: "later" }, context)).id; await harness.abortTask(tree.p, context); @@ -307,7 +289,7 @@ describe("scheduler retention", () => { await harness.close(context); }); - it("keeps a scope busy while the chain of new work below a forgotten conversation loads", async () => { + it("loads the chain of new work below a forgotten conversation before admitting it, and keeps the scope busy", async () => { const path = await sqlitePath(); const storage = await openNodeSqliteStorage(path); const { harness, root } = await open(storage); @@ -318,28 +300,36 @@ describe("scheduler retention", () => { }, context); gate("b").resolve(); await harness.waitForTask(tree.b, context); - // Hold the chain load's read of C: the second read once holding, after the creating commit's own. - const read = storage.conversation.bind(storage); + await harness.close(context); + // Reopen, so that B is in no cache: the chain load reads it from Storage. Hold that read. + const reopened = await openNodeSqliteStorage(path); + const read = reopened.task.bind(reopened); const held = deferred(); const reached = deferred(); let holding = false; - let readsOfC = 0; - storage.conversation = async (id, readContext) => { - if (holding && id === tree.c && ++readsOfC === 2) { + reopened.task = async (id, readContext) => { + if (holding && id === tree.b) { reached.resolve(); await held.promise; } return read(id, readContext); }; - const c = (await harness.conversation(tree.c, context))!; + const second = (await open(reopened)).harness; + const c = (await second.conversation(tree.c, context))!; holding = true; - await c.commit((tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), context); + const committing = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + // Its admission loads the chain, for the memberships it writes: the commit waits for the read. await reached.promise; - expect(await settled(harness.waitForIdle(context))).toBe(false); + expect(await settled(committing)).toBe(false); held.resolve(); + await committing; + expect(await settled(second.waitForIdle(context))).toBe(false); gate("d").resolve(); - await harness.waitForIdle(context); - await harness.close(context); + await second.waitForIdle(context); + await second.close(context); }); it("keeps a chain when one commit ends its last task and creates new work below it", async () => { @@ -452,7 +442,7 @@ describe("scheduler retention", () => { await harness.waitForTask(tree.b, context); const c = (await harness.conversation(tree.c, context))!; await c.commit(async (tx) => { - (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [] }; + (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [], turns: 0 }; }, context); // Queued while nothing cancels: no withdrawal scan. const queued = (await c.submit({ type: "input", content: "later" }, context)).id; @@ -529,7 +519,7 @@ describe("scheduler retention", () => { context, ); await root.commit(async (tx) => { - (await tx.doc(LiveDoc, root.id)).run = { taskId: busy, inputs: [] }; + (await tx.doc(LiveDoc, root.id)).run = { taskId: busy, inputs: [], turns: 0 }; }, context); const before = scans; await root.submit({ type: "input", content: "queued" }, context); diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 9169e62c7..58a5543b4 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -11,13 +11,12 @@ import { MemoryStorage, StorageRequestError, type TaskId, - watchEvents, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { safeNow } from "../src/harness/harness.ts"; import { conversationViews } from "../src/harness/view.ts"; import { contained } from "../src/session/session.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, onValue } from "./session-support.ts"; import { aborted, deferred, openTasks } from "./task-support.ts"; /** Waits for its signal, records that it saw it, and ends `aborted` through its abort handler. */ @@ -95,8 +94,8 @@ describe("a failed Storage call", () => { const task = harness.waitForTask(id, context); const idle = harness.waitForIdle(context); const watch = (await harness.watchDoc(LiveDoc, root.id, context))!; - const stream = await watchEvents(harness, root.id, context); - stream.start(async () => {}); + const view = await root.watch(context); + view.start(async () => {}); const failure = new Error("disk gone"); storage.failNextCommit(failure); @@ -106,13 +105,13 @@ describe("a failed Storage call", () => { const failed = { name: "SessionFailed", cause: failure }; await expect(task).rejects.toMatchObject(failed); await expect(idle).rejects.toMatchObject(failed); - expect(await watch.closed).toEqual({ reason: "failed", error: failure }); - expect(await stream.closed).toEqual({ reason: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); + expect(await view.closed).toEqual({ kind: "failed", error: failure }); // Every later call, and a wait that starts now. await expect(harness.getTask(id, context)).rejects.toMatchObject(failed); await expect(harness.waitForIdle(context)).rejects.toMatchObject(failed); await expect(root.submit({ type: "input", content: "x" }, context)).rejects.toMatchObject(failed); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(signals).toEqual(["aborted"]); expect(reports).toEqual([failure]); await harness.close(context); @@ -124,7 +123,7 @@ describe("a failed Storage call", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); storage.failing.add("scanRange"); await expect(root.context(context)).rejects.toBe(storage.error); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); expect(reports).toEqual([storage.error]); await harness.close(context); }); @@ -149,7 +148,7 @@ describe("a failed Storage call", () => { // The read succeeds at the backend, yet its caller gets the failure: nothing it read is used. release.resolve(); await expect(reading).rejects.toMatchObject({ name: "SessionFailed", cause: storage.error }); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); await harness.close(context); }); @@ -186,7 +185,7 @@ describe("a failed Storage call", () => { await expect(reading).rejects.toThrow("is closed"); await closing; expect(pages).toEqual(["first", "close"]); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); it("fails it when Storage cannot close; an earlier failure stays the cause", async () => { @@ -199,7 +198,7 @@ describe("a failed Storage call", () => { } const healthy = await started(new FailingClose()); await expect(healthy.harness.close(context)).rejects.toBe(closeError); - expect(await healthy.harness.closed).toEqual({ reason: "failed", error: closeError }); + expect(await healthy.harness.closed).toEqual({ kind: "failed", error: closeError }); expect(healthy.reports).toEqual([closeError]); const storage = new FailingClose(); @@ -209,7 +208,7 @@ describe("a failed Storage call", () => { await expect(failed.root.commit((tx) => tx.appendEntry(failed.root.id, { kind: "note" }), context)).rejects.toBe( failure, ); - expect(await failed.harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await failed.harness.closed).toEqual({ kind: "failed", error: failure }); expect(failed.reports).toEqual([failure]); }); @@ -260,7 +259,7 @@ describe("a failed Storage call", () => { storage.failNextCommit(failure); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); await opened.harness.closed; - expect(ends).toEqual([{ reason: "failed", error: failure }, "SessionFailed"]); + expect(ends).toEqual([{ kind: "failed", error: failure }, "SessionFailed"]); await opened.harness.close(context); }); @@ -269,7 +268,7 @@ describe("a failed Storage call", () => { const { harness, root } = await started(storage); storage.failing.add("mintId"); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(storage.error); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); await harness.close(context); }); @@ -294,7 +293,7 @@ describe("a failed Storage call", () => { const rejection = new StorageRequestError("bad batch"); storage.failNextCommit(rejection); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(rejection); - expect(await harness.closed).toEqual({ reason: "failed", error: rejection }); + expect(await harness.closed).toEqual({ kind: "failed", error: rejection }); await harness.close(context); }); @@ -329,12 +328,12 @@ describe("a failed Storage call", () => { const storage = new ControlledStorage(); const { harness, root, reports } = await started(storage); const bug = new Error("listener bug"); - (harness as unknown as { observeCommits(listener: () => void): void }).observeCommits(() => { + (harness as unknown as { followCommits(listener: () => void): void }).followCommits(() => { throw bug; }); // The commit itself is durable and resolves; the Session fails behind it. await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); - expect(await harness.closed).toEqual({ reason: "failed", error: bug }); + expect(await harness.closed).toEqual({ kind: "failed", error: bug }); expect(reports).toEqual([bug]); await harness.close(context); }); @@ -351,7 +350,7 @@ describe("a failed Storage call", () => { held.release(); await expect(committing).rejects.toBe(failure); await closing; - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); }); @@ -420,6 +419,8 @@ describe("a failed Storage call", () => { } } const { harness, root } = await started(new CancelledRead()); + // An entry to scan: the Session's cache answers the bounds of the context, not its entries. + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); const cancelling: Context = { ...context, abortSignal: controller.signal }; await expect(root.context(cancelling)).rejects.toBe(cancelled); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); @@ -437,7 +438,7 @@ describe("listener and report errors", () => { const asyncThrow = async () => { throw new Error("async listener"); }; - harness.subscribeCommits(asyncThrow); + harness.observeCommits({ advance: asyncThrow, end: () => {} }); harness.subscribeClose(asyncThrow); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); await harness.close(context); @@ -450,8 +451,11 @@ describe("listener and report errors", () => { { models: createModels(), registry, onReport: asyncThrow }, context, ); - rejecting.subscribeCommits(() => { - throw new Error("sync listener"); + rejecting.observeCommits({ + advance: () => { + throw new Error("sync listener"); + }, + end: () => {}, }); const again = await rejecting.root(context); await again.commit((tx) => tx.appendEntry(again.id, { kind: "note" }), context); @@ -467,10 +471,13 @@ describe("listener and report errors", () => { const storage = new MemoryStorage(); const { harness, root, reports } = await started(storage); const seen: string[] = []; - harness.subscribeCommits(() => { - throw new Error("commit listener"); + harness.observeCommits({ + advance: () => { + throw new Error("commit listener"); + }, + end: () => {}, }); - harness.subscribeCommits(() => seen.push("commit")); + harness.observeCommits({ advance: () => seen.push("commit"), end: () => {} }); harness.subscribeClose(() => { throw new Error("close listener"); }); @@ -487,8 +494,9 @@ describe("listener and report errors", () => { const failure = new Error("observer broke"); await views.attach( root.id, - async () => ({ - publication: () => { + undefined, + () => ({ + advance: () => { throw failure; }, end: () => {}, @@ -497,14 +505,16 @@ describe("listener and report errors", () => { ); const watch = await root.watch(context); const seen: unknown[] = []; - watch.start(async (value) => { - seen.push(value); - }); + watch.start( + onValue(async ({ value }) => { + seen.push(value); + }), + ); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); while (seen.length === 0) await new Promise((resolve) => setTimeout(resolve, 1)); expect(reports).toEqual([failure]); await harness.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); }); it("contains a fulfilled thenable that calls only its fulfillment handler", async () => { @@ -559,7 +569,7 @@ describe("listener and report errors", () => { await root.commit(async (tx) => { (await tx.doc(LiveDoc, root.id)).compactions = []; }, context); - expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await watch.closed).toEqual({ kind: "listener_error", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); }); @@ -595,7 +605,7 @@ describe("listener and report errors", () => { await root.commit(async (tx) => { (await tx.doc(LiveDoc, root.id)).compactions = []; }, context); - expect(await watch.closed).toMatchObject({ reason: "listener_error" }); + expect(await watch.closed).toMatchObject({ kind: "listener_error" }); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); await opened.harness.close(context); }); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 92ef991a5..5ef6f75ba 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -20,8 +19,8 @@ import { type ToolRegistration, ToolResultEntry, ToolTask, - watchEvents, } from "@earendil-works/pi-durable"; +import { nul, num, obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool } from "../src/tools/index.ts"; @@ -29,14 +28,14 @@ import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support. import { addHooks, addTool } from "./harness-support.ts"; import { context } from "./session-support.ts"; -const Count = Type.Object({ count: Type.Number() }); +const Count = obj({ count: num() }); /** A tool that returns `result`, with `extra` such as a schema. */ function returning(name: string, result: ToolExecutionResult, extra: Partial = {}): ToolRegistration { return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => result, ...extra, }); @@ -56,7 +55,7 @@ async function nested(setup: ChatSetup, name: string): Promise { received = await api.executeTool(name, {}, callContext); return {}; @@ -133,17 +132,14 @@ describe("structured output", () => { it("stores exactly what callers and events get, without the model's output, and keeps a stored null", async () => { const setup = chatSetup(); addTool(setup.registry, returning("plain", { output: [{ type: "text", text: "a" }] })); - addTool( - setup.registry, - returning("nothing", { structuredOutput: null }, { structuredOutputSchema: Type.Null() }), - ); + addTool(setup.registry, returning("nothing", { structuredOutput: null }, { structuredOutputSchema: nul() })); const received: NestedToolExecutionResult[] = []; addTool( setup.registry, defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("plain", {}, callContext)); received.push(await api.executeTool("nothing", {}, callContext)); @@ -156,32 +152,30 @@ describe("structured output", () => { const { harness, root } = await openChat(storage, setup); // The result each nested call's document got, as committed. const stored: JsonValue[] = []; - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "document" || change.record.kind !== NestedCallDoc.definition.kind) continue; - const result = (change.value as NestedCallState | null)?.result; - if (result !== undefined && change.ops.length > 0) stored.push(result as unknown as JsonValue); - } - }); - const stream = await watchEvents(harness, root.id, context); - const ends: NestedToolExecutionResult[] = []; - stream.start(async (batch) => { - for (const event of batch) { - if (event.type === "tool_execution_end" && event.parentToolCallId !== undefined) { - ends.push(event.result as NestedToolExecutionResult); + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if ( + change.type !== "document" || + change.kind !== "change" || + change.record.kind !== NestedCallDoc.definition.kind + ) + continue; + const result = (change.value as NestedCallState)?.result; + if (result !== undefined && change.ops.length > 0) stored.push(result as unknown as JsonValue); } - } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); await harness.close(context); expect(received[0]).toMatchObject({ structuredOutput: "a", isError: false }); expect(received[1]).toMatchObject({ structuredOutput: null, isError: false }); expect(received[0]).not.toHaveProperty("output"); + // A client reads a live caller's nested results from the same documents. expect(stored).toEqual(received); - expect(ends).toEqual(received); }); it("gives callers a schema tool's validated structured output, which the transcript does not store", async () => { @@ -275,7 +269,7 @@ describe("structured output", () => { defineTool({ name: "throws", description: "throws", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Count, execute: async () => { throw new Error("permission denied"); @@ -347,7 +341,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { plain = await api.executeTool("plain", {}, callContext); counter = await api.executeTool("counter", {}, callContext); @@ -378,7 +372,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("bash", { command: "printf ok" }, callContext)); received.push(await api.executeTool("bash", { command: "printf bad; exit 4" }, callContext)); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 7bc47916d..26c7d7fb6 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -8,12 +8,11 @@ import { fauxToolCall, type Models, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type Conversation, type ConversationId, + type ConversationView, createSession, defineDoc, defineTask, @@ -26,19 +25,22 @@ import { type NextTaskState, type Registry, type Storage, + type StorageWrite, + type TaskGraph, type TaskId, type TaskOptions, type TaskRuntime, type ToolRegistration, ToolResultEntry, type Tx, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { generations, graphHistory, slotsOf, viewHistory } from "./live-support.ts"; +import { ControlledStorage, context, refusingAt } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; // ─── A scriptable task ────────────────────────────────────────────────────── @@ -1190,7 +1192,7 @@ function blockingTool(name: string): { started: Promise; registration: Too registration: defineTool({ name, description: `The ${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), executionMode: "sequential", execute: async (_args, api, callContext) => { api.output("partial"); @@ -1216,7 +1218,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1251,7 +1253,9 @@ describe("tool rounds", () => { expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); const results = (await allEntries(root)).filter((entry) => ToolResultEntry.is(entry)); expect(results.map((entry) => (entry.model![0] as ToolResultMessage).toolCallId)).toEqual(["c1", "c2"]); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: { taskId: generation, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); expect((await harness.waitForTask(generation, context)).state.outcome.status).toBe("aborted"); await harness.close(context); }); @@ -1286,7 +1290,7 @@ describe("tool rounds", () => { await harness.close(context); }); - it("ends a turn at the generation's hold, before its successor's turn starts", async () => { + it("hands run control to the successor while the generation holds, and keeps it in the graph until it ends", async () => { const setup = chatSetup(); addTask(setup.registry, Node); addTool( @@ -1294,7 +1298,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1308,21 +1312,23 @@ describe("tool rounds", () => { setup.faux.setResponses([toolCalls(["noop", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); opened = harness; - const stream = await watchEvents(harness, root.id, context); - const events: AgentEvent[] = []; - stream.start(async (batch) => { - events.push(...batch); - }); + const graph = await graphHistory(harness); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const first = (await harness.commit((tx) => tx.scanTasks({ kind: "pi.generation" }, 1), context)).items[0]!; expect((await state(harness, first.id)).status).toBe("completing"); - const turns = () => events.filter((event) => event.type.startsWith("turn_")).map((event) => event.type); - await waitFor(() => turns().length === 4); - expect(turns()).toEqual(["turn_start", "turn_end", "turn_start", "turn_end"]); + // The first generation's outcome is committed at its hold: the successor already runs the run, and the graph shows + // the first one completing beside it. + const handed = graph.graphs.find((value) => generations(value).length === 2)!; + const [held, successor] = generations(handed); + expect(held).toMatchObject({ id: first.id, state: { status: "completing" } }); + expect(successor!.state.status).not.toBe("completing"); + expect(history.live().some((live) => live.run?.taskId === successor!.id)).toBe(true); open("hooked"); expect(await outcomeOf(harness, first.id)).toBe("completed"); - await waitFor(() => turns().length === 4); - await stream.stop(); + await waitFor(() => graph.graphs.at(-1)!.tasks[String(first.id)] === undefined); + graph.detach(); + history.detach(); await harness.close(context); }); @@ -1336,6 +1342,9 @@ describe("tool rounds", () => { }, }); const setup = { ...base, models }; + // The partial is stored at once, so the failed final commit leaves it for the reopened run to convert; held, it would + // be lost with the failed Session. + setup.settings.progress = { persistWithinMs: 0 }; addTask(setup.registry, Node); let opened: Harness | undefined; addHooks(setup.registry, GenerationTask, { @@ -1353,19 +1362,20 @@ describe("tool rounds", () => { let reject: TaskId | undefined; const failure = new Error("disk gone"); class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const finalizes = writes.some( - (write) => write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", - ); - if (finalizes) { - reject = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const finalizes = writes.some( + (write) => + write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", + ); + if (finalizes) reject = undefined; + return finalizes; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -1385,7 +1395,7 @@ describe("tool rounds", () => { reject = generation; open("abort.hooked"); // The final commit, with the run's cleanup, fails: nothing of the cleanup lands, and the Harness fails. - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); await expect(submission.wait(context)).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); await harness.close(context); // Reopening finalizes the run; the partial becomes one aborted entry. @@ -1393,7 +1403,15 @@ describe("tool rounds", () => { opened = reopened.harness; const again = (await reopened.harness.submission(submission.id, context))!; expect(await again.wait(context)).toMatchObject({ status: "unanswered", reason: "faulted" }); - expect(await reopened.harness.snapshot(LiveDoc, reopened.root.id, context)).toEqual({}); + expect(await reopened.harness.snapshot(LiveDoc, reopened.root.id, context)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [submission.id], + reason: "faulted", + detail: "Value contains a non-JSON function; expected strict JSON", + first: expect.any(Number), + }, + }); expect((await allEntries(reopened.root)).map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); await reopened.harness.close(context); }); @@ -1631,19 +1649,20 @@ describe("conversation abort and boundaries", () => { let reject: TaskId | undefined; const failure = new Error("disk gone"); class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const finalizes = writes.some( - (write) => write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", - ); - if (finalizes) { - reject = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const finalizes = writes.some( + (write) => + write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", + ); + if (finalizes) reject = undefined; + return finalizes; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -1653,7 +1672,7 @@ describe("conversation abort and boundaries", () => { await until(async () => (await state(harness, parent)).status === "completing"); reject = parent; open("child"); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); const reopened = await openNodes(storage.reopen()); @@ -1754,25 +1773,12 @@ describe("recovery through ownership edges", () => { }); }); -describe("tool rounds and events", () => { - async function listen(harness: Harness, conversation: Conversation) { - const stream = await watchEvents(harness, conversation.id, context); - const events: AgentEvent[] = []; - stream.start(async (batch) => { - events.push(...batch); - }); - return { stream, events }; - } - - /** Event types, with tool ends and message ends labelled by call ID and whether an entry came along. */ - function labels(events: readonly AgentEvent[]): string[] { - return events.flatMap((event) => { - if (event.type === "tool_execution_end") return [`end:${event.toolCallId}:${event.entry !== undefined}`]; - if (event.type === "message_end") { - const message = event.entry.model?.[0]; - return [message?.role === "toolResult" ? `result:${message.toolCallId}` : `message:${message?.role}`]; - } - return []; +describe("tool rounds as the view shows them", () => { + /** The tool result entries of a view, by call ID, in transcript order. */ + function resultsOf(view: ConversationView): string[] { + return view.entries.flatMap((entry) => { + const message = entry.model?.[0]; + return message?.role === "toolResult" ? [message.toolCallId] : []; }); } @@ -1783,25 +1789,36 @@ describe("tool rounds and events", () => { addTool(setup.registry, defineTool({ ...blockingTool("two").registration, name: "two" })); setup.faux.setResponses([toolCalls(["one", "c1"], ["two", "c2"], ["two", "c3"])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); + const history = await viewHistory(harness, root.id); const submission = await root.submit({ type: "input", content: "go" }, context); await one.started; await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); await submission.wait(context); - await waitFor(() => labels(events).includes("result:c3")); - expect(labels(events).filter((label) => !label.startsWith("message:"))).toEqual([ - "end:c1:true", - "result:c1", - "end:c2:true", - "result:c2", - "end:c3:true", - "result:c3", + history.detach(); + // The started call ends with its result entry; the commit that ends the run appends the unstarted calls' results, + // in call order, and removes the round. + const states = history.live(); + const c1Done = slotsOf(states, "c1").find((slot) => slot?.status === "done"); + const last = history.views.at(-1)!; + expect(resultsOf(last)).toEqual(["c1", "c2", "c3"]); + const c1Result = last.entries.find( + (entry) => + entry.model?.[0]?.role === "toolResult" && (entry.model[0] as ToolResultMessage).toolCallId === "c1", + ); + expect(c1Done?.entry).toBe(c1Result?.id); + const ending = history.views.findIndex((view) => resultsOf(view).includes("c2")); + expect(resultsOf(history.views[ending]!)).toEqual(["c1", "c2", "c3"]); + expect(states[ending]!.run).toBeUndefined(); + expect(states[ending]!.tools).toBeUndefined(); + expect(states[ending - 1]!.tools?.map((slot) => [slot.callId, slot.status])).toEqual([ + ["c1", "done"], + ["c2", "pending"], + ["c3", "pending"], ]); - await stream.stop(); await harness.close(context); }); - it("emits one turn_end per generation, also for a stream attached while it holds", async () => { + it("shows a held generation as completing to an observer attached while it holds", async () => { const setup = chatSetup(); addTask(setup.registry, Node); addTool( @@ -1809,7 +1826,7 @@ describe("tool rounds and events", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1822,19 +1839,22 @@ describe("tool rounds and events", () => { setup.faux.setResponses([toolCalls(["noop", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); opened = harness; - const early = await listen(harness, root); + const early = await graphHistory(harness); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - const late = await listen(harness, root); + const late = await graphHistory(harness); open("hooked"); const [first] = (await harness.commit((tx) => tx.scanTasks({ kind: "pi.generation" }, 1), context)).items; await harness.waitForTask(first!.id, context); - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - await waitFor(() => late.events.some((event) => event.type === "entry_appended")); - const turns = (events: readonly AgentEvent[]) => events.filter((event) => event.type === "turn_end").length; - expect(turns(early.events)).toBe(2); - expect(turns(late.events)).toBe(0); - await early.stream.stop(); - await late.stream.stop(); + // The early observer saw both generations; the late one starts with the held one, completing, and sees it leave. + const seen = (graphs: readonly TaskGraph[]) => + new Set(graphs.flatMap((graph) => generations(graph).map((node) => node.id))); + expect(seen(early.graphs).size).toBe(2); + expect(generations(late.graphs[0]!)).toEqual([ + expect.objectContaining({ id: first!.id, state: expect.objectContaining({ status: "completing" }) }), + ]); + expect(late.graphs.at(-1)!.tasks[String(first!.id)]).toBeUndefined(); + early.detach(); + late.detach(); await harness.close(context); }); @@ -1846,7 +1866,7 @@ describe("tool rounds and events", () => { defineTool({ name: "delegate", description: "Starts work in a conversation it owns", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit(async (tx) => { const child = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1858,7 +1878,6 @@ describe("tool rounds and events", () => { ); setup.faux.setResponses([toolCalls(["delegate", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); const submission = await root.submit({ type: "input", content: "go" }, context); await waitFor(async () => (await harness.snapshot(LiveDoc, root.id, context))?.tools?.[0]?.status === "done"); const live = (await harness.snapshot(LiveDoc, root.id, context))!; @@ -1866,16 +1885,14 @@ describe("tool rounds and events", () => { expect(live.tools![0]!.entry).toBeDefined(); expect((await state(harness, tool)).status).toBe("completing"); expect((await state(harness, live.run!.taskId)).status).toBe("waiting"); - await waitFor(() => labels(events).includes("end:c1:true")); expect(await settled(submission.wait(context))).toBe(false); open("sub"); expect((await submission.wait(context)).status).toBe("done"); expect(await outcomeOf(harness, tool)).toBe("completed"); - await stream.stop(); await harness.close(context); }); - it("holds a faulted tool's slot and task_failed until the work it owns drained", async () => { + it("holds a faulted tool's slot and its graph node until the work it owns drained", async () => { const setup = chatSetup(); addTask(setup.registry, Node); script("held", { @@ -1889,7 +1906,7 @@ describe("tool rounds and events", () => { defineTool({ name: "broken", description: "Starts owned work, then returns a result that is not strict JSON", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit((tx) => tx.createTask(Node, { name: "held" }, owned(api.taskId)), callContext); return { output: [], details: { fn: (() => 1) as unknown as JsonValue } }; @@ -1898,20 +1915,26 @@ describe("tool rounds and events", () => { ); setup.faux.setResponses([toolCalls(["broken", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); + const history = await viewHistory(harness, root.id); + const graph = await graphHistory(harness); const submission = await root.submit({ type: "input", content: "go" }, context); await waitFor(() => log.includes("abort:held")); const live = (await harness.snapshot(LiveDoc, root.id, context))!; const tool = live.tools![0]!.taskId!; expect(await state(harness, tool)).toMatchObject({ status: "completing", outcome: { status: "faulted" } }); expect(live.tools![0]!.status).not.toBe("done"); - expect(events.some((event) => event.type === "task_failed")).toBe(false); + // The graph shows the held fault; the task is not terminal yet. + expect(graph.graphs.at(-1)!.tasks[String(tool)]?.state).toEqual({ status: "completing", outcome: "faulted" }); open("abort.held"); expect((await submission.wait(context)).status).toBe("done"); expect(await outcomeOf(harness, tool)).toBe("faulted"); - await waitFor(() => events.some((event) => event.type === "task_failed")); - expect(labels(events)).toContain("end:c1:false"); - await stream.stop(); + await waitFor(() => graph.graphs.at(-1)!.tasks[String(tool)] === undefined); + // The slot ends done without a result entry. + const done = slotsOf(history.live(), "c1").find((slot) => slot?.status === "done"); + expect(done).toBeDefined(); + expect(done).not.toHaveProperty("entry"); + history.detach(); + graph.detach(); await harness.close(context); }); }); diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index 07afaa547..5ee8eac22 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -18,7 +18,7 @@ import { afterEach, describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, unanswered } from "./chat-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, LateResolvingStorage } from "./session-support.ts"; import { deferred } from "./task-support.ts"; const directories = new Set(); @@ -64,7 +64,7 @@ describe("submissions", () => { const submission = await root.submit({ type: "input", content: "hi" }, context); await busy.reached; const record = await submission.status(context); - if (record.status !== "placed") throw new Error(`Unexpected ${record.status}`); + if (record.type !== "input" || record.status !== "placed") throw new Error(`Unexpected ${record.status}`); const entry = await root.commit((tx) => tx.entry(UserEntry, record.entry), context); expect(entry?.model).toEqual([{ role: "user", content: "hi", timestamp: 42 }]); const live = await harness.snapshot(LiveDoc, root.id, context); @@ -163,9 +163,14 @@ describe("submissions", () => { } } const setup = chatSetup(); - setup.faux.setResponses([unanswered().step]); - const { harness, root } = await openChat(new HeldReads(), setup); - const submission = await root.submit({ type: "input", content: "hi" }, context); + setup.faux.setResponses([unanswered().step, unanswered().step]); + const storage = new HeldReads(); + const first = await openChat(storage, setup); + const { id } = await first.root.submit({ type: "input", content: "hi" }, context); + await first.harness.close(context); + // Reopen: the Session caches the submissions it writes, so only a reopened one reads it from Storage. + const { harness } = await openChat(storage.reopen(), setup); + const submission = (await harness.submission(id, context))!; hold = true; const waiting = submission.wait(context); await entered.promise; @@ -275,4 +280,23 @@ describe("submissions", () => { expect([UserEntry.kind, AssistantEntry.kind]).toEqual(["pi.user", "pi.assistant"]); await harness.close(context); }); + + it("resumes a wait only once observeCommits() observers saw the commit that settled it", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage("hi")]); + const { harness, root } = await openChat(new LateResolvingStorage(), setup); + const settled = new Set(); + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "submission" && change.value.status === "done") settled.add(change.value.id); + } + }, + end: () => {}, + }); + const submission = await root.submit({ type: "input", content: "hello" }, context); + await submission.wait(context); + expect(settled.has(submission.id)).toBe(true); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-submit-in-commit.test.ts b/packages/durable/test/harness-submit-in-commit.test.ts new file mode 100644 index 000000000..9f4e08d9e --- /dev/null +++ b/packages/durable/test/harness-submit-in-commit.test.ts @@ -0,0 +1,647 @@ +// Admission for hosts that must not start runs: `whenIdle: "reject"`, and `submit(tx, draft)` inside the caller's +// commit (#11149). + +import { withTelemetry } from "@earendil-works/chord/context"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, +} from "@earendil-works/pi-ai"; +import { + type Conversation, + ConversationBusy, + type ConversationHandle, + ConversationIdle, + defineDoc, + defineTask, + defineTool, + Harness, + InboxDoc, + LiveDoc, + MemoryStorage, + ReadAfterWrite, + type SubmissionId, + type Tx, + UserEntry, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { describe, expect, it } from "vitest"; +import { type ChatSetup, chatSetup, openChat, unanswered, waitFor } from "./chat-support.ts"; +import { addTask, addTool } from "./harness-support.ts"; +import { ControlledStorage, context } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +const MODEL = { provider: "faux", modelId: "faux-1" } as const; + +const failure = (): AssistantMessage => + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + +/** A host's own conversation document. */ +const Sent = defineDoc<{ ids: number[] }>({ + checkpointWhen: () => false, + kind: "test.sent", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ ids: [] }), +}); + +/** Open a chat whose run hangs until aborted, and wait until it works on its first input. */ +async function openBusy(storage = new ControlledStorage()) { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const opened = await openChat(storage, setup); + const first = await opened.root.submit({ type: "input", content: "work" }, context); + await busy.reached; + return { ...opened, storage, first }; +} + +/** Open a chat whose Harness records telemetry. */ +async function openTraced(setup: ChatSetup, telemetry: InMemoryTelemetryContext) { + const harness = await Harness.open( + new MemoryStorage(), + { models: setup.models, registry: setup.registry, settings: setup.settings, telemetry }, + context, + ); + const root = await harness.root(context, { agent: { model: MODEL } }); + return { harness, root }; +} + +/** Install tool `hold`, which reports its call and waits for `release`; its first call answers `tool`. */ +function addHold(setup: ChatSetup) { + const reached = deferred(); + const release = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Holds until released", + parameters: obj({}), + execute: async () => { + reached.resolve(); + await release.promise; + return { output: [] }; + }, + }), + ); + const call = fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }); + return { reached, release, call }; +} + +/** Settled status of submission `id`, polled without asking for progress (`Submission.wait()` would). */ +async function settledStatus(harness: Harness, id: SubmissionId) { + const submission = (await harness.submission(id, context))!; + await waitFor(async () => { + const { status } = await submission.status(context); + return status === "done" || status === "unanswered"; + }); + return submission.status(context); +} + +describe("whenIdle reject", () => { + // #11149 + it("rejects input to an idle conversation without writing", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const rejected = root.submit({ type: "input", content: "hi", whenBusy: "steer", whenIdle: "reject" }, context); + await expect(rejected).rejects.toBeInstanceOf(ConversationIdle); + await expect(rejected).rejects.toMatchObject({ conversationId: root.id }); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("rejects input to an idle conversation that keeps queued items after a failed run", async () => { + const setup = chatSetup(); + const failing = deferred(); + const requested = deferred(); + setup.faux.setResponses([ + async () => { + requested.resolve(); + await failing.promise; + return failure(); + }, + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await requested.promise; + const queued = await root.submit({ type: "input", content: "queued" }, context); + failing.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + await harness.waitForIdle(context); + const rejected = root.submit({ type: "input", content: "steer", whenBusy: "steer", whenIdle: "reject" }, context); + await expect(rejected).rejects.toBeInstanceOf(ConversationIdle); + expect(await queued.status(context)).toMatchObject({ status: "queued" }); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("steers a working run", async () => { + const { harness, root, first } = await openBusy(); + const steer = await root.submit( + { type: "input", content: "also this", whenBusy: "steer", whenIdle: "reject" }, + context, + ); + expect(await steer.status(context)).toMatchObject({ type: "input", status: "queued" }); + const inbox = await harness.snapshot(InboxDoc, root.id, context); + expect(inbox?.items).toEqual([{ id: steer.id, mode: "steer", content: "also this" }]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.inputs).toEqual([first.id]); + await harness.close(context); + }); + + // #11149 + it("returns the submission of a known request ID, also once idle", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "work", requestId: "r1" }, context); + await first.wait(context); + const draft = { type: "input", content: "work", requestId: "r1", whenBusy: "steer", whenIdle: "reject" } as const; + expect((await root.submit(draft, context)).id).toBe(first.id); + // Deduplication runs before the idle rejection in the caller's commit too. + expect(await harness.commit((tx) => root.submit(tx, draft), context)).toBe(first.id); + await harness.close(context); + }); + + // #11149 + it("rejects through a tool's conversation handle", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let rejection: unknown; + addTool( + setup.registry, + defineTool({ + name: "nudge", + description: "Steers another conversation", + parameters: obj({}), + execute: async (_args, api, callContext) => { + const handle = (await api.conversation(other!.id, callContext))!; + const draft = { type: "input", content: "nudge", whenBusy: "steer", whenIdle: "reject" } as const; + rejection = await handle.submit(draft, callContext).catch((error: unknown) => error); + return { output: [{ type: "text", text: "nudged" }] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("nudge", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("ok")]), + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(rejection).toBeInstanceOf(ConversationIdle); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); +}); + +describe("submit inside the caller's commit", () => { + // #11149 + it("admits with the host's own check and writes in one commit", async () => { + const { harness, root, storage, first } = await openBusy(); + const commits = storage.commits.length; + const id = await harness.commit(async (tx) => { + if ((await tx.doc(LiveDoc, root.id)).run === undefined) throw new Error("not working"); + // The host's own document write may come before the admission. + const sent = await tx.doc(Sent, root.id); + const admitted = await root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }); + sent.ids.push(admitted); + // The host's own table write comes after it. + await tx.appendEntry(root.id, { kind: "app.sent", data: { submission: admitted } }); + return admitted; + }, context); + expect(storage.commits.length).toBe(commits + 1); + const writes = storage.commits.at(-1)!; + expect(writes.some((write) => write.type === "submission" && write.value.id === id)).toBe(true); + expect(writes.some((write) => write.type === "entry" && write.value.kind === "app.sent")).toBe(true); + expect((await harness.snapshot(Sent, root.id, context))?.ids).toEqual([id]); + const inbox = await harness.snapshot(InboxDoc, root.id, context); + expect(inbox?.items).toEqual([{ id, mode: "steer", content: "steer" }]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.inputs).toEqual([first.id]); + await harness.close(context); + }); + + // #11149 + it("is rolled back with the commit when the callback throws later", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const failed = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "hi", requestId: "r1" }); + throw new Error("host check failed"); + }, context); + await expect(failed).rejects.toThrow("host check failed"); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + expect(await harness.commit((tx) => tx.submissionByRequest(root.id, "r1"), context)).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("leaves no submission and no run when Storage fails the commit, also after reopening", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup(); + const { harness, root } = await openChat(storage, setup); + const failure = new Error("disk full"); + storage.failNextCommit(failure); + const failed = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "hi", requestId: "r1" }); + await tx.appendEntry(root.id, { kind: "app.sent" }); + }, context); + await expect(failed).rejects.toBe(failure); + await harness.closed; + const reopened = await openChat(storage.reopen(), setup); + expect(reopened.root.id).toBe(root.id); + const found = await reopened.harness.commit(async (tx) => { + const submission = await tx.submissionByRequest(root.id, "r1"); + const tasks = await tx.scanTasks({ conversationId: root.id }, 10); + const entries = await tx.scanEntries({ conversationId: root.id }, 10); + return { submission, tasks: tasks.items, entries: entries.items }; + }, context); + expect(found).toEqual({ submission: undefined, tasks: [], entries: [] }); + expect((await reopened.harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await reopened.harness.close(context); + }); + + // #11149 + it("starts the run of an idle input once the commit is stored", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("answer")])]); + // No `resume()`: the admission asks for progress itself. + const { harness, root } = await openChat(new ControlledStorage(), setup); + const id = await harness.commit((tx) => root.submit(tx, { type: "input", content: "hi" }), context); + expect(await settledStatus(harness, id)).toMatchObject({ type: "input", status: "done" }); + await harness.close(context); + }); + + // #11149 + it("queues a model-visible write while busy and places it at the next boundary", async () => { + const setup = chatSetup(); + const hold = addHold(setup); + setup.faux.setResponses([hold.call, fauxAssistantMessage([fauxText("answer")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "work" }, context); + await hold.reached.promise; + const note = { role: "user", content: "a note the model sees", timestamp: 1 } as const; + const id = await harness.commit( + (tx) => root.submit(tx, { type: "write", entry: { kind: "test.note", model: [note] } }), + context, + ); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items.map((item) => item.id)).toEqual([id]); + hold.release.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "done" }); + expect(await settledStatus(harness, id)).toMatchObject({ + type: "write", + status: "done", + entry: expect.any(Number), + }); + await harness.close(context); + }); + + // #11149 + it("deduplicates request IDs", async () => { + const { harness, root, storage } = await openBusy(); + const draft = { type: "input", content: "steer", whenBusy: "steer", requestId: "r1" } as const; + const first = await harness.commit((tx) => root.submit(tx, draft), context); + const commits = storage.commits.length; + const again = await harness.commit((tx) => root.submit(tx, draft), context); + expect(again).toBe(first); + expect(storage.commits.length).toBe(commits); + await harness.close(context); + }); + + // #11149 + it("rejects before any write, so the callback may catch the rejection", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const outcome = await harness.commit(async (tx) => { + try { + await root.submit(tx, { type: "input", content: "hi", whenBusy: "steer", whenIdle: "reject" }); + return "admitted"; + } catch (error) { + return error instanceof ConversationIdle ? "idle" : "other"; + } + }, context); + expect(outcome).toBe("idle"); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("throws ReadAfterWrite after a table write of the commit, on every path", async () => { + const { harness, root } = await openBusy(); + // A busy input without request ID or trace reads no table, and is still refused. + const afterAppend = harness.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "app.note" }); + await root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }); + }, context); + await expect(afterAppend).rejects.toBeInstanceOf(ReadAfterWrite); + await expect(afterAppend).rejects.toThrow("submit(tx, draft) cannot read tables after the first table write"); + const second = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "one", whenBusy: "steer" }); + await root.submit(tx, { type: "input", content: "two", whenBusy: "steer" }); + }, context); + await expect(second).rejects.toBeInstanceOf(ReadAfterWrite); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items).toEqual([]); + + const idle = await openChat(new ControlledStorage(), chatSetup()); + const late = idle.harness.commit(async (tx) => { + await tx.appendEntry(idle.root.id, { kind: "app.note" }); + await idle.root.submit(tx, { type: "input", content: "hi" }); + }, context); + await expect(late).rejects.toBeInstanceOf(ReadAfterWrite); + await idle.harness.close(context); + await harness.close(context); + }); + + // #11149 + it("continues a failed run, and must precede the commit's table writes", async () => { + const setup = chatSetup(); + setup.faux.setResponses([failure, fauxAssistantMessage([fauxText("recovered")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "Analyze this" }, context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + + const late = harness.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "app.note" }); + await root.submit(tx, { type: "continue" }); + }, context); + await expect(late).rejects.toBeInstanceOf(ReadAfterWrite); + + const id = await harness.commit(async (tx) => { + const continued = await root.submit(tx, { type: "continue" }); + await tx.appendEntry(root.id, { kind: "app.note" }); + return continued; + }, context); + expect(await settledStatus(harness, id)).toMatchObject({ type: "continue", status: "done" }); + await harness.close(context); + }); + + // #11149 + it("rejects a busy continue and steers with whenIdle inside the commit", async () => { + const { harness, root } = await openBusy(); + const busy = harness.commit((tx) => root.submit(tx, { type: "continue" }), context); + await expect(busy).rejects.toBeInstanceOf(ConversationBusy); + const steer = await harness.commit( + (tx) => root.submit(tx, { type: "input", content: "steer", whenBusy: "steer", whenIdle: "reject" }), + context, + ); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items.map((item) => item.id)).toEqual([steer]); + await harness.close(context); + }); + + // #11149 + it("throws for a transaction of another Session, or a wrapped one", async () => { + const a = await openChat(new ControlledStorage(), chatSetup()); + const b = await openChat(new ControlledStorage(), chatSetup()); + const draft = { type: "input", content: "hi" } as const; + const foreign = b.harness.commit((tx) => a.root.submit(tx, draft), context); + await expect(foreign).rejects.toThrow("The transaction belongs to another Session"); + const proxied = a.harness.commit((tx) => a.root.submit(new Proxy(tx, {}), draft), context); + await expect(proxied).rejects.toThrow("The transaction belongs to another Session"); + const plain = a.harness.commit((tx) => a.root.submit({ doc: tx.doc.bind(tx) } as unknown as Tx, draft), context); + await expect(plain).rejects.toThrow("The transaction belongs to another Session"); + expect((await a.harness.snapshot(LiveDoc, a.root.id, context))?.run).toBeUndefined(); + await a.harness.close(context); + await b.harness.close(context); + }); + + // #11149 + it("records the trace carrier and links a steer to its caller and the running trace", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openTraced(setup, telemetry); + await root.submit({ type: "input", content: "work" }, context); + await busy.reached; + const run = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.generation")!.record; + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit( + (tx) => root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }), + withTelemetry(caller, context), + ), + ); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find( + (span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id, + )!; + expect(submit).toMatchObject({ parentId: null, attributes: { result: "queued" } }); + expect(submit.links).toEqual([ + { context: caller.context, attributes: {} }, + { context: run.trace, attributes: {} }, + ]); + expect((await (await harness.submission(id, context))!.status(context)).trace).toEqual(submit.context); + await harness.close(context); + }); + + // #11149 + it("starts an idle input's run in the admission's trace", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openTraced(setup, telemetry); + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit((tx) => root.submit(tx, { type: "input", content: "hi" }), withTelemetry(caller, context)), + ); + await busy.reached; + const submit = telemetry + .getSpans() + .find((span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id)!; + const record = await (await harness.submission(id, context))!.status(context); + expect(record.trace).toEqual(submit.context); + const run = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.generation")!.record; + expect(run.trace).toEqual(record.trace); + await harness.close(context); + }); + + // #11149 + it("makes a write a child of its caller and leaves the commit's events to the caller's commit", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const { harness, root } = await openTraced(setup, telemetry); + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit( + (tx) => root.submit(tx, { type: "write", entry: { kind: "app.note" } }), + withTelemetry(caller, context), + ), + ); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find((span) => span.name === "pi.durable.submit")!; + expect(submit).toMatchObject({ + parentContext: caller.context, + settled: true, + attributes: { "submission.id": id, result: "done" }, + events: [], + }); + expect(caller.events.map((event) => event.name)).toEqual(["pi.durable.entry", "pi.durable.submission"]); + await harness.close(context); + }); + + // #11149 + it("ends its span before the commit, and a later throw leaves no submission", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const { harness, root } = await openTraced(setup, telemetry); + await telemetry.startSpan({ name: "caller" }, async (caller) => { + const failed = harness.commit( + async (tx) => { + await root.submit(tx, { type: "write", entry: { kind: "app.note" } }); + throw new Error("later"); + }, + withTelemetry(caller, context), + ); + await expect(failed).rejects.toThrow("later"); + }); + const spans = telemetry.getSpans(); + const submit = spans.find((span) => span.name === "pi.durable.submit")!; + // The span reported an admission the commit then discarded. + expect(submit).toMatchObject({ settled: true, attributes: { result: "done" } }); + const id = submit.attributes["submission.id"] as SubmissionId; + expect(await harness.submission(id, context)).toBeUndefined(); + expect(spans.find((span) => span.name === "caller")!.events).toEqual([]); + await harness.close(context); + }); + + // #11149 + it("admits atomically from a tool's commit through its conversation handle", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let toolTask: number | undefined; + addTool( + setup.registry, + defineTool({ + name: "delegate", + description: "Delegates to another conversation", + parameters: obj({}), + execute: async (_args, api, callContext) => { + toolTask = api.taskId; + const handle = (await api.conversation(other!.id, callContext))!; + const id = await api.commit(async (tx) => { + const admitted = await handle.submit(tx, { type: "input", content: "from the tool" }); + await tx.appendEntry(other!.id, { kind: "app.delegated", data: { submission: admitted } }); + return admitted; + }, callContext); + return { output: [{ type: "text", text: `submitted ${id}` }] }; + }, + }), + ); + const reply: FauxResponseStep = (request) => { + const last = request.messages.at(-1); + if (last?.role === "user" && last.content === "delegate") { + return fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }); + } + return fauxAssistantMessage([fauxText("ok")]); + }; + setup.faux.setResponses([reply, reply, reply]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + const run = await root.submit({ type: "input", content: "delegate" }, context); + expect(await run.wait(context)).toMatchObject({ status: "done" }); + await harness.waitForIdle(context); + const entries = [...(await other.entries({}, 100, undefined, context)).items].reverse(); + const delegated = entries.find((entry) => entry.kind === "app.delegated"); + const submission = (delegated?.data as { submission: SubmissionId }).submission; + const settled = await settledStatus(harness, submission); + if (settled.type !== "input" || settled.status !== "done") throw new Error(`Unexpected ${settled.status}`); + // The placed input carries the tool task's byTaskId, as the commit's own appends do. + const user = await other.commit((tx) => tx.entry(UserEntry, settled.entry), context); + expect(user?.byTaskId).toBe(toolTask); + await harness.close(context); + }); + + // #11149 + it("admits inside a task's terminal commit", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("answer")])]); + const Admitting = defineTask({ + name: "test.admitting", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + const handle = (await runtime.conversation(runtime.conversationId, taskContext))!; + await runtime.commit(async (tx) => { + const id = await handle.submit(tx, { type: "input", content: "from the task" }); + return { status: "terminal", outcome: { status: "completed", result: id } }; + }, taskContext); + }, + }, + abort: (_task, runtime, taskContext) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), taskContext), + }); + addTask(setup.registry, Admitting); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const task = await root.commit( + (tx) => tx.createTask(Admitting, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + const settled = await harness.waitForTask(task, context); + if (settled.state.outcome.status !== "completed") throw new Error(settled.state.outcome.status); + const status = await settledStatus(harness, settled.state.outcome.result); + if (status.type !== "input" || status.status !== "done") throw new Error(`Unexpected ${status.status}`); + const user = await root.commit((tx) => tx.entry(UserEntry, status.entry), context); + expect(user?.byTaskId).toBe(task); + await harness.close(context); + }); + + // #11149 + it("rejects through a handle once its invocation is aborted, and once it ended", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let handle: ConversationHandle | undefined; + const started = deferred(); + const abortSeen = deferred(); + const release = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Holds until released, also when aborted", + parameters: obj({}), + execute: async (_args, api, callContext) => { + handle = (await api.conversation(other!.id, callContext))!; + callContext.abortSignal!.addEventListener("abort", () => abortSeen.resolve(), { once: true }); + started.resolve(); + await release.promise; + return { output: [] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("ok")]), + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + const run = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const tool = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.tool")!.record; + const aborting = harness.abortTask(tool.id, context); + await abortSeen.promise; + const draft = { type: "input", content: "late" } as const; + await expect(handle!.submit(draft, context)).rejects.toThrow(); + await expect(harness.commit((tx) => handle!.submit(tx, draft), context)).rejects.toThrow(); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + release.resolve(); + await aborting; + await run.wait(context); + await expect(harness.commit((tx) => handle!.submit(tx, draft), context)).rejects.toThrow(); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-support.ts b/packages/durable/test/harness-support.ts index 956b2e675..d2c443c45 100644 --- a/packages/durable/test/harness-support.ts +++ b/packages/durable/test/harness-support.ts @@ -7,7 +7,7 @@ import type { ToolResultMessage, UserMessage, } from "@earendil-works/pi-ai"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -23,10 +23,11 @@ import { section, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { context } from "./session-support.ts"; export function tool(name: string, description = `${name} tool`): ToolRegistration { - return defineTool({ name, description, parameters: Type.Object({}), execute: async () => ({ output: [] }) }); + return defineTool({ name, description, parameters: obj({}), execute: async () => ({ output: [] }) }); } /** Open a Harness with a fresh registry holding the named tools. */ @@ -40,7 +41,11 @@ export async function openHarness( registry.install(defineExtension({ name: "tools", tools: toolNames.map((name) => tool(name)) })); const harness = await Harness.open( storage, - { models: createModels(), registry, ...(options.onReport === undefined ? {} : { onReport: options.onReport }) }, + { + models: createModels(), + registry, + ...(options.onReport === undefined ? {} : { onReport: (report) => options.onReport?.(report.error) }), + }, context, ); return { harness, registry }; diff --git a/packages/durable/test/harness-task-graph.test.ts b/packages/durable/test/harness-task-graph.test.ts index 85dd66cd4..a23a95d49 100644 --- a/packages/durable/test/harness-task-graph.test.ts +++ b/packages/durable/test/harness-task-graph.test.ts @@ -5,15 +5,17 @@ import type { JsonValue } from "@earendil-works/chord"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { type ConversationId, + type ConversationView, defineTask, type EntryId, MemoryStorage, + type Observer, type TaskGraph, type TaskId, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; import { aborted, completed, deferred, eventually, openTasks } from "./task-support.ts"; type ParentState = { phase: "spawn" } | { phase: "join"; child: TaskId } | { phase: "finish" }; @@ -90,9 +92,11 @@ describe("task graph view", () => { const seen: TaskGraph[] = []; const observe = async () => { const opened = await harness.watchTaskGraph(context); - opened.start(async (value) => { - seen.push(value); - }); + opened.start( + onValue(async ({ value }) => { + seen.push(value); + }), + ); return opened; }; let graph = await observe(); @@ -215,11 +219,13 @@ describe("task graph view", () => { // Exact frames: replaying their operations from the acquisition revision gives each delivered value. let replica: JsonValue = watch.value as unknown as JsonValue; const frames: (readonly Op[])[] = []; - watch.start(async (value, ops) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - frames.push(ops); - }); + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + frames.push(ops); + }), + ); harness.resume(); await eventually(() => watch.value.tasks[String(background)]?.state.status === "running"); expect(await harness.abortTask(background, context)).toBe("marked"); @@ -334,9 +340,11 @@ describe("task graph view", () => { const second = await harness.watchTaskGraph(context); expect(second.value).toBe(first.value); const updates: string[] = []; - first.start(async (value) => { - updates.push(value.tasks[String(id)]?.state.status ?? "gone"); - }); + first.start( + onValue(async ({ value }) => { + updates.push(value.tasks[String(id)]?.state.status ?? "gone"); + }), + ); harness.resume(); await reached.promise; await flush(); @@ -356,4 +364,29 @@ describe("task graph view", () => { await rebuilt.stop(); await harness.close(context); }); + + it("advances with the conversation view at the publication of the one commit that changes both", async () => { + const storage = new ControlledStorage(); + const { Parent, Child } = family(gatesNever()); + const { harness } = await openTasks(storage, [Parent, Child]); + const root = await harness.root(context); + const seen: string[] = []; + const record = (name: string): Observer => ({ + advance: () => { + seen.push(`${name} adopting=${storage.adopting}`); + }, + end: () => {}, + }); + const view = await harness.observeConversation(root.id, record("view"), context); + const graph = await harness.observeTaskGraph(record("graph"), context); + await root.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "note" }); + await tx.createTask(Parent, null, { ownership: { kind: "conversation" } }); + }, context); + // Both at the one publication, after Storage stored it, in no promised order. + expect([...seen].sort()).toEqual(["graph adopting=false", "view adopting=false"]); + view.detach(); + graph.detach(); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-tasks-recovery.test.ts b/packages/durable/test/harness-tasks-recovery.test.ts index 29b98d8d9..07fcd7f27 100644 --- a/packages/durable/test/harness-tasks-recovery.test.ts +++ b/packages/durable/test/harness-tasks-recovery.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { createRegistry, defineDoc, @@ -15,6 +15,7 @@ import { type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { addTask, addTool } from "./harness-support.ts"; @@ -482,7 +483,7 @@ describe("blocked tasks", () => { defineTool({ name: "unrelated", description: "unrelated", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); @@ -644,11 +645,11 @@ describe("definition handover", () => { }); } - async function startHandover(log: string[], gates: Gates, storage = new MemoryStorage()) { + async function startHandover(log: string[], gates: Gates, storage = new MemoryStorage(), version = 1) { const registry = createRegistry(); - const old = addTask(registry, handoverTask("old", 1, log, { gates })); + const old = addTask(registry, handoverTask("old", version, log, { gates })); const opened = await openTasks(storage, [], { registry }); - const id = await createIn(opened.harness, handoverTask("old", 1, log)); + const id = await createIn(opened.harness, handoverTask("old", version, log)); opened.harness.resume(); await eventually(() => log.length === 1); return { ...opened, old, id }; @@ -734,7 +735,7 @@ describe("definition handover", () => { ); gate.resolve(); await eventually(() => infos.length === 1); - expect(infos).toMatchObject([{ type: "migration_failed", task: { id } }]); + expect(infos).toMatchObject([{ kind: "migration_failed", task: { id } }]); await flush(); expect(await harness.getTask(id, context)).toMatchObject({ version: 1, @@ -753,17 +754,30 @@ describe("definition handover", () => { gateA.resolve(); await eventually(() => log.includes("old:b start")); // A newer definition without a migration cannot take the task either. - addTask(registry, handoverTask("incompatible", 2, log)); + addTask(registry, handoverTask("unmigratable", 2, log)); gateB.resolve(); await harness.waitForTask(id, context); expect(log).toEqual(["old:a start", "old:a end", "old:b start", "old:b end", "old:c"]); expect(infos).toMatchObject([ - { type: "task_outdated", task: { id }, reason: "missing_task" }, - { type: "task_outdated", task: { id }, reason: "incompatible_task" }, + { kind: "task_outdated", task: { id }, reason: "missing_task" }, + { kind: "task_outdated", task: { id }, reason: "migration_failed" }, ]); await harness.close(context); }); + it("reports a replacement older than the running task as `task_too_old` and keeps the old definition", async () => { + const log: string[] = []; + const gate = deferred(); + const { harness, registry, id, infos } = await startHandover(log, { a: gate.promise }, new MemoryStorage(), 2); + // The same extension name replaces the running definition with an older version. + addTask(registry, handoverTask("older", 1, log)); + gate.resolve(); + await harness.waitForTask(id, context); + expect(log).toEqual(["old:a start", "old:a end", "old:b start", "old:b end", "old:c"]); + expect(infos).toMatchObject([{ kind: "task_outdated", task: { id }, reason: "task_too_old" }]); + await harness.close(context); + }); + it("rejects a runtime commit of the old invocation queued behind its handover commit", async () => { const log: string[] = []; const gate = deferred(); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index 23c9acec8..17bfe604f 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -1,7 +1,8 @@ import type { Context } from "@earendil-works/chord"; import { withAbortSignal, withCancel } from "@earendil-works/chord/context"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { + type CommitPublication, type Conversation, createRegistry, defineDoc, @@ -13,12 +14,16 @@ import { MemoryStorage, type RegistryReader, type RegistrySnapshot, + type StorageWrite, type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; +import type { SessionImpl } from "../src/session/session.ts"; import { addHooks, addTask, addTool, user } from "./harness-support.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, followCommits, LateResolvingStorage, onValue } from "./session-support.ts"; import { aborted, abortedWith, completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; type Step = { phase: "run" }; @@ -82,6 +87,20 @@ function withSignal(signal: AbortSignal): Context { return withAbortSignal(signal, context); } +/** Hold the Session line with a commit whose callback waits: the jobs queued after it start once it is released. */ +function holdLine(harness: { commit(change: () => Promise, context: Context): Promise }): { + readonly entered: Promise; + release(): void; +} { + const entered = deferred(); + const release = deferred(); + void harness.commit(async () => { + entered.resolve(); + await release.promise; + }, context); + return { entered: entered.promise, release: () => release.resolve() }; +} + describe("task phases", () => { it("continues one invocation through checkpoint progress and completes with a typed result", async () => { const seen: number[] = []; @@ -232,7 +251,15 @@ describe("task phases", () => { }, abort: async () => {}, }); - const { harness, root } = await openRoot([Writer, Child]); + class CountingStorage extends MemoryStorage { + documentScans = 0; + override scanDocuments(...args: Parameters) { + this.documentScans++; + return super.scanDocuments(...args); + } + } + const storage = new CountingStorage(); + const { harness, root } = await openRoot([Writer, Child], { storage }); const conversation = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); const id = await conversation.commit( (tx) => tx.createTask(Writer, null, { ownership: { kind: "conversation" } }), @@ -249,6 +276,10 @@ describe("task phases", () => { expect(entry?.conversationId).toBe(conversation.id); expect(progressSeen).toEqual(["wrote"]); expect(await harness.snapshot(Progress, id, context)).toBeUndefined(); + // The Session indexed the documents of the tasks it created, so no terminal commit scanned for them, and it dropped + // each task's entry at its end. + expect(storage.documentScans).toBe(0); + expect((harness as unknown as SessionImpl).documentCacheSizes.taskDocuments).toBe(0); expect(await harness.getTask(id, context)).toEqual(receipt); expect((await harness.waitForTask(childId!, context)).conversationId).toBe(conversation.id); expect(root.id).not.toBe(conversation.id); @@ -394,7 +425,7 @@ describe("task phases", () => { defineTool({ name: "late", description: "late", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); @@ -545,9 +576,11 @@ describe("task runtime", () => { handlerContext = ctx; absent = await runtime.watchDoc(Absent, ctx); const watch = await runtime.watchDoc(Notes, ctx); - watch!.start(async (value) => { - delivered.push(value?.text ?? "retired"); - }); + watch!.start( + onValue(async ({ value }) => { + delivered.push(value.text); + }), + ); watchClosed = watch?.closed; await runtime.commit(() => completed(null), ctx); }); @@ -560,7 +593,7 @@ describe("task runtime", () => { await harness.waitForTask(id, context); expect(absent).toBeUndefined(); // The watch stops when the step after the phase ends the invocation; later commits deliver nothing. - expect(await watchClosed).toEqual({ reason: "stopped" }); + expect(await watchClosed).toEqual({ kind: "stopped" }); await harness.commit(async (tx) => { (await tx.doc(Notes)).text = "after"; }, context); @@ -709,7 +742,7 @@ describe("task scheduling", () => { storage.failNextCommit(failure); harness.resume(); const end = await harness.closed; - expect(end).toEqual({ reason: "failed", error: failure }); + expect(end).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); // The pending wait and every later call get SessionFailed with the storage error. await expect(waiting).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); @@ -740,7 +773,7 @@ describe("task scheduling", () => { const { harness, root, reports } = await openRoot([Throws], { storage }); const id = await start(root, Throws); harness.resume(); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(runs).toBe(1); expect(reports).toEqual([failure]); await harness.close(context); @@ -1198,18 +1231,16 @@ describe("task close", () => { await reached.promise; await harness.close(context); await expect(fromListener).rejects.toThrow("closed"); - expect(watchEnd).toEqual({ reason: "ended" }); + expect(watchEnd).toEqual({ kind: "ended" }); }); it("writes no fault when close seals while the step after a failed phase is queued", async () => { const storage = new ControlledStorage(); - let held: ReturnType | undefined; + let held: ReturnType | undefined; let harnessRef: Awaited>["harness"] | undefined; - const Throws = oneStep("test.close-fault", async (_task, runtime) => { - held = storage.holdCommits(); - void harnessRef!.commit(async (tx) => { - await tx.appendEntry(runtime.conversationId, { kind: "blocker" }); - }, context); + const Throws = oneStep("test.close-fault", async () => { + // The step after this phase queues behind a job that holds the line. + held = holdLine(harnessRef!); throw new Error("would fault"); }); const { harness, root } = await openRoot([Throws], { storage }); @@ -1264,14 +1295,11 @@ describe("task close", () => { it("rejects a runtime commit that was queued on the line when close sealed it", async () => { const storage = new ControlledStorage(); let queued: Promise | undefined; - let held: ReturnType | undefined; + let held: ReturnType | undefined; let harnessRef: Awaited>["harness"] | undefined; const proceed = deferred(); const Queued = oneStep("test.close-queued-commit", async (_task, runtime) => { - held = storage.holdCommits(); - void harnessRef!.commit(async (tx) => { - await tx.appendEntry(runtime.conversationId, { kind: "blocker" }); - }, context); + held = holdLine(harnessRef!); queued = runtime.commit(() => completed(null), context).catch((error: unknown) => error); // Ignores the close signal, so the invocation is still alive when the queued commit reaches the line. await proceed.promise; @@ -1364,3 +1392,310 @@ describe("task close", () => { expect(storage.commits.at(-1)).toMatchObject([{ type: "task", value: { id, state: { status: "pending" } } }]); }); }); + +describe("publication at adoption", () => { + /** Both storages: one that resolves right after adopting, one that still works after it (JSONL's sidecar reclaim). */ + const storages = [ + { name: "memory", open: () => new ControlledStorage() }, + { name: "work after adopt", open: () => new LateResolvingStorage() }, + ] as const; + + /** Whether `publication` makes task `id` terminal. */ + const terminal = (publication: CommitPublication, id: TaskId): boolean => + publication.changes.some( + (change) => change.type === "task" && change.value.id === id && change.value.state.status === "terminal", + ); + + /** Whether `writes` make a task terminal. */ + const endsTask = (writes: readonly StorageWrite[]): boolean => + writes.some((write) => write.type === "task" && write.value.state.status === "terminal"); + + it("resumes task and idle waits only once observeCommits() observers saw the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.wake-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage: new LateResolvingStorage() }); + const seen = new Set(); + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") seen.add(change.value.id); + } + }, + end: () => {}, + }); + const id = await start(root, Gated); + harness.resume(); + const task = harness.waitForTask(id, context).then(() => seen.has(id)); + const idle = harness.waitForIdle(context).then(() => seen.has(id)); + const conversationIdle = root.waitForIdle(context).then(() => seen.has(id)); + gate.resolve(); + expect(await Promise.all([task, idle, conversationIdle])).toEqual([true, true, true]); + await harness.close(context); + }); + + for (const { name, open } of storages) { + it(`runs host listeners before the waits a terminal commit resolves (${name})`, async () => { + const storage = open(); + const gate = deferred(); + const Gated = gated("test.order-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, + }); + harness.resume(); + const waited = harness.waitForTask(id, context).then(() => order.push("wait")); + gate.resolve(); + await waited; + expect(order).toEqual(["host adopting=false", "wait"]); + await harness.close(context); + }); + + it(`signals a newly marked run after host listeners saw the mark, outside the storage's adopt (${name})`, async () => { + const storage = open(); + const reached = deferred(); + const order: string[] = []; + const Signalled = oneStep("test.order-signalled", async (_task, runtime) => { + runtime.signal.addEventListener("abort", () => order.push(`abort adopting=${storage.adopting}`)); + reached.resolve(); + await aborted(runtime.signal); + }); + const { harness, root } = await openRoot([Signalled], { storage }); + const id = await start(root, Signalled); + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) + if (change.type === "task" && change.value.id === id && change.value.abortRequested) + order.push("host"); + }, + end: () => {}, + }); + harness.resume(); + await reached.promise; + expect(await harness.abortTask(id, context)).toBe("marked"); + expect(order).toEqual(["host", "abort adopting=false"]); + await harness.close(context); + }); + } + + it("resolves a waitForIdle() called while the storage stores the commit at its publication, after host listeners", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.gap-idle", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) order.push("host"); + }, + end: () => {}, + }); + const gapped = deferred(); + let idle: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + idle = harness.waitForIdle(context).then(() => order.push(`idle committing=${storage.committing}`)); + gapped.resolve(); + return true; + }; + harness.resume(); + gate.resolve(); + await gapped.promise; + await idle; + expect(order).toEqual(["host", "idle committing=false"]); + await harness.close(context); + }); + + it("publishes a commit the storage rejects after adopting it, resolves its waits, then fails the Session", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const gate = deferred(); + const Gated = gated("test.reject-after-adopt", gate.promise); + const { harness, root, infos } = await openRoot([Gated], { storage, telemetry }); + const id = await start(root, Gated); + const seen: string[] = []; + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) seen.push("terminal"); + }, + end: () => {}, + }); + storage.rejectAfterAdopt = endsTask; + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + expect(seen).toEqual(["terminal"]); + expect(await harness.closed).toEqual({ kind: "failed", error: new Error("reclaim failed") }); + expect(infos).toContainEqual({ kind: "storage_failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("counts a task created by a commit the storage rejects after adopting it, and its close balances it", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const Idle = oneStep("test.reject-created", async () => {}); + const { harness, root } = await openRoot([Idle], { storage, telemetry }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "task"); + await expect(start(root, Idle)).rejects.toThrow("reclaim failed"); + expect(await harness.closed).toMatchObject({ kind: "failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("keeps a task wait resolved when a host listener closes the Harness on the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.close-in-listener", gate.promise); + const { harness, root } = await openRoot([Gated]); + const id = await start(root, Gated); + let closing: Promise | undefined; + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) closing = harness.close(context); + }, + end: () => {}, + }); + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + await closing; + }); + + it("ends a task wait when the Harness closes while the terminal commit is stored, and still stores it", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.close-in-gap", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const id = await start(root, Gated); + const gapped = deferred(); + let closing: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + closing = harness.close(context); + gapped.resolve(); + return true; + }; + harness.resume(); + const waited = harness.waitForTask(id, context).then( + (record) => record.state.outcome, + (error: unknown) => error, + ); + gate.resolve(); + await gapped.promise; + await closing; + // Close ends the waits before it drains the commit in flight, which is published and stored all the same. + expect(await waited).toMatchObject({ message: expect.stringContaining("closed") }); + expect(await harness.closed).toEqual({ kind: "closed" }); + const reopened = await openRoot([Gated], { storage: storage.reopen() }); + expect((await reopened.harness.getTask(id, context))?.state.status).toBe("terminal"); + await reopened.harness.close(context); + }); + + it("fails the Session with the storage error when it rejects after adopting, and reports an observer's throw", async () => { + const storage = new LateResolvingStorage(); + const { harness, root, reports, infos } = await openRoot([], { storage }); + const bug = new Error("observer bug"); + followCommits(harness, (publication) => { + if (publication.kind === "stored" && publication.changes.some((change) => change.type === "entry")) throw bug; + }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "entry"); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "reclaim failed", + ); + expect(await harness.closed).toEqual({ kind: "failed", error: new Error("reclaim failed") }); + expect(reports).toContain(bug); + expect(infos[reports.indexOf(bug)]).toEqual({ kind: "listener_error" }); + expect(infos).toContainEqual({ kind: "storage_failed" }); + }); + + it("stores a host listener's own commit before the reconcile commit that the same publication asks for", async () => { + const storage = new ControlledStorage(); + const gate = deferred(); + const Child = gated("test.order-child", gate.promise); + const Parent = oneStep("test.order-parent", async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + await tx.createTask(Child, null, { ownership: { kind: "task", taskId: task.id } }); + }, ctx); + // The child is still live, so the outcome is held as `completing` until a reconcile pass finalizes it. + await runtime.commit(() => completed(null), ctx); + }); + const { harness, root } = await openRoot([Parent, Child], { storage }); + const parent = await start(root, Parent); + harness.resume(); + await eventually(async () => (await harness.getTask(parent, context))?.state.status === "completing"); + harness.observeCommits({ + advance: (publication) => { + const child = publication.changes.find( + (change) => + change.type === "task" && change.value.id !== parent && change.value.state.status === "terminal", + ); + if (child !== undefined) + void root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + }, + end: () => {}, + }); + const before = storage.commits.length; + gate.resolve(); + await harness.waitForTask(parent, context); + const kinds = storage.commits.slice(before).map((writes) => { + if (writes.some((write) => write.type === "entry")) return "host entry"; + const task = writes.find((write) => write.type === "task"); + if (task?.type !== "task" || task.value.state.status !== "terminal") return "other"; + return task.value.id === parent ? "parent terminal" : "child terminal"; + }); + expect(kinds.filter((kind) => kind !== "other")).toEqual(["child terminal", "host entry", "parent terminal"]); + await harness.close(context); + }); + + it("calls a host listener added on a Harness during a publication only when one was registered at it", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openRoot([], { storage }); + const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + const late: number[] = []; + let stopLate = { detach: (): void => {} }; + // Runs once, during the next publication, after its dispatch was queued or not. + let during: (() => void) | undefined; + followCommits(harness, () => { + const run = during; + during = undefined; + run?.(); + }); + // No host listener at the publication, although the Harness's own observers run: no dispatch, so the late one + // misses it. + during = () => { + stopLate = harness.observeCommits({ + advance: (publication) => { + late.push(publication.seq); + }, + end: () => {}, + }); + }; + await note(); + await flush(); + expect(late).toEqual([]); + await note(); + expect(late).toHaveLength(1); + stopLate.detach(); + // One registered at adopt: the dispatch reads the listeners when it runs, the late one included. + harness.observeCommits({ advance: () => {}, end: () => {} }); + const second: number[] = []; + during = () => { + harness.observeCommits({ + advance: (publication) => { + second.push(publication.seq); + }, + end: () => {}, + }); + }; + await note(); + expect(second).toHaveLength(1); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index 20bf7f15f..58bcbc9e8 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -1,5 +1,5 @@ import { withTelemetry } from "@earendil-works/chord/context"; -import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CompactionTask, @@ -11,13 +11,19 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; -import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { obj, str } from "@earendil-works/pi-schema"; +import { + InMemoryTelemetryContext, + NOOP_TELEMETRY_CONTEXT, + NOOP_TELEMETRY_METRICS, + type TelemetryContext, +} from "@earendil-works/pi-telemetry"; import { describe, expect, it, vi } from "vitest"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { type ChatSetup, chatSetup, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; -import { aborted } from "./task-support.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; +import { aborted, openTasks } from "./task-support.ts"; async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemoryTelemetryContext) { const harness = await Harness.open( @@ -28,7 +34,7 @@ async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemo settings: setup.settings, now: () => setup.now(), telemetry, - onReport: (error) => setup.reports.push(error), + onReport: ({ error }) => setup.reports.push(error), }, context, ); @@ -48,7 +54,7 @@ describe("Durable telemetry", () => { defineTool({ name: "inner", description: "inner", - parameters: Type.Object({ secret: Type.String() }), + parameters: obj({ secret: str() }), execute: async () => ({ output: [{ type: "text", text: "private-result" }] }), }), ); @@ -57,7 +63,7 @@ describe("Durable telemetry", () => { defineTool({ name: "outer", description: "outer", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { await api.executeTool("inner", { secret: "private-argument" }, ctx); return { output: [{ type: "text", text: "private-output" }] }; @@ -342,7 +348,7 @@ describe("Durable telemetry", () => { defineTool({ name: "hold", description: "hold", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { reached.resolve(); await gate.promise; @@ -410,7 +416,7 @@ describe("Durable telemetry", () => { defineTool({ name: "delegate", description: "delegate", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { const child = await api.commit( (tx) => tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }), @@ -717,8 +723,8 @@ describe("Durable telemetry", () => { phases: { report: async (_task, runtime, ctx) => { runtime.report(error); - runtime.report(error, { type: "extension", extension: "reporting-extension" }); - runtime.report(error, { type: "host", callback: "models" }); + runtime.report(error, { kind: "extension", extension: "reporting-extension" }); + runtime.report(error, { kind: "host", callback: "models" }); await runtime.commit( () => ({ status: "terminal", outcome: { status: "completed", result: null } }), ctx, @@ -851,7 +857,7 @@ describe("Durable telemetry", () => { .getMetrics() .filter((metric) => metric.name === "pi.durable.reports") .map((metric) => metric.attributes), - ).toEqual([{ type: "session_failed" }]); + ).toEqual([{ type: "storage_failed" }]); }); it("counts reports raised with a NOOP span without creating a root", async () => { @@ -859,8 +865,11 @@ describe("Durable telemetry", () => { const telemetry = new InMemoryTelemetryContext(); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const error = new Error("private-noop-report"); - const unsubscribe = harness.subscribeCommits(() => { - throw error; + const observation = harness.observeCommits({ + advance: () => { + throw error; + }, + end: () => {}, }); await root.commit((tx) => tx.appendEntry(root.id, { kind: "app.note" }), context); expect(setup.reports).toEqual([error]); @@ -871,7 +880,7 @@ describe("Durable telemetry", () => { .filter((metric) => metric.name === "pi.durable.reports") .map((metric) => metric.attributes), ).toEqual([{ type: "listener_error" }]); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -881,9 +890,12 @@ describe("Durable telemetry", () => { setup.faux.setResponses([fauxAssistantMessage("answer")]); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const error = new Error("private-listener-error"); - const unsubscribe = harness.subscribeCommits((publication) => { - if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) - throw error; + const observation = harness.observeCommits({ + advance: (publication) => { + if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) + throw error; + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "input" }, context)).wait(context); await harness.waitForIdle(context); @@ -892,7 +904,7 @@ describe("Durable telemetry", () => { const span = telemetry.getSpans().find((span) => span.name === "pi.durable.submit")!; expect(span.events).toContainEqual({ name: "pi.durable.report", attributes: { type: "listener_error" } }); expect(JSON.stringify(telemetry.getSpans())).not.toContain("private-listener-error"); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -902,9 +914,12 @@ describe("Durable telemetry", () => { setup.faux.setResponses([fauxAssistantMessage("answer")]); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const failed = Promise.withResolvers(); - const unsubscribe = harness.subscribeCommits((publication) => { - if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) - return failed.promise; + const observation = harness.observeCommits({ + advance: (publication) => { + if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) + return failed.promise; + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "input" }, context)).wait(context); await harness.waitForIdle(context); @@ -920,7 +935,7 @@ describe("Durable telemetry", () => { .map((metric) => metric.attributes), ).toEqual([{ type: "listener_error" }]); expect(JSON.stringify(telemetry.getSpans())).not.toContain("private-async-listener-error"); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -938,7 +953,7 @@ describe("Durable telemetry", () => { defineTool({ name: "once", description: "once", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { runs++; return { output: [] }; @@ -951,7 +966,7 @@ describe("Durable telemetry", () => { name: "hold", description: "hold", replay: "safe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, _api, ctx) => { if (!recover) { reached.resolve(); @@ -985,4 +1000,93 @@ describe("Durable telemetry", () => { await harness.close(context); await sandbox.close(); }); + + it("reports telemetry adapters that throw while a commit is published, and the commit stands", async () => { + const Idle = defineTask({ + name: "test.telemetry-idle", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: async () => {} }, + abort: async () => {}, + }); + const throwingMetric = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, upDownCounter: () => throwingMetric }, + }; + const { harness, infos } = await openTasks(new MemoryStorage(), [Idle], { telemetry }); + const root = await harness.root(context); + const view = await root.watch(context); + const shown: (number | undefined)[] = []; + view.start( + onValue(async ({ value }) => { + shown.push(value.entries.at(-1)?.id); + }), + ); + const hosts: number[] = []; + harness.observeCommits({ + advance: (publication) => { + hosts.push(publication.seq); + }, + end: () => {}, + }); + const failing = withTelemetry( + { + ...NOOP_TELEMETRY_CONTEXT, + addEvent: () => { + throw new Error("telemetry down"); + }, + }, + context, + ); + const { entry, task } = await root.commit(async (tx) => { + const entry = await tx.appendEntry(root.id, { kind: "note", data: {} }); + const task = await tx.createTask(Idle, null, { ownership: { kind: "conversation" } }); + return { entry: entry.id, task }; + }, failing); + // The entry event, then the active-task metric; the report's own event throws too and is dropped. + expect(infos).toEqual([ + { kind: "host_error", callback: "telemetry" }, + { kind: "host_error", callback: "telemetry" }, + ]); + expect(hosts).toHaveLength(1); + await flush(); + expect(shown).toEqual([entry]); + expect((await harness.getTask(task, context))?.state.status).toBe("pending"); + await view.stop(); + await harness.close(context); + expect(await harness.closed).toEqual({ kind: "closed" }); + }); + + it("drops a report's own metric that throws, after onReport got the report", async () => { + const throwingCounter = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, counter: () => throwingCounter }, + }; + const { harness, reports, infos } = await openTasks(new MemoryStorage(), [], { telemetry }); + const root = await harness.root(context); + const bug = new Error("listener bug"); + harness.observeCommits({ + advance: () => { + throw bug; + }, + end: () => {}, + }); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + // The listener runs in the commit's dispatch microtask; a throw escaping its report would be unhandled there. + await flush(); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ kind: "listener_error" }]); + await harness.close(context); + expect(await harness.closed).toEqual({ kind: "closed" }); + }); }); diff --git a/packages/durable/test/harness-tool-hooks.test.ts b/packages/durable/test/harness-tool-hooks.test.ts new file mode 100644 index 000000000..beca471e1 --- /dev/null +++ b/packages/durable/test/harness-tool-hooks.test.ts @@ -0,0 +1,806 @@ +// `afterTool` runs on every result a tool task writes, also those the Harness decides without executing the tool +// (#11043). + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, +} from "@earendil-works/pi-ai"; +import { + defineTool, + type EntryRecord, + type Harness, + LiveDoc, + MemoryStorage, + type NestedToolExecutionResult, + ResetEntry, + ROOT_CONVERSATION_ID, + type Storage, + type TaskId, + type ToolExecutionResult, + type ToolHookCall, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { bool, obj, opt, str } from "@earendil-works/pi-schema"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, waitFor } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-hooks-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over the SQLite file at `path`; `kill()` simulates a crash: every later storage call hangs. */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + opened.harness.resume(); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +/** Let deferred progress commits (`persistWithinMs: 0`) reach Storage before a crash. */ +function stored(): Promise { + return new Promise((resolve) => setTimeout(resolve, 50)); +} + +const Parameters = obj({ text: opt(str()) }); + +function tool( + name: string, + execute: ToolRegistration["execute"], + extra: Partial> = {}, +): ToolRegistration { + return defineTool({ name, description: name, parameters: Parameters, execute, ...extra }); +} + +function calls(...list: readonly (readonly [string, Record, string])[]): AssistantMessage { + return fauxAssistantMessage( + list.map(([name, args, id]) => fauxToolCall(name, args, { id })), + { stopReason: "toolUse" }, + ); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +function results(entries: readonly EntryRecord[]): Map { + const messages = entries + .filter((entry) => ToolResultEntry.is(entry)) + .map((entry) => entry.model![0] as ToolResultMessage); + return new Map(messages.map((message) => [message.toolCallId, message])); +} + +function text(message: ToolResultMessage | undefined): string { + return (message?.content ?? []).map((item) => (item.type === "text" ? item.text : "")).join("|"); +} + +function codes(result: { readonly diagnostics?: readonly { readonly code?: string }[] }): (string | undefined)[] { + return (result.diagnostics ?? []).map((diagnostic) => diagnostic.code); +} + +async function diagnosticsOf(entries: readonly EntryRecord[], callId: string): Promise<(string | undefined)[]> { + const entry = entries.find( + (each) => ToolResultEntry.is(each) && (each.model![0] as ToolResultMessage).toolCallId === callId, + ); + return codes((entry?.data ?? {}) as { diagnostics?: { code?: string }[] }); +} + +type Seen = { readonly call: ToolHookCall; readonly result: ToolExecutionResult }; + +async function toolTaskId(harness: Harness, index = 0): Promise { + let id: TaskId | undefined; + await waitFor(async () => { + id = (await harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context))?.tools?.[index]?.taskId; + return id !== undefined; + }); + return id!; +} + +describe("afterTool on results the call phase decides", () => { + // #11043 + it("sees and replaces unavailable, invalid, and blocked results, with the raw arguments", async () => { + const setup = chatSetup(); + let ran = 0; + const echo = tool("echo", async () => { + ran++; + return { output: [{ type: "text", text: "ran" }] }; + }); + addTool(setup.registry, echo); + addTool( + setup.registry, + tool("gone", async () => ({ output: [] })), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => { + if (call.id === "block") return { block: "not today" }; + if (call.id === "rewrite") return { arguments: { text: { not: "a string" } } }; + return undefined; + }, + afterTool: (call, result) => { + seen.push({ call, result }); + return { ...result, output: [{ type: "text", text: `hooked ${call.id}` }], details: { by: "hook" } }; + }, + }); + let root!: Awaited>["root"]; + setup.faux.setResponses([ + async () => { + // The tool is offered but no longer resolves when its task runs. + await root.configure({ tools: [echo] }, context); + return calls( + ["gone", {}, "unavailable"], + ["echo", { text: { not: "a string" } }, "invalid"], + ["echo", { text: "x" }, "rewrite"], + ["echo", {}, "block"], + ); + }, + DONE, + ]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(ran).toBe(0); + const byId = new Map(seen.map((each) => [each.call.id, each])); + expect([...byId.keys()].sort()).toEqual(["block", "invalid", "rewrite", "unavailable"]); + expect(codes(byId.get("unavailable")!.result)).toEqual(["tool_unavailable"]); + expect(codes(byId.get("invalid")!.result)).toEqual(["invalid_arguments"]); + expect(codes(byId.get("rewrite")!.result)).toEqual(["invalid_arguments"]); + expect(codes(byId.get("block")!.result)).toEqual(["blocked"]); + // Hooks see the call as the model made it, also when it is invalid or `beforeTool` rewrote it. + expect(byId.get("invalid")!.call.arguments).toEqual({ text: { not: "a string" } }); + expect(byId.get("rewrite")!.call.arguments).toEqual({ text: "x" }); + const entries = await allEntries(root); + const messages = results(entries); + for (const id of ["unavailable", "invalid", "rewrite", "block"]) { + const message = messages.get(id)!; + expect(message).toMatchObject({ isError: true, details: { by: "hook" } }); + expect(text(message).startsWith(`hooked ${id}|`)).toBe(true); + } + await opened.harness.close(context); + }); + + // #11043 + it("bounds a replacement output with the tool's limits, or the defaults for an unresolved tool", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("small", async () => ({ output: [] }), { outputLimits: { maxBytes: 10 } }), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: () => ({ block: "no" }), + afterTool: (call, result) => ({ + ...result, + output: [{ type: "text", text: call.id === "c2" ? "x".repeat(100) : "y".repeat(60 * 1024) }], + }), + }); + let root!: Awaited>["root"]; + setup.faux.setResponses([ + async () => { + await root.configure({ tools: [] }, context); + return calls(["small", {}, "c1"]); + }, + DONE, + ]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.configure({ tools: null }, context); + setup.faux.setResponses([calls(["small", {}, "c2"]), DONE]); + await (await root.submit({ type: "input", content: "again" }, context)).wait(context); + const entries = await allEntries(root); + const messages = results(entries); + // Unresolved: the default 50 KB. + const unresolved = messages.get("c1")!.content[0]; + expect(unresolved?.type === "text" && unresolved.text.length).toBe(50 * 1024); + expect(await diagnosticsOf(entries, "c1")).toEqual(["tool_unavailable", "truncated"]); + const bounded = messages.get("c2")!.content[0]; + expect(bounded?.type === "text" && bounded.text).toBe("x".repeat(10)); + expect(await diagnosticsOf(entries, "c2")).toEqual(["blocked", "truncated"]); + await opened.harness.close(context); + }); + + // #11043 + it("lets a hook's terminate on a blocked call count toward the round", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [], control: { terminate: true } })), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.id === "c2" ? { block: "no" } : undefined), + afterTool: (_call, result) => ({ ...result, control: { terminate: true } }), + }); + setup.faux.setResponses([calls(["echo", {}, "c1"], ["echo", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + // The round terminated: no second request. + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("runs on nested invalid and blocked calls; the caller gets diagnostics and details, not output", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [] })), + ); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("echo", { text: { not: "a string" } }, callContext)); + nested.push(await api.executeTool("echo", { text: "secret" }, callContext)); + nested.push(await api.executeTool("missing", {}, callContext)); + return {}; + }), + ); + const seen: ToolHookCall[] = []; + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.arguments.text === "secret" ? { block: "no secrets" } : undefined), + afterTool: (call, result) => { + if (call.parent === undefined) return undefined; + seen.push(call); + return { + ...result, + output: [{ type: "text", text: "dropped" }], + structuredOutput: { dropped: true }, + details: { hooked: call.id }, + diagnostics: [...(result.diagnostics ?? []), { severity: "info", code: "hooked", message: "seen" }], + }; + }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(seen.map((call) => call.id)).toEqual(["c1/1", "c1/2", "c1/3"]); + expect(seen.every((call) => call.parent?.callId === "c1")).toBe(true); + expect(nested.map((result) => [result.isError, codes(result)])).toEqual([ + [true, ["invalid_arguments", "hooked"]], + [true, ["blocked", "hooked"]], + [true, ["tool_unavailable", "hooked"]], + ]); + expect(nested.map((result) => result.details)).toEqual([ + { hooked: "c1/1" }, + { hooked: "c1/2" }, + { hooked: "c1/3" }, + ]); + // A Harness error result has no structured output, even when a hook supplies one. + expect(nested[1]!.structuredOutput).toBeUndefined(); + expect(nested[2]!.structuredOutput).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("gives a nested call a hook turns into a success its output as value, unless the tool has a schema", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("plain", async () => ({ output: [] })), + ); + addTool( + setup.registry, + tool("typed", async () => ({ output: [], structuredOutput: { ok: true } }), { + structuredOutputSchema: obj({ ok: bool() }), + }), + ); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("plain", {}, callContext)); + nested.push(await api.executeTool("typed", {}, callContext)); + return {}; + }), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.parent === undefined ? undefined : { block: "no" }), + afterTool: (call) => + call.parent === undefined + ? undefined + : { output: [{ type: "text", text: `fine ${call.name}` }], structuredOutput: { ok: false } }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(nested.map((result) => [result.isError, result.structuredOutput])).toEqual([ + [false, "fine plain"], + [false, undefined], + ]); + await harness.close(context); + }); +}); + +describe("afterTool on results rebuilt from the slot", () => { + /** A tool that writes output, then blocks until its invocation is cancelled. */ + function blocking(name: string, extra: Partial> = {}) { + const started = deferred(); + const state = { runs: 0 }; + const registration = tool( + name, + async (_args, api, callContext) => { + state.runs++; + api.output(`${name} ${"o".repeat(40)}\n`); + await api.details({ run: state.runs }, callContext); + started.resolve(); + return aborted(callContext.abortSignal!); + }, + extra, + ); + return { registration, started, state }; + } + + // #11043 + it("runs on an interrupted call; a replaced output loses the slot's truncation diagnostic", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const limits = { outputLimits: { maxBytes: 10, retain: "tail" } } as const; + const work = blocking("work", limits); + const keep = blocking("keep", limits); + addTool(setup.registry, work.registration); + addTool(setup.registry, keep.registration); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return call.name === "work" ? { ...result, output: [{ type: "text", text: "redacted" }] } : undefined; + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"], ["keep", {}, "c2"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + await keep.started.promise; + await stored(); + opened.kill(); + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen.map((each) => [each.call.id, codes(each.result)]).sort()).toEqual([ + ["c1", ["interrupted"]], + ["c2", ["interrupted"]], + ]); + const entries = await allEntries(opened.root); + const messages = results(entries); + expect(text(messages.get("c1")).startsWith("redacted|")).toBe(true); + expect(messages.get("c1")).toMatchObject({ isError: true, details: { run: 1 } }); + expect(await diagnosticsOf(entries, "c1")).toEqual(["interrupted"]); + // The kept output keeps its truncation diagnostic, after the hooks' result. + expect(await diagnosticsOf(entries, "c2")).toEqual(["interrupted", "truncated"]); + await opened.harness.close(context); + }); + + // #11043 + it("runs on an abort result, but not on the aborted results of calls that never started", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const first = blocking("first"); + addTool(setup.registry, first.registration); + addTool( + setup.registry, + tool("second", async () => ({ output: [] })), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return { ...result, details: { hooked: true } }; + }, + }); + setup.faux.setResponses([calls(["first", {}, "c1"], ["second", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await first.started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + expect(seen.map((each) => [each.call.id, codes(each.result)])).toEqual([["c1", ["aborted"]]]); + const messages = results(await allEntries(root)); + expect(messages.get("c1")).toMatchObject({ isError: true, details: { hooked: true } }); + expect(messages.get("c2")?.details).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("reports a throwing hook on an abort result and settles the result as built", async () => { + const setup = chatSetup(); + const work = blocking("work"); + addTool(setup.registry, work.registration); + addHooks(setup.registry, ToolTask, { + afterTool: () => { + throw new Error("hook failed"); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await work.started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + const entries = await allEntries(root); + expect(text(results(entries).get("c1")).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect(setup.reports.map((error) => (error as Error).message)).toContain("hook failed"); + await harness.close(context); + }); + + // #11043 + it("settles an abort without the hooks in the abort handler's last attempt", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + const hanging = deferred(); + addHooks(setup.registry, ToolTask, { + afterTool: async (call, result) => { + seen.push({ call, result }); + hanging.resolve(); + // Never returns: the host dies inside the hook. + return new Promise(() => {}); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + const taskId = await toolTaskId(opened.harness); + await stored(); + void opened.root.abort(context).catch(() => {}); + await hanging.promise; + opened.kill(); + + // One restart, at the limit minus one: the attempt the scheduler would fault when cut off runs no hooks. + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("unanswered"); + expect(seen.map((each) => codes(each.result))).toEqual([["aborted"]]); + const entries = await allEntries(opened.root); + expect(text(results(entries).get("c1")).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "aborted" }, + }); + await opened.harness.close(context); + }); + + // #11043 + it("sees the held progress of an aborted call, and the result stores it", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + // Progress commits are held in memory until the result commit stores them. + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 60_000 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return undefined; + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await work.started.promise; + await opened.root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + expect(seen).toHaveLength(1); + expect(seen[0]!.result).toMatchObject({ details: { run: 1 } }); + expect(text({ content: seen[0]!.result.output } as ToolResultMessage)).toBe(`work ${"o".repeat(40)}\n`); + await opened.harness.close(context); + + opened = await open(path, setup); + const message = results(await allEntries(opened.root)).get("c1"); + expect(message).toMatchObject({ isError: true, details: { run: 1 } }); + expect(text(message).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + await opened.harness.close(context); + }); + + // #11043 + it("runs on an abandoned nested call's result", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const gated = deferred(); + let gate = true; + addTool( + setup.registry, + tool("gated", async () => ({ output: [] })), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("gated", {}, callContext); + return {}; + }), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + // Holds the nested call in its `call` phase, before intent, until the Harness closes. + beforeTool: async (call, _api, hookContext) => { + if (call.name !== "gated" || !gate) return undefined; + gated.resolve(); + return aborted(hookContext.abortSignal!); + }, + afterTool: (call, result) => { + if (call.parent === undefined) return undefined; + seen.push({ call, result }); + return { ...result, details: { hooked: true } }; + }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await gated.promise; + await opened.harness.close(context); + gate = false; + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen.map((each) => [each.call.id, codes(each.result)])).toEqual([["c1/1", ["abandoned"]]]); + await opened.harness.close(context); + }); + + // #11043 + it("runs on a restart_limit result at the limit, and not once the hooks themselves were cut off", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + const hanging = deferred(); + addHooks(setup.registry, ToolTask, { + afterTool: async (call, result) => { + seen.push({ call, result }); + hanging.resolve(); + // Never returns: the host dies inside the hook. + return new Promise(() => {}); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + const taskId = await toolTaskId(opened.harness); + await stored(); + opened.kill(); + + // Cut off once, at the limit: the task ends itself, and the hooks run on its result. + opened = await open(path, setup); + await hanging.promise; + expect(seen.map((each) => codes(each.result))).toEqual([["restart_limit"]]); + opened.kill(); + + // Past the limit: ends without the hooks. + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen).toHaveLength(1); + expect(work.state.runs).toBe(1); + const entries = await allEntries(opened.root); + expect(await diagnosticsOf(entries, "c1")).toEqual(["restart_limit"]); + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "failed", error: { detail: { reason: "restart_limit" } } }, + }); + await opened.harness.close(context); + }); +}); + +describe("calls to tools the request did not offer", () => { + // #11043 + it("run afterTool inside their own tool task", async () => { + const setup = chatSetup(); + const seen: { readonly id: string; readonly taskId: TaskId }[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result, api) => { + seen.push({ id: call.id, taskId: api.taskId }); + return { ...result, output: [{ type: "text", text: "try another tool" }] }; + }, + }); + setup.faux.setResponses([calls(["ghost", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(seen.map((each) => each.id)).toEqual(["c1"]); + expect(await harness.getTask(seen[0]!.taskId, context)).toMatchObject({ + kind: "pi.tool", + input: { kind: "model", callId: "c1", unoffered: true }, + }); + const entries = await allEntries(root); + expect(text(results(entries).get("c1")).startsWith("try another tool|")).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["tool_unavailable"]); + await harness.close(context); + }); + + // #11043 + it("count toward a round's terminate", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [] })), + ); + addHooks(setup.registry, ToolTask, { + afterTool: (_call, result) => ({ ...result, control: { terminate: true } }), + }); + setup.faux.setResponses([calls(["echo", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("take their turn in a sequential round, and the last handoff in call order wins", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const order: string[] = []; + for (const name of ["a", "b"]) { + addTool( + setup.registry, + tool(name, async () => { + order.push(name); + return { output: [] }; + }), + ); + } + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + order.push(`hook ${call.id}`); + return call.id === "c1" ? undefined : { ...result, control: { handoff: `from ${call.id}` } }; + }, + }); + setup.faux.setResponses([calls(["a", {}, "c1"], ["ghost", {}, "c2"], ["b", {}, "c3"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(order).toEqual(["a", "hook c1", "hook c2", "b", "hook c3"]); + const resets = (await allEntries(root)).filter((entry) => ResetEntry.is(entry)); + expect(resets.map((entry) => (entry.model?.[0] as { content: string }).content)).toEqual(["from c3"]); + await harness.close(context); + }); + + // #11043 + it("each get a tool task in a parallel round of only such calls", async () => { + const setup = chatSetup(); + const seen: string[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call) => { + seen.push(call.id); + return undefined; + }, + }); + setup.faux.setResponses([calls(["ghost", {}, "c1"], ["spirit", {}, "c2"], ["ghost", {}, "c3"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(seen.sort()).toEqual(["c1", "c2", "c3"]); + const entries = await allEntries(root); + for (const id of ["c1", "c2", "c3"]) expect(await diagnosticsOf(entries, id)).toEqual(["tool_unavailable"]); + // The round's results go to the next request. + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + // #11043 + it("keep their flag in a version 2 generation's sequential round, also one migrated from version 1", async () => { + for (const version of [2, 1]) { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const started = deferred(); + let runs = 0; + const a = tool( + "a", + async (_args, _api, callContext) => { + runs++; + if (runs > 1) return { output: [] }; + started.resolve(); + return aborted(callContext.abortSignal!); + }, + { replay: "safe" }, + ); + addTool(setup.registry, a); + addTool( + setup.registry, + tool("ghost", async () => ({ output: [{ type: "text", text: "ran" }] })), + ); + setup.faux.setResponses([calls(["a", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + let opened = await open(path, setup); + let root = opened.root; + // `ghost` is registered, but not offered to this request. + await root.configure({ tools: [a] }, context); + const id = (await root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + const live = (await opened.harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context))!; + const generation = live.run!.taskId; + expect(live.tools?.map((slot) => [slot.callId, slot.unoffered])).toEqual([ + ["c1", undefined], + ["c2", true], + ]); + expect((await opened.harness.getTask(generation, context))?.version).toBe(2); + // The tools are offered when the round goes on: only the flag keeps `ghost` from running. + await root.configure({ tools: null }, context); + await opened.harness.close(context); + if (version === 1) { + const storage = await openNodeSqliteStorage(path); + const record = (await storage.task(generation, context))!; + await commitWrites(storage, [{ type: "task", value: { ...record, version: 1 } }], context); + await storage.close(context); + } + + opened = await open(path, setup); + root = opened.root; + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect((await opened.harness.getTask(generation, context))?.version).toBe(2); + const entries = await allEntries(root); + expect(await diagnosticsOf(entries, "c2")).toEqual(["tool_unavailable"]); + expect(text(results(entries).get("c2"))).not.toContain("ran"); + await opened.harness.close(context); + } + }); + + // #11043 + it("settle unavailable, not aborted, when their sequential round is aborted before they start", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const started = deferred(); + addTool( + setup.registry, + tool("wait", async (_args, _api, callContext) => { + started.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + setup.faux.setResponses([calls(["wait", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + const entries = await allEntries(root); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect(await diagnosticsOf(entries, "c2")).toEqual(["tool_unavailable"]); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts index 226bade81..cf050f8df 100644 --- a/packages/durable/test/harness-tools-recovery.test.ts +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -21,6 +20,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; @@ -55,7 +55,7 @@ function tool( execute: ToolRegistration["execute"], extra: Partial = {}, ): ToolRegistration { - return defineTool({ name, description: name, parameters: Type.Object({}), execute, ...extra }); + return defineTool({ name, description: name, parameters: obj({}), execute, ...extra }); } function call(name: string, id = "c1"): AssistantMessage { @@ -83,7 +83,7 @@ async function toolTaskId(harness: Harness): Promise { /** * A tool that writes output, then blocks until its invocation is cancelled the first `blocking` times it runs. - * `started` resolves once the output is durable. + * `started` resolves once the output is committed; a close stores it. */ function blockingTool(name: string, extra: Partial = {}) { const started = deferred(); @@ -133,6 +133,70 @@ describe("tool recovery", () => { await opened.harness.close(context); }); + it("reruns a safe tool whose running slot a crash after its intent lost, or a sibling stored, and shows it running", async () => { + for (const sibling of [false, true]) { + const path = await sqlitePath(); + const setup = chatSetup(); + // The running slot is presentation, held in memory; no deadline stores it before the crash. + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + const started = deferred(); + let runs = 0; + let reopened: Harness | undefined; + const seen: (string | undefined)[] = []; + addTool( + setup.registry, + tool( + "work", + async (_args, api, callContext) => { + runs++; + if (runs === 1) { + started.resolve(); + await aborted(callContext.abortSignal!); + } + const slots = (await reopened!.snapshot(LiveDoc, 1 as never, callContext))?.tools ?? []; + seen.push(slots.find((slot) => slot.taskId === api.taskId)?.status); + return {}; + }, + { replay: "safe" }, + ), + ); + // A sibling that ends once `work` started: its result commit reads `pi.live`, which stores what is held there. + addTool( + setup.registry, + tool("quick", async () => { + await started.promise; + return {}; + }), + ); + const calls = [fauxToolCall("work", {}, { id: "w" })]; + if (sibling) calls.push(fauxToolCall("quick", {}, { id: "q" })); + setup.faux.setResponses([fauxAssistantMessage(calls, { stopReason: "toolUse" }), DONE]); + const crashed = await open(path, setup); + const id = (await crashed.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + const slot = async (harness: Harness, name: string) => + (await harness.snapshot(LiveDoc, crashed.root.id, context))?.tools?.find((each) => each.name === name); + const live = (harness: Harness) => slot(harness, "work"); + const workId = (await live(crashed.harness))!.taskId!; + if (sibling) await waitFor(async () => (await slot(crashed.harness, "quick"))?.status === "done"); + // The process dies here: another one opens the same database, without the held presentation. + const next = await openChat(await openNodeSqliteStorage(path), setup); + reopened = next.harness; + expect((await next.harness.getTask(workId, context))?.state).toMatchObject({ + checkpoint: { phase: "execute" }, + }); + expect((await live(next.harness))?.status).toBe(sibling ? "running" : "pending"); + next.harness.resume(); + expect((await (await next.harness.submission(id, context))!.wait(context)).status).toBe("done"); + // The rerun shows the slot running again. + expect(seen).toEqual(["running"]); + const [result] = results(await allEntries(next.root)).filter((message) => message.toolCallId === "w"); + expect(result!.isError).toBe(false); + await next.harness.close(context); + await crashed.harness.close(context); + } + }); + it("reruns a tool only when both the stored and the current replay policy are safe", async () => { const cases: [stored: "safe" | "unsafe", current: "safe" | "unsafe", reruns: boolean][] = [ ["safe", "safe", true], diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index 29f6fd3f7..92aec1cf6 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -11,7 +11,6 @@ import { type Models, type SystemMessage, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { AgentDoc, @@ -32,6 +31,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; @@ -40,7 +40,7 @@ import { addHooks, addTask, addTool, type Installed } from "./harness-support.ts import { context } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; -const EchoParameters = Type.Object({ text: Type.Optional(Type.String()) }); +const EchoParameters = obj({ text: opt(str()) }); type Echo = ToolRegistration; type Execute = Echo["execute"]; @@ -138,7 +138,7 @@ describe("tool round", () => { await harness.close(context); }); - it("answers calls to tools the request did not offer without a task", async () => { + it("answers calls to tools the request did not offer through a flagged tool task", async () => { const setup = chatSetup(); addTool( setup.registry, @@ -157,8 +157,34 @@ describe("tool round", () => { expect(ghostEntry?.data).toEqual({ diagnostics: [{ severity: "error", code: "tool_unavailable", message: "Tool ghost is not available" }], }); + // #11043: the unoffered call has its own task, so `afterTool` and `control` apply to it as to any call. const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); - expect(tasks.items.filter((task) => task.kind === "pi.tool")).toHaveLength(1); + const inputs = tasks.items.filter((task) => task.kind === "pi.tool").map((task) => task.input); + expect(inputs).toHaveLength(2); + expect(inputs).toContainEqual(expect.objectContaining({ callId: "c1", unoffered: true })); + expect(inputs).toContainEqual(expect.not.objectContaining({ unoffered: true })); + await harness.close(context); + }); + + // #11043 + it("never runs a tool the request did not offer, even when the agent has it", async () => { + const setup = chatSetup(); + let ran = 0; + addTool( + setup.registry, + tool( + "hidden", + async () => { + ran++; + return { output: [] }; + }, + { deferred: true }, + ), + ); + const { harness, entries, status } = await run(setup, [calls(["hidden", {}, "c1"]), DONE]); + expect(status).toBe("done"); + expect(ran).toBe(0); + expect(resultText(results(entries)[0]!)).toBe("\n[error] Tool hidden is not available\n"); await harness.close(context); }); @@ -726,6 +752,75 @@ describe("generation hooks", () => { await harness.close(context); }); + it("chains beforeProviderRequest over every provider request body", async () => { + const setup = chatSetup(); + const seen: { hook: string; model: string; payload: unknown }[] = []; + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ model, payload }) => { + seen.push({ hook: "first", model: `${model.provider}/${model.id}`, payload }); + return { payload: { ...(payload as object), first: true } }; + }, + }); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ model, payload }) => { + seen.push({ hook: "keep", model: `${model.provider}/${model.id}`, payload }); + return undefined; + }, + }); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ payload }) => ({ payload: { ...(payload as object), last: true } }), + }); + const sent: unknown[] = []; + // pi-ai builds a body per provider call, and again when it resends within one attempt. + const respond: FauxResponseStep = async (_context, options, _state, model) => { + sent.push(await options!.onPayload!({ body: 1 }, model)); + sent.push(await options!.onPayload!({ body: 2 }, model)); + return DONE; + }; + const { harness, status } = await run(setup, [respond]); + expect(status).toBe("done"); + expect(sent).toEqual([ + { body: 1, first: true, last: true }, + { body: 2, first: true, last: true }, + ]); + expect(seen).toEqual([ + { hook: "first", model: "faux/faux-1", payload: { body: 1 } }, + { hook: "keep", model: "faux/faux-1", payload: { body: 1, first: true } }, + { hook: "first", model: "faux/faux-1", payload: { body: 2 } }, + { hook: "keep", model: "faux/faux-1", payload: { body: 2, first: true } }, + ]); + await harness.close(context); + }); + + it("reports a throwing beforeProviderRequest hook and continues the chain", async () => { + const setup = chatSetup(); + addHooks( + setup.registry, + GenerationTask, + { + beforeProviderRequest: () => { + throw new Error("broken hook"); + }, + }, + "broken", + ); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ payload }) => ({ payload: { ...(payload as object), after: true } }), + }); + let sent: unknown; + const respond: FauxResponseStep = async (_context, options, _state, model) => { + sent = await options!.onPayload!({ body: 1 }, model); + return DONE; + }; + const { harness, status } = await run(setup, [respond]); + expect(status).toBe("done"); + expect(sent).toEqual({ body: 1, after: true }); + expect(setup.infos.filter((info) => info.kind === "extension_error")).toEqual([ + expect.objectContaining({ extension: "broken" }), + ]); + await harness.close(context); + }); + it("keeps the run's input open across an onYield continuation and answers it with the final answer", async () => { const setup = chatSetup(); let yields = 0; @@ -1168,7 +1263,9 @@ describe("tool progress and lifetime", () => { }), ); addHooks(setup.registry, ToolTask, { - afterTool: async (_call, _result, _api, callContext) => { + afterTool: async (_call, result, _api, callContext) => { + // The abort handler's result runs the hooks too (#11043); only the executed call's waits. + if (result.diagnostics?.some((diagnostic) => diagnostic.code === "aborted")) return undefined; inAfterTool.resolve(); await aborted(callContext.abortSignal!); return undefined; diff --git a/packages/durable/test/harness-turn-limit.test.ts b/packages/durable/test/harness-turn-limit.test.ts new file mode 100644 index 000000000..7f8ad61e1 --- /dev/null +++ b/packages/durable/test/harness-turn-limit.test.ts @@ -0,0 +1,359 @@ +// Per-conversation turn limits (`AgentState.maxTurns`) and continuing a run that reached one (#11043). + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; +import { + type AfterFailedRun, + type Conversation, + defineTool, + GenerationTask, + LiveDoc, + MemoryStorage, + ResetEntry, + type Storage, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +const Parameters = obj({}); + +function tool( + name: string, + execute: ToolRegistration["execute"], +): ToolRegistration { + return defineTool({ name, description: name, parameters: Parameters, execute }); +} + +/** A tool-calling answer with one `noop` call. */ +function round(id: string): AssistantMessage { + return fauxAssistantMessage([fauxToolCall("noop", {}, { id })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +/** A chat with a `noop` tool and the given turn limit. */ +async function limited(maxTurns: number | undefined, configure?: (setup: ChatSetup) => void) { + const setup = chatSetup(); + addTool( + setup.registry, + tool("noop", async () => ({ output: [] })), + ); + configure?.(setup); + const opened = await openChat(new MemoryStorage(), setup); + if (maxTurns !== undefined) await opened.root.configure({ maxTurns }, context); + return { setup, ...opened }; +} + +async function lastKind(root: Conversation): Promise { + return (await allEntries(root)).at(-1)?.kind; +} + +describe("turn limit", () => { + // #11043 + it("ends a run after the tool round that reaches the limit, like a failure", async () => { + const { setup, harness, root } = await limited(1); + setup.faux.setResponses([round("c1"), DONE]); + const submission = await root.submit({ type: "input", content: "go" }, context); + expect(await submission.wait(context)).toMatchObject({ + status: "unanswered", + reason: "turn_limit", + detail: "1 turn", + }); + expect(setup.faux.state.callCount).toBe(1); + // The transcript ends with the round's results; the generation completed, so nothing is reported. + expect(await lastKind(root)).toBe(ToolResultEntry.kind); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toMatchObject({ + reason: "turn_limit", + inputs: [submission.id], + }); + const generations = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); + const generation = generations.items.find((task) => task.kind === GenerationTask.definition.name); + expect(generation?.state).toMatchObject({ status: "terminal", outcome: { status: "completed" } }); + expect(setup.reports).toEqual([]); + await harness.close(context); + }); + + // #11043 + it("counts tool rounds only: a run answers below its limit, and an answer at the limit settles done", async () => { + const { setup, harness, root } = await limited(2); + setup.faux.setResponses([round("c1"), DONE, round("c2"), round("c3"), DONE]); + expect((await (await root.submit({ type: "input", content: "one" }, context)).wait(context)).status).toBe("done"); + // A new run starts with a fresh budget. + const second = await (await root.submit({ type: "input", content: "two" }, context)).wait(context); + expect(second).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await harness.close(context); + }); + + // #11043 + it("runs onYield at the limit, and a continuation starts the count over", async () => { + let yields = 0; + const { setup, harness, root } = await limited(2, (chat) => + addHooks(chat.registry, GenerationTask, { + onYield: () => (yields++ === 0 ? { continue: "keep going" } : undefined), + }), + ); + // Round (1 turn), answer: the continuation resets the count; then two rounds reach the limit. + setup.faux.setResponses([round("c1"), DONE, round("c2"), round("c3"), DONE]); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit" }); + expect(yields).toBe(1); + expect(setup.faux.state.callCount).toBe(4); + await harness.close(context); + }); + + // #11043 + it("settles a steer that joined the run at an earlier round with turn_limit too", async () => { + const gate = deferred(); + const started = deferred(); + const setup = chatSetup(); + let calls = 0; + addTool( + setup.registry, + tool("noop", async () => { + if (calls++ === 0) { + started.resolve(); + await gate.promise; + } + return { output: [] }; + }), + ); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 2 }, context); + setup.faux.setResponses([round("c1"), round("c2"), DONE]); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const steer = await root.submit({ type: "input", content: "also this", whenBusy: "steer" }, context); + gate.resolve(); + for (const submission of [first, steer]) { + expect(await submission.wait(context)).toMatchObject({ + status: "unanswered", + reason: "turn_limit", + detail: "2 turns", + }); + } + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.inputs).toEqual([first.id, steer.id]); + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + // #11043 + it("leaves items queued during the last round to afterFailedRun", async () => { + for (const policy of ["keep", "continue", "withdraw"] as const satisfies readonly AfterFailedRun[]) { + const gate = deferred(); + const started = deferred(); + const setup = chatSetup(); + setup.settings.afterFailedRun = policy; + addTool( + setup.registry, + tool("noop", async () => { + started.resolve(); + await gate.promise; + return { output: [] }; + }), + ); + setup.faux.setResponses([round("c1"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const steer = await root.submit({ type: "input", content: "also", whenBusy: "followUp" }, context); + gate.resolve(); + expect((await first.wait(context)).status).toBe("unanswered"); + await harness.waitForIdle(context); + const queued = await steer.status(context); + if (policy === "keep") expect(queued.status).toBe("queued"); + if (policy === "continue") expect(queued.status).toBe("done"); + if (policy === "withdraw") expect(queued).toMatchObject({ status: "unanswered", reason: "run_failed" }); + await harness.close(context); + } + }); + + // #11043 + it("applies a changed limit at the end of the round; null removes it", async () => { + const setup = chatSetup(); + let root!: Conversation; + let rounds = 0; + addTool( + setup.registry, + tool("noop", async () => { + rounds++; + // Lowered during the second round: that round ends the run. + if (rounds === 2) await root.configure({ maxTurns: 1 }, context); + return { output: [] }; + }), + ); + setup.faux.setResponses([round("c1"), round("c2"), round("c3"), DONE]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await root.configure({ maxTurns: 5 }, context); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await root.configure({ maxTurns: null }, context); + expect((await root.agent(context)).maxTurns).toBeUndefined(); + setup.faux.setResponses([round("c4"), round("c5"), DONE]); + expect((await (await root.submit({ type: "input", content: "again" }, context)).wait(context)).status).toBe( + "done", + ); + await opened.harness.close(context); + }); + + // #11043 + it("rejects a limit that is not an integer 1 or more", async () => { + const { harness, root } = await limited(undefined); + for (const value of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + await expect(root.configure({ maxTurns: value }, context)).rejects.toBeInstanceOf(RangeError); + } + await root.configure({ maxTurns: 3 }, context); + expect((await root.agent(context)).maxTurns).toBe(3); + await harness.close(context); + }); + + // #11043 + it("lets terminate settle done at the limit, and writes a handoff's reset before ending", async () => { + const { setup, harness, root } = await limited(1, (chat) => + addHooks(chat.registry, ToolTask, { + afterTool: (call, result) => + call.id === "t" + ? { ...result, control: { terminate: true } } + : { ...result, control: { handoff: "fresh" } }, + }), + ); + setup.faux.setResponses([round("t"), round("h"), DONE]); + expect((await (await root.submit({ type: "input", content: "one" }, context)).wait(context)).status).toBe("done"); + const handedOff = await (await root.submit({ type: "input", content: "two" }, context)).wait(context); + expect(handedOff).toMatchObject({ status: "unanswered", reason: "turn_limit" }); + const resets = (await allEntries(root)).filter((entry) => ResetEntry.is(entry)); + expect(resets.map((entry) => (entry.model?.[0] as { content: string }).content)).toEqual(["fresh"]); + await harness.close(context); + }); + + // #11043 + it("is resumed by a continue, with a fresh budget, on the round's results", async () => { + const { setup, harness, root } = await limited(1); + let continued: string | undefined; + setup.faux.setResponses([ + round("c1"), + (request) => { + continued = request.messages.at(-1)?.role; + return fauxAssistantMessage([fauxText("finished")]); + }, + ]); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const resumed = await root.submit({ type: "continue" }, context); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.turns).toBe(0); + const settled = await resumed.wait(context); + if (settled.status !== "done" || settled.type === "write") throw new Error(`Unexpected ${settled.status}`); + const answer = (await allEntries(root)).find((entry) => entry.id === settled.answer); + expect(textOf(answer?.model?.[0])).toBe("finished"); + expect(continued).toBe("toolResult"); + await harness.close(context); + }); + + // #11043 + it("is inherited by forks and children, which may set their own", async () => { + const setup = chatSetup(); + let child: number | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + child = await api.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } })).id, + callContext, + ); + return { output: [] }; + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("spawn", {}, { id: "c1" })], { stopReason: "toolUse" }), + DONE, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 4 }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const spawned = await harness.conversation(child as never, context); + expect((await spawned!.agent(context)).maxTurns).toBe(4); + const at = (await allEntries(root)).at(-1)!.id; + const fork = await root.fork(at, { ownership: { kind: "ownerless" } }, context); + expect((await fork.agent(context)).maxTurns).toBe(4); + const own = await harness.createConversation( + { ownership: { kind: "ownerless" }, agent: { maxTurns: 2 } }, + context, + ); + expect((await own.agent(context)).maxTurns).toBe(2); + await harness.close(context); + }); + + // #11043 + it("keeps the stored count across a restart", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-turns-")); + directories.add(directory); + const path = join(directory, "session.sqlite"); + const setup = chatSetup(); + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const started = deferred(); + let runs = 0; + addTool( + setup.registry, + defineTool({ + name: "noop", + description: "noop", + parameters: Parameters, + replay: "safe", + execute: async (_args, _api, callContext) => { + runs++; + if (runs === 2) { + started.resolve(); + return aborted(callContext.abortSignal!); + } + return { output: [] }; + }, + }), + ); + setup.faux.setResponses([round("c1"), round("c2"), DONE]); + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const first = await openChat(storage, setup); + await first.root.configure({ maxTurns: 2 }, context); + const id = (await first.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + expect((await first.harness.snapshot(LiveDoc, first.root.id, context))?.run?.turns).toBe(2); + dead = true; + killed.add(inner); + + // The replay-safe call runs again after reopen; the stored count still ends the run at this round. + runs = 0; + const second = await openChat(await openNodeSqliteStorage(path), setup); + second.harness.resume(); + expect((await second.harness.snapshot(LiveDoc, second.root.id, context))?.run?.turns).toBe(2); + const settled = await (await second.harness.submission(id, context))!.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await second.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts new file mode 100644 index 000000000..79f446098 --- /dev/null +++ b/packages/durable/test/harness-view-run.test.ts @@ -0,0 +1,619 @@ +// What a renderer sees of a run through the conversation view: the transcript window and `pi.live`, every change a +// commit makes, rebuilt from operations alone. This replaced the event stream (`watchEvents`, #11327). + +import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { Draft } from "@earendil-works/chord"; +import { applyImmutable, type Op } from "@earendil-works/chord/delta"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxThinking, + fauxToolCall, +} from "@earendil-works/pi-ai"; +import { + type Conversation, + type ConversationView, + defineTool, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, +} from "@earendil-works/pi-durable"; +import { num, obj } from "@earendil-works/pi-schema"; +import { describe, expect, it } from "vitest"; +import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { slotsOf, viewHistory } from "./live-support.ts"; +import { context, documentValues, followCommits, onValue } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +type Frame = { value: ConversationView; ops: readonly Op[] }; + +/** + * Start a watch of `conversation` that records every call and, with `check`, checks that its operations turn the value + * before it into its value. The frames are what a renderer that keeps up gets. + */ +async function listen(conversation: Conversation, check = true) { + const watch = await conversation.watch(context); + const frames: Frame[] = []; + let replayed = watch.value; + let mismatches = 0; + watch.start( + onValue(async ({ value, ops }) => { + if (check) { + replayed = applyImmutable(replayed, ops); + if (JSON.stringify(replayed) !== JSON.stringify(value)) mismatches++; + } + frames.push({ value, ops }); + }), + ); + return { watch, frames, mismatches: () => mismatches }; +} + +const liveOf = (view: ConversationView) => (view.docs["pi.live"] ?? {}) as LiveState; + +async function drained(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +function slow(): ChatSetup { + return chatSetup({ tokensPerSecond: 400, tokenSize: { min: 1, max: 1 } }); +} + +/** Committed partials of the Harness's generations, one per commit that has one. */ +function partialsOf(harness: Harness): AssistantMessage[] { + const partials: AssistantMessage[] = []; + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { + const message = (change.value as LiveState)?.generation?.message as AssistantMessage | undefined; + if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); + } + }); + return partials; +} + +/** The distinct in-flight partials in `states`, in order. */ +function partialsIn(states: readonly LiveState[]): AssistantMessage[] { + const partials: AssistantMessage[] = []; + for (const state of states) { + const message = state.generation?.message as AssistantMessage | undefined; + if (message !== undefined && message !== partials.at(-1)) partials.push(message); + } + return partials; +} + +describe("a run as the conversation view shows it", () => { + it("streams the answer as operations that rebuild every committed partial, then appends it", async () => { + const setup = slow(); + const text = "streamed answer text ".repeat(20); + setup.faux.setResponses([fauxAssistantMessage([fauxText(text)])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const history = await viewHistory(harness, root.id); + const { watch, mismatches } = await listen(root); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await submission.wait(context); + await drained(); + history.detach(); + expect(mismatches()).toBe(0); + const states = history.live(); + expect(partials.length).toBeGreaterThan(1); + expect(partialsIn(states).map(textOf)).toEqual(partials.map(textOf)); + // The run shows from the input's commit until the commit that appends the answer and clears the partial. + const started = states.findIndex((state) => state.run !== undefined); + expect(states[started]!.run).toEqual({ + taskId: expect.any(Number), + inputs: [submission.id], + turns: 0, + first: expect.any(Number), + }); + expect(history.views[started]!.entries.at(-1)!.kind).toBe("pi.user"); + const ended = states.findIndex((state, index) => index > started && state.run === undefined); + expect(states[ended]!.generation).toBeUndefined(); + expect(textOf(history.views[ended]!.entries.at(-1)!.model![0])).toBe(text); + expect(states.at(-1)).toEqual({}); + await watch.stop(); + await harness.close(context); + }); + + it("shows a tool call's slot from pending through its running output to its result entry", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "print", + description: "Prints", + parameters: obj({ n: num() }), + execute: async (_args, api) => { + api.output("one\n"); + await new Promise((resolve) => setTimeout(resolve, 150)); + api.output("two\n"); + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("print", { n: 1 }, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => slotsOf(history.live(), "c1").at(-1)?.output === "one\ntwo\n"); + gate.resolve(); + await submission.wait(context); + history.detach(); + const slots = slotsOf(history.live(), "c1"); + const listed = slots.findIndex((slot) => slot !== undefined); + // Listed pending with the tool-calling answer, in the same commit. + expect(slots[listed]).toEqual({ callId: "c1", name: "print", status: "pending", taskId: expect.any(Number) }); + expect(history.views[listed]!.entries.at(-1)!.kind).toBe("pi.assistant"); + const outputs = slots.flatMap((slot) => (slot?.status === "running" ? [slot.output ?? ""] : [])); + expect([...new Set(outputs)]).toEqual(["", "one\n", "one\ntwo\n"]); + // Done with its result entry, appended in the same commit. + const done = slots.findIndex((slot) => slot?.status === "done"); + expect(slots[done]).toEqual({ + callId: "c1", + name: "print", + status: "done", + taskId: expect.any(Number), + entry: history.views[done]!.entries.at(-1)!.id, + }); + expect(history.views[done]!.entries.at(-1)!.model![0]).toMatchObject({ role: "toolResult", toolCallId: "c1" }); + // The run's two generations, the tool round and the answer, hand the run over. + const runTasks = new Set(history.live().flatMap((state) => (state.run === undefined ? [] : [state.run.taskId]))); + expect(runTasks.size).toBe(2); + await harness.close(context); + }); + + it("shows queued inputs in the inbox, a retry backoff, and the next run", async () => { + const setup = chatSetup(); + const release = deferred(); + const held: FauxResponseStep = async (_request, options) => { + await Promise.race([release.promise, aborted(options!.signal!)]); + return fauxAssistantMessage([fauxText("first")]); + }; + const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); + setup.faux.setResponses([held, error, fauxAssistantMessage([fauxText("second")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const history = await viewHistory(harness, root.id); + const first = await root.submit({ type: "input", content: "a" }, context); + const followUp = await root.submit({ type: "input", content: "f" }, context); + const inbox = (view: ConversationView) => + (view.docs["pi.inbox"] as { items: { id: number; mode: string }[] }).items; + expect(inbox(history.views.at(-1)!)).toEqual([expect.objectContaining({ id: followUp.id, mode: "followUp" })]); + release.resolve(); + await followUp.wait(context); + history.detach(); + const states = history.live(); + expect(states.some((state) => state.generation?.retry?.error === "503 Service Unavailable")).toBe(true); + const retried = states.findIndex((state) => state.generation?.retry !== undefined); + expect(states.slice(retried).some((state) => state.generation?.attempt === 2)).toBe(true); + const runs = states.flatMap((state, index) => + state.run !== undefined && state.run.inputs !== states[index - 1]?.run?.inputs ? [state.run.inputs] : [], + ); + expect(runs.at(0)).toEqual([first.id]); + expect(runs.at(-1)).toEqual([followUp.id]); + expect(inbox(history.views.at(-1)!)).toEqual([]); + await harness.close(context); + }); + + it("rebuilds every committed partial of thinking, text, and tool-call arguments from the view's operations", async () => { + const setup = chatSetup({ tokensPerSecond: 150, tokenSize: { min: 1, max: 1 } }); + const message = fauxAssistantMessage( + [ + fauxThinking("thinking about it ".repeat(10)), + fauxText("some text ".repeat(10)), + fauxToolCall("missing", { path: "a/long/path/".repeat(10), note: "x".repeat(60) }, { id: "c1" }), + ], + { stopReason: "toolUse" }, + ); + setup.faux.setResponses([message, fauxAssistantMessage([fauxText("done")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const views: ConversationView[] = []; + const ops: Op[][] = []; + const observation = await harness.observeConversation( + root.id, + { + advance: (call) => { + if (call.kind !== "value") return; + views.push(call.value); + ops.push([...call.ops]); + }, + end: () => {}, + }, + context, + ); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + observation.detach(); + // Replaying the operations alone, from the observation's value, gives every committed partial. + let replayed = observation.value; + const rebuilt: unknown[] = []; + for (const batch of ops) { + replayed = applyImmutable(replayed, batch); + const partial = liveOf(replayed).generation?.message; + if (partial !== undefined && partial !== rebuilt.at(-1)) rebuilt.push(partial); + } + expect(replayed).toEqual(views.at(-1)); + expect(partials.length).toBeGreaterThan(2); + expect(rebuilt.map((partial) => (partial as AssistantMessage).content)).toEqual( + partials.map((partial) => partial.content), + ); + // Growing text and thinking travel as appends. + const appends = ops.flat().filter((op) => op[0] === "a" && op[1][2] === "generation"); + expect(appends.length).toBeGreaterThan(2); + await harness.close(context); + }); + + it("rebuilds a sliding tail window from output trims and appends", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "tail", + description: "Prints lines", + parameters: obj({}), + outputLimits: { maxLines: 3, retain: "tail" }, + execute: async (_args, api) => { + for (let line = 0; line < 6; line++) { + api.output(`line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 120)); + } + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("tail", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const outputs: string[] = []; + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { + const output = (change.value as LiveState)?.tools?.[0]?.output; + if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { + outputs.push(output); + } + } + }); + const { watch, frames, mismatches } = await listen(root); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => outputs.at(-1)?.endsWith("line 5\n") === true); + gate.resolve(); + await submission.wait(context); + await drained(); + await watch.stop(); + expect(mismatches()).toBe(0); + const shown: string[] = []; + for (const frame of frames) { + const output = liveOf(frame.value).tools?.[0]?.output; + if (output !== undefined && output !== shown.at(-1)) shown.push(output); + } + expect(shown).toEqual(outputs); + expect(frames.some((frame) => frame.ops.some((op) => op[0] === "t"))).toBe(true); + await harness.close(context); + }); + + it("ends a run and starts the queued follow-up's in one value", async () => { + const setup = chatSetup(); + const release = deferred(); + const held: FauxResponseStep = async (_request, options) => { + await Promise.race([release.promise, aborted(options!.signal!)]); + return fauxAssistantMessage([fauxText("first")]); + }; + setup.faux.setResponses([held, fauxAssistantMessage([fauxText("second")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + const followUp = await root.submit({ type: "input", content: "f" }, context); + const history = await viewHistory(harness, root.id); + release.resolve(); + await followUp.wait(context); + history.detach(); + const states = history.live(); + const boundary = states.findIndex((state) => state.run?.inputs[0] === followUp.id); + expect(states[boundary - 1]!.run?.inputs).toEqual([input.id]); + const before = history.views[boundary - 1]!; + const after = history.views[boundary]!; + const appended = after.entries.slice(before.entries.length); + expect(appended.map((entry) => [entry.kind, textOf(entry.model?.[0])])).toEqual([ + ["pi.assistant", "first"], + ["pi.user", "f"], + ]); + expect((after.docs["pi.inbox"] as { items: unknown[] }).items).toEqual([]); + await harness.close(context); + }); + + it("ends a call that never runs through its own task, and a tool aborted with its generation", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + defineTool({ + name: "wait", + description: "Waits until aborted", + parameters: obj({}), + execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("ghost", {}, { id: "c1" }), fauxToolCall("wait", {}, { id: "c2" })], { + stopReason: "toolUse", + }), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => slotsOf(history.live(), "c2").some((slot) => slot?.status === "running")); + // Aborting the generation aborts its round: the tool ends with its aborted result first (spec §8.5). + await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); + await submission.wait(context); + history.detach(); + const states = history.live(); + // #11043: the call not offered is listed pending with its task in the commit of its calling answer, and its task + // ends it done with its result entry. + const listed = slotsOf(states, "c1").findIndex((slot) => slot !== undefined); + expect(slotsOf(states, "c1")[listed]).toMatchObject({ status: "pending", unoffered: true }); + expect(slotsOf(states, "c1")[listed]!.taskId).toBeDefined(); + expect(history.views[listed]!.entries.at(-1)!.kind).toBe("pi.assistant"); + const ended = slotsOf(states, "c1").findIndex((slot) => slot?.status === "done"); + expect(slotsOf(states, "c1")[ended]!.entry).toBe(history.views[ended]!.entries.at(-1)!.id); + // The aborted tool ends with its result entry, then the run ends without an answer. + const done = slotsOf(states, "c2").findIndex((slot) => slot?.status === "done"); + expect(slotsOf(states, "c2")[done]!.entry).toBe(history.views[done]!.entries.at(-1)!.id); + expect(states.at(-1)!.runFailure).toMatchObject({ inputs: [submission.id], reason: "aborted" }); + await harness.close(context); + }); + + it("keeps a steer in the current run, shows nothing for other conversations, and appends a reset", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: obj({}), + execute: async () => { + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + const history = await viewHistory(harness, root.id); + const input = await root.submit({ type: "input", content: "a" }, context); + await waitFor(() => slotsOf(history.live(), "c1").some((slot) => slot?.status === "running")); + const steer = await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); + const count = history.views.length; + await other.commit((tx) => tx.appendEntry(other.id, { kind: "note" }), context); + expect(history.views.length).toBe(count); + gate.resolve(); + await input.wait(context); + await root.reset(undefined, context); + history.detach(); + const states = history.live(); + // One run: the steer joined it. + const runs = new Set(states.flatMap((state) => (state.run === undefined ? [] : [state.run.inputs[0]]))); + expect([...runs]).toEqual([input.id]); + expect(states.some((state) => state.run?.inputs.includes(steer.id))).toBe(true); + // The reset is appended in place, with no run. + expect(history.views.at(-1)!.entries.at(-1)!.kind).toBe("pi.reset"); + expect(states.at(-1)!.run).toBeUndefined(); + await harness.close(context); + }); + + it("converts the streamed partial into one aborted answer entry when the run is aborted", async () => { + const setup = chatSetup({ tokensPerSecond: 100, tokenSize: { min: 1, max: 1 } }); + setup.faux.setResponses([fauxAssistantMessage([fauxText("x".repeat(400))])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => history.live().some((state) => state.generation?.message !== undefined)); + const live = await harness.snapshot(LiveDoc, root.id, context); + await harness.abortTask(live!.run!.taskId, context); + await submission.wait(context); + history.detach(); + const states = history.live(); + const ended = states.findIndex((state, index) => index > 0 && state.run === undefined); + expect(states[ended - 1]!.generation?.message).toBeDefined(); + const appended = history.views[ended]!.entries.slice(history.views[ended - 1]!.entries.length); + expect(appended.map((entry) => entry.kind)).toEqual(["pi.assistant"]); + expect(appended[0]!.model![0]).toMatchObject({ role: "assistant", stopReason: "aborted" }); + expect(states[ended]!.runFailure).toMatchObject({ reason: "aborted" }); + await harness.close(context); + }); + + it("applies later operations to an overflow replacement that holds an in-flight partial", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const partial = fauxAssistantMessage([fauxText("hel")]); + const watch = await root.watch(context, { min: 1, max: 1 }); + const change = (edit: (live: Draft) => void) => + root.commit(async (tx) => edit(await tx.doc(LiveDoc, root.id)), context); + await change((live) => { + live.generation = { attempt: 1, message: JSON.parse(JSON.stringify(partial)) }; + }); + // Past 10,000 pending operations: an append, a splice, and a `before` per commit. + for (let index = 0; index < 3400; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + } + const frames: Frame[] = []; + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); + await drained(); + expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); + await change((live) => { + (live.generation!.message!.content[0] as { text: string }).text += "lo"; + }); + await drained(); + const update = frames.at(-1)!; + expect(update.ops).toEqual([["a", ["docs", "pi.live", "generation", "message", "content", 0, "text"], "lo"]]); + const rebuilt = applyImmutable(frames[0]!.value, update.ops); + expect(liveOf(rebuilt).generation?.message?.content).toEqual([{ type: "text", text: "hello" }]); + await watch.stop(); + await harness.close(context); + }, 30_000); +}); + +const gc = (() => { + setFlagsFromString("--expose-gc"); + return runInNewContext("gc") as () => void; +})(); + +/** Bytes `run` allocates, counting what collections during it freed. */ +async function allocatedBytes(run: () => Promise): Promise { + gc(); + const profiler = new GCProfiler(); + profiler.start(); + const before = getHeapStatistics().used_heap_size; + await run(); + const after = getHeapStatistics().used_heap_size; + let freed = 0; + for (const collection of profiler.stop().statistics) { + freed += collection.beforeGC.heapStatistics.usedHeapSize - collection.afterGC.heapStatistics.usedHeapSize; + } + return after - before + freed; +} + +/** Heap bytes in use after a full collection. */ +async function heapInUse(): Promise { + for (let round = 0; round < 3; round++) { + await drained(); + gc(); + } + return getHeapStatistics().used_heap_size; +} + +// #9265 +describe("streamed partials", () => { + it("streams a tool-call partial as appends at its argument paths", async () => { + const setup = chatSetup({ tokensPerSecond: 500, tokenSize: { min: 1, max: 1 } }); + setup.settings.progress = { partialIntervalMs: 20, outputIntervalMs: 100 }; + const args = { path: "src/file.ts", content: "const a = 1;\n".repeat(150) }; + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("missing", args, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const ops: Op[] = []; + const observation = await harness.observeConversation( + root.id, + { advance: (call) => call.kind === "value" && ops.push(...call.ops), end: () => {} }, + context, + ); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + observation.detach(); + const MESSAGE = ["docs", "pi.live", "generation", "message"]; + // Operations on the in-flight message, with their paths relative to it. + const onMessage = ops.flatMap((op) => { + const path = op[0] === "r" ? [] : (op[1] as readonly (string | number)[]); + return MESSAGE.every((segment, index) => path[index] === segment) + ? [{ kind: op[0], path: path.slice(MESSAGE.length) }] + : []; + }); + const argumentAppends = onMessage.filter((op) => op.kind === "a" && op.path[2] === "arguments"); + expect(argumentAppends.length).toBeGreaterThan(10); + expect(argumentAppends.every((op) => ["path", "content"].includes(String(op.path.at(-1))))).toBe(true); + // The call is set whole only when a member appears (at most once per argument); growing strings are appends. + // With the raw argument text on the block, every commit resent the whole call. + const blockSets = onMessage.filter((op) => op.kind !== "a" && op.path.length === 2); + expect(blockSets.length).toBeLessThanOrEqual(2); + // The committed partials carry only the parsed arguments, never the raw argument text. + for (const partial of partials) { + for (const block of partial.content) { + expect(Object.keys(block).sort()).toEqual(["arguments", "id", "name", "type"]); + } + } + await harness.close(context); + }); + + it("commits a growing text and tool call at a cost that follows the new text, not the string lengths", async () => { + /** Bytes allocated while streaming one answer of `size` characters of text and as much tool-call content. */ + const stream = async (size: number) => { + const setup = chatSetup({ tokensPerSecond: 64_000, tokenSize: { min: 64, max: 64 } }); + setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; + // Keep `pi.live` cached between commits, as the default retention does. Evicted after each use (the suite's + // evicting run), every commit replays the document's delta chain: a cost of eviction, not of appending. + setup.settings.documentRetentionMs = 60_000; + const text = 'streamed text with "quotes" and \\ escapes\n'.repeat(size / 40); + setup.faux.setResponses([ + fauxAssistantMessage( + [fauxText(text), fauxToolCall("missing", { path: "a.ts", content: text }, { id: "c1" })], + { + stopReason: "toolUse", + }, + ), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + let commits = 0; + followCommits(harness, () => commits++); + const bytes = await allocatedBytes(async () => { + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + }); + await harness.close(context); + return { bytes, commits }; + }; + const small = await stream(40_000); + const large = await stream(400_000); + expect(large.commits).toBeGreaterThan(small.commits * 5); + // Per commit, the cost stays flat: comparing the whole strings on every commit made it grow with their length. + expect(large.bytes / large.commits / (small.bytes / small.commits)).toBeLessThan(2); + }, 60_000); + + it("keeps a watcher's memory proportional to a long streamed text and tool call, not to their commits", async () => { + const setup = chatSetup({ tokensPerSecond: 16_000, tokenSize: { min: 16, max: 16 } }); + setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; + const line = (index: number) => `line ${index}: some streamed source text with "quotes" and \\ escapes\n`; + const text = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); + const content = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); + setup.faux.setResponses([ + fauxAssistantMessage([fauxText(text), fauxToolCall("missing", { path: "a.ts", content }, { id: "c1" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const { watch, frames } = await listen(root, false); + const before = await heapInUse(); + const submission = await root.submit({ type: "input", content: "go" }, context); + const partialContent = async () => + (await harness.snapshot(LiveDoc, root.id, context))?.generation?.message?.content; + const shown = async (index: number): Promise => { + const block = (await partialContent())?.[index]; + if (block?.type === "text") return block.text.length; + return block?.type === "toolCall" ? String(block.arguments.content ?? "").length : 0; + }; + const retained: number[] = []; + // Measure while the text, then the tool call, is almost complete: the watcher's view holds the most then. + await waitFor(async () => (await shown(0)) > text.length * 0.95, 30_000); + retained.push((await heapInUse()) - before); + await waitFor(async () => (await shown(1)) > content.length * 0.95, 30_000); + retained.push((await heapInUse()) - before); + await submission.wait(context); + const commits = frames.length; + expect(commits).toBeGreaterThan(500); + // The two 140 KB strings, the watcher's recorded frames and the rest of the stream: about 3 MB. Sliced append ops + // made the watcher's view hold a copy of the text per commit: well over 100 MB. + for (const bytes of retained) expect(bytes).toBeLessThan(20_000_000); + await watch.stop(); + await harness.close(context); + }, 60_000); +}); diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index a2214c762..8e5aaa78a 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -1,10 +1,17 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { type AssistantMessage, type FauxResponseStep, fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, type ConversationView, + type ConversationWindow, defineDoc, + type EntryRecord, type Harness, InboxDoc, LiveDoc, @@ -13,8 +20,9 @@ import { UsageDoc, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; type Frame = { value: ConversationView; ops: readonly Op[] }; @@ -22,32 +30,44 @@ type Frame = { value: ConversationView; ops: readonly Op[] }; const MOUNTED = ["pi.agent", "pi.inbox", "pi.live", "pi.provider", "pi.usage"]; /** Start a watch of `conversation` that records its acquisition revision and every delivered frame. */ -async function record(conversation: Conversation) { - const watch = await conversation.watch(context); +async function record(conversation: Conversation, window?: ConversationWindow) { + const watch = await conversation.watch(context, window); const initial = watch.value; const frames: Frame[] = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); return { initial, frames, stop: () => watch.stop() }; } /** A freshly built view of `conversation`, for comparing with an advanced one. */ -async function fresh(conversation: Conversation): Promise { - const watch = await conversation.watch(context); +async function fresh(conversation: Conversation, window?: ConversationWindow): Promise { + const watch = await conversation.watch(context, window); const value = watch.value; await watch.stop(); return value; } -/** The view as committed state defines it, read without any mount: the active entries and the built-in documents. */ -async function committed(harness: Harness, conversation: Conversation, record: ConversationView["conversation"]) { +/** + * The view as committed state defines it, read without any mount: the newest `size` entries, the entry before them, and + * the built-in documents. + */ +async function committed( + harness: Harness, + conversation: Conversation, + record: ConversationView["conversation"], + size: number, +) { const docs: Record = {}; for (const token of [AgentDoc, LiveDoc, InboxDoc, ProviderDoc, UsageDoc] as const) { const value = await harness.snapshot(token as typeof LiveDoc, conversation.id, context); if (value !== undefined) docs[token.definition.kind] = value; } - return { conversation: record, entries: (await conversation.context(context)).entries, docs }; + const newest = [...(await conversation.entries({}, size + 1, undefined, context)).items]; + const before = newest.length > size ? newest.pop()!.id : undefined; + return { conversation: record, entries: newest.reverse(), ...(before === undefined ? {} : { before }), docs }; } /** Replay every frame's operations from `initial`, checking each delivered revision on the way. */ @@ -71,22 +91,26 @@ async function drained(): Promise { function touches(harness: Harness, conversation: Conversation): { count: number } { const counter = { count: 0 }; - harness.subscribeCommits((publication) => { - const touched = publication.changes.some( - (change) => - (change.type === "entry" && change.value.conversationId === conversation.id) || - (change.type === "document" && - change.conversationId === conversation.id && - MOUNTED.includes(change.record.kind) && - change.ops.length > 0), - ); - if (touched) counter.count++; + harness.observeCommits({ + advance: (publication) => { + const touched = publication.changes.some( + (change) => + (change.type === "entry" && change.value.conversationId === conversation.id) || + (change.type === "document" && + change.kind === "change" && + change.conversationId === conversation.id && + MOUNTED.includes(change.record.kind) && + change.ops.length > 0), + ); + if (touched) counter.count++; + }, + end: () => {}, }); return counter; } describe("conversation view", () => { - it("hydrates the active entries and the built-in documents", async () => { + it("hydrates the newest entries and the built-in documents", async () => { const setup = chatSetup(); setup.faux.setResponses([answer("hello")]); const { harness, root } = await openChat(new MemoryStorage(), setup); @@ -94,6 +118,7 @@ describe("conversation view", () => { const view = await fresh(root); expect(view.conversation).toEqual({ id: root.id }); expect(view.entries).toEqual(await allEntries(root)); + expect(view.before).toBeUndefined(); expect(Object.keys(view.docs).sort()).toEqual(MOUNTED); expect(view.docs["pi.live"]).toEqual({}); expect(view.docs["pi.inbox"]).toEqual({ items: [] }); @@ -117,8 +142,10 @@ describe("conversation view", () => { await submission.wait(context); await harness.waitForIdle(context); await drained(); - expect(frames).toHaveLength(touching.count); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + // One frame per stored commit, and one for the attempt, a deferred commit stored with the answer. + expect(frames).toHaveLength(touching.count + 1); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, last.entries.length)); expect(frames[0]!.ops).toContainEqual(["p", ["entries"], 0, 0, [expect.objectContaining({ kind: "pi.user" })]]); // Document operations keep their exact shape under the mount path. expect(frames.flatMap((frame) => frame.ops)).toContainEqual([ @@ -148,7 +175,7 @@ describe("conversation view", () => { await harness.close(context); }); - it("cuts the entries at a head marker, keeping the entries from its head", async () => { + it("keeps head markers in place: a summary and a reset are appended entries", async () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const note = (kind: string) => root.commit((tx) => tx.appendEntry(root.id, { kind }), context); await note("a"); @@ -160,28 +187,21 @@ describe("conversation view", () => { await root.reset(undefined, context); await drained(); const kinds = frames.map((frame) => frame.value.entries.map((entry) => entry.kind)); - expect(kinds).toEqual([["summary", "b", "c"], ["summary", "b", "c", "d"], ["pi.reset"]]); - expect(frames[0]!.ops).toEqual([["p", ["entries"], 0, 1, [summary]]]); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + expect(kinds).toEqual([ + ["a", "b", "c", "summary"], + ["a", "b", "c", "summary", "d"], + ["a", "b", "c", "summary", "d", "pi.reset"], + ]); + expect(frames[0]!.ops).toEqual([["p", ["entries"], 3, 0, [summary]]]); + // The summary's `head` tells a renderer where the entries it summarizes end. + expect(frames[0]!.value.entries[3]!.head).toBe(b.id); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, 6)); await stop(); await harness.close(context); }); - it("keeps only mounted entries for a raw head write that targets before the active range", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const old = await root.commit((tx) => tx.appendEntry(root.id, { kind: "old" }), context); - await root.reset(undefined, context); - const { frames, stop } = await record(root); - await root.commit((tx) => tx.appendEntry(root.id, { kind: "summary", head: old.id }), context); - await drained(); - // Model context now starts at `old` again, but the mount never held it (spec §12); a rebuilt mount shows it. - expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["summary"]); - await stop(); - expect((await fresh(root)).entries.map((entry) => entry.kind)).toEqual(["summary", "old"]); - await harness.close(context); - }); - - it("cuts a fork's view into its inherited entries", async () => { + it("shows a fork's inherited entries in ID order, its head markers in place", async () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const a = await root.commit((tx) => tx.appendEntry(root.id, { kind: "a" }), context); const b = await root.commit((tx) => tx.appendEntry(root.id, { kind: "b" }), context); @@ -190,8 +210,8 @@ describe("conversation view", () => { await fork.commit((tx) => tx.appendEntry(fork.id, { kind: "summary", head: b.id }), context); await drained(); expect(initial.entries.map((entry) => entry.id)).toEqual([a.id, b.id]); - expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["summary", "b"]); - expect(frames[0]!.value).toEqual(await committed(harness, fork, initial.conversation)); + expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["a", "b", "summary"]); + expect(frames[0]!.value).toEqual(await committed(harness, fork, initial.conversation, 3)); await stop(); await harness.close(context); }); @@ -236,14 +256,17 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); await drained(); expect(frames).toHaveLength(1); expect(frames[0]!.ops[0]?.[0]).not.toBe("r"); expect(applyImmutable(initial, frames[0]!.ops)).toEqual(frames[0]!.value); - expect(frames[0]!.value.entries).toHaveLength(101); + // The default window took 100 appends, then dropped its oldest 51 at the 101st, down to 50. + expect(frames[0]!.value).toEqual(await committed(harness, root, initial.conversation, 50)); await watch.stop(); await harness.close(context); }); @@ -273,7 +296,7 @@ describe("conversation view", () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const watch = await root.watch(context); await harness.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value.entries).toEqual([]); await expect(root.watch(context)).rejects.toThrow(); }); @@ -309,7 +332,9 @@ describe("conversation view", () => { const failing = await root.watch(context); const { frames, stop } = await record(root); const shared = await root.watch(context); - expect(failing.value).toBe(shared.value); + // Each observer has its own window; the documents are the mount's. + expect(failing.value).toEqual(shared.value); + expect(failing.value.docs).toBe(shared.value.docs); await shared.stop(); failing.start(async () => { throw new Error("listener failed"); @@ -317,7 +342,7 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "one" }), context); await root.commit((tx) => tx.appendEntry(root.id, { kind: "two" }), context); await drained(); - expect(await failing.closed).toMatchObject({ reason: "listener_error" }); + expect(await failing.closed).toMatchObject({ kind: "listener_error" }); expect(frames).toHaveLength(2); await stop(); await harness.close(context); @@ -347,8 +372,227 @@ describe("conversation view", () => { await drained(); expect(frames).toHaveLength(1); expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["a", "b"]); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation, 2)); await stop(); await harness.close(context); }); }); + +/** Append one entry of `kind` per commit; returns the entries. */ +async function notes(conversation: Conversation, count: number, kind = "note"): Promise { + const entries: EntryRecord[] = []; + for (let index = 0; index < count; index++) { + entries.push(await conversation.commit((tx) => tx.appendEntry(conversation.id, { kind, data: index }), context)); + } + return entries; +} + +const ids = (entries: readonly EntryRecord[]) => entries.map((entry) => entry.id); + +const gc = (() => { + setFlagsFromString("--expose-gc"); + return runInNewContext("gc") as () => void; +})(); + +describe("conversation view window", () => { + it("starts with the newest `min` entries and the entry before them, or the whole short transcript", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 10); + const view = await fresh(root, { min: 4 }); + expect(ids(view.entries)).toEqual(ids(written.slice(6))); + expect(view.before).toBe(written[5]!.id); + // The default window: the newest 50, here all ten, starting the transcript. + const all = await fresh(root); + expect(ids(all.entries)).toEqual(ids(written)); + expect(all.before).toBeUndefined(); + expect("before" in all).toBe(false); + await harness.close(context); + }); + + it("takes appends up to `max`, then drops its oldest in one splice down to `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 4); + const { initial, frames, stop } = await record(root, { min: 3, max: 5 }); + expect(ids(initial.entries)).toEqual(ids(written.slice(1))); + const appended = await notes(root, 3); + await drained(); + expect(frames.map((frame) => frame.value.entries.length)).toEqual([4, 5, 3]); + const [e5, e6, e7] = appended as [EntryRecord, EntryRecord, EntryRecord]; + expect(frames[0]!.ops).toEqual([["p", ["entries"], 3, 0, [e5]]]); + expect(frames[2]!.ops).toEqual([ + ["p", ["entries"], 5, 0, [e7]], + ["p", ["entries"], 0, 3, []], + ["s", ["before"], written[3]!.id], + ]); + expect(ids(frames[2]!.value.entries)).toEqual(ids([e5, e6, e7])); + expect(frames[2]!.value.before).toBe(written[3]!.id); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, 3)); + // Entries that stay keep their identity: a splice moves them, it never copies them. + expect(frames[2]!.value.entries[0]).toBe(frames[1]!.value.entries[3]); + await stop(); + await harness.close(context); + }); + + it("keeps the newest `min` of a commit that appends more than `max` at once", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + await notes(root, 2); + const { initial, frames, stop } = await record(root, { min: 2, max: 3 }); + const burst = await root.commit(async (tx) => { + const entries: EntryRecord[] = []; + for (let index = 0; index < 5; index++) entries.push(await tx.appendEntry(root.id, { kind: "burst" })); + return entries; + }, context); + await drained(); + expect(frames).toHaveLength(1); + expect(ids(frames[0]!.value.entries)).toEqual(ids(burst.slice(3))); + expect(frames[0]!.value.before).toBe(burst[2]!.id); + expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation, 2)); + await stop(); + await harness.close(context); + }); + + it("starts at `from` when the entries since fit in `max`, otherwise at the newest `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 30); + const window = { min: 5, max: 10 }; + // Ten entries since `from`: all of them, with the entry before `from`. + const within = await fresh(root, { ...window, from: written[20]!.id }); + expect(ids(within.entries)).toEqual(ids(written.slice(20))); + expect(within.before).toBe(written[19]!.id); + // Fewer than `min` since `from`: only those. + const short = await fresh(root, { ...window, from: written[28]!.id }); + expect(ids(short.entries)).toEqual(ids(written.slice(28))); + expect(short.before).toBe(written[27]!.id); + // Resuming after the newest entry: an empty window right after it. + const resumed = await fresh(root, { ...window, from: (written[29]!.id + 1) as EntryRecord["id"] }); + expect(resumed.entries).toEqual([]); + expect(resumed.before).toBe(written[29]!.id); + // Eleven since `from`: more than `max`, so the newest `min`; `before` shows the client a gap. + const beyond = await fresh(root, { ...window, from: written[19]!.id }); + expect(ids(beyond.entries)).toEqual(ids(written.slice(25))); + expect(beyond.before).toBe(written[24]!.id); + // From the transcript's start. + const whole = await fresh(root, { min: 40, from: written[0]!.id }); + expect(ids(whole.entries)).toEqual(ids(written)); + expect(whole.before).toBeUndefined(); + await harness.close(context); + }); + + it("advances a resumed window by appends from its start", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 5); + const { initial, frames, stop } = await record(root, { from: (written[4]!.id + 1) as EntryRecord["id"] }); + const later = await notes(root, 2); + await drained(); + expect(ids(frames.at(-1)!.value.entries)).toEqual(ids(later)); + expect(frames.at(-1)!.value.before).toBe(written[4]!.id); + expect(replay(initial, frames)).toEqual(frames.at(-1)!.value); + await stop(); + await harness.close(context); + }); + + it("uses `from` only for the first value: a `from` past the newest entry starts empty and takes every append", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 3); + const from = (written[2]!.id + 1000) as EntryRecord["id"]; + const { initial, frames, stop } = await record(root, { from }); + expect(initial.entries).toEqual([]); + expect(initial.before).toBe(written[2]!.id); + // An entry below `from` is still appended: the window continues from its attachment. + const [next] = await notes(root, 1); + expect(next!.id).toBeLessThan(from); + await drained(); + expect(frames.at(-1)!.value.entries).toEqual([next]); + expect(frames.at(-1)!.value.before).toBe(written[2]!.id); + await stop(); + await harness.close(context); + }); + + it("rejects window bounds that are not integers or put `max` below `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + await expect(root.watch(context, { min: 0 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { min: 1.5 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { min: 10, max: 9 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { from: Number.NaN as EntryRecord["id"] })).rejects.toThrow(RangeError); + const recorder = { advance: () => {}, end: () => {} }; + await expect(harness.observeConversation(root.id, recorder, context, { max: 0 })).rejects.toThrow(RangeError); + // `max` may equal `min`. + await (await root.watch(context, { min: 3, max: 3 })).stop(); + await harness.close(context); + }); + + it("gives each observer its own window over one mount", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 6); + const changes: ConversationView[] = []; + const observation = await harness.observeConversation( + root.id, + { advance: (call) => call.kind === "value" && changes.push(call.value), end: () => {} }, + context, + { min: 2, max: 3 }, + ); + const wide = await record(root, { min: 5 }); + expect(ids(observation.value.entries)).toEqual(ids(written.slice(4))); + expect(ids(wide.initial.entries)).toEqual(ids(written.slice(1))); + const [next] = await notes(root, 1); + await drained(); + expect(ids(changes[0]!.entries)).toEqual(ids([...written.slice(4), next!])); + expect(ids(wide.frames[0]!.value.entries)).toEqual(ids([...written.slice(1), next!])); + expect(changes[0]!.docs).toBe(wide.frames[0]!.value.docs); + observation.detach(); + await wide.stop(); + await harness.close(context); + }); + + it("bounds an overflow replacement by `max`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const watch = await root.watch(context, { min: 5, max: 6 }); + // Past 10,000 pending operations, the next call is one root replacement: here an append per commit, and a splice + // and a `before` every other commit. + for (let index = 0; index < 6000; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + } + const frames: Frame[] = []; + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); + await drained(); + expect(frames).toHaveLength(1); + expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); + expect(frames[0]!.value.entries.length).toBeLessThanOrEqual(6); + await watch.stop(); + await harness.close(context); + }, 30_000); + + it("lets a long-lived watch drop the entries that left its window", async () => { + // On SQLite, which keeps no entry objects, so only the watch can hold the first one. + const directory = await mkdtemp(join(tmpdir(), "pi-durable-view-")); + const storage = await openNodeSqliteStorage(join(directory, "session.sqlite")); + const { harness, root } = await openChat(storage, chatSetup()); + const watch = await root.watch(context, { min: 5, max: 10 }); + let first: WeakRef | undefined; + let count = 0; + watch.start( + onValue(async ({ value }) => { + count = value.entries.length; + if (first === undefined && value.entries[0] !== undefined) first = new WeakRef(value.entries[0]); + }), + ); + await notes(root, 1); + await waitFor(() => first !== undefined); + await notes(root, 200); + await drained(); + expect(count).toBeLessThanOrEqual(10); + for (let round = 0; round < 3 && first!.deref() !== undefined; round++) { + await drained(); + gc(); + } + expect(first!.deref()).toBeUndefined(); + await watch.stop(); + await harness.close(context); + await rm(directory, { recursive: true, force: true }); + }); +}); diff --git a/packages/durable/test/indexeddb-storage.test.ts b/packages/durable/test/indexeddb-storage.test.ts index 352b7ca0e..85ba1751d 100644 --- a/packages/durable/test/indexeddb-storage.test.ts +++ b/packages/durable/test/indexeddb-storage.test.ts @@ -3,11 +3,13 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; -import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb"; -import { describe, expect, it } from "vitest"; +import { IDBKeyRange as FakeIDBKeyRange, IDBObjectStore as FakeIDBObjectStore, IDBFactory } from "fake-indexeddb"; +import { describe, expect, it, vi } from "vitest"; import { idFromNumber } from "../src/ids.ts"; import { IndexedDbStorage } from "../src/storage/indexeddb.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -22,9 +24,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - Object.defineProperty(globalThis, "IDBKeyRange", { configurable: true, value: FakeIDBKeyRange }); const context = BACKGROUND_CONTEXT; @@ -42,10 +41,14 @@ class ReopeningStorage implements Storage { this.indexedDB = indexedDB; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await IndexedDbStorage.open(this.name, { indexedDB: this.indexedDB }); @@ -135,7 +138,7 @@ function pendingTask(id: TaskId, conversationId = ROOT_CONVERSATION_I } async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } it("persists minted IDs across reopen", async () => { @@ -164,15 +167,16 @@ it("rolls back table writes when a later document copy fails", async () => { history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record: source, content: { kind: "base", version: 1, value: { count: 1 } } }], - adopt, context, ); const taskId = await storage.mintId>(); const copyId = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -181,7 +185,6 @@ it("rolls back table writes when a later document copy fails", async () => { source: { id: sourceId, at: "current" }, }, ], - adopt, context, ), ).rejects.toThrow("was rejected"); @@ -199,7 +202,8 @@ it("scans entries and head markers through fork ancestry", async () => { await createRoot(storage); const rootFirst = await storage.mintId(); const rootMarker = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: { id: rootFirst, conversationId: ROOT_CONVERSATION_ID, kind: "first" } }, { @@ -212,24 +216,23 @@ it("scans entries and head markers through fork ancestry", async () => { }, }, ], - adopt, context, ); const childId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId, parent: { conversationId: ROOT_CONVERSATION_ID, at: rootMarker } }, }, ], - adopt, context, ); const childEntry = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: { id: childEntry, conversationId: childId, kind: "child" } }], - adopt, context, ); expect( @@ -279,7 +282,7 @@ it("keeps one readwrite transaction active through many mixed writes", async () }, ); } - await storage.commit(writes, adopt, context); + await commitWrites(storage, writes, context); expect((await storage.entry(lastEntryId!, context))?.entry.data).toEqual({ index: 39 }); expect(await storage.task(lastTaskId!, context)).toEqual(pendingTask(lastTaskId!)); expect((await storage.submission(lastSubmissionId!, context))?.requestId).toBe("bulk-39"); @@ -288,3 +291,171 @@ it("keeps one readwrite transaction active through many mixed writes", async () await storage.close(context); } }); + +describe("commit", () => { + /** An entry of the root conversation. */ + const entryWrite = (id: EntryId): StorageWrite => ({ + type: "entry", + value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "batch" }, + }); + const sourceRecord = (id: DocumentId, conversationId: ConversationId = ROOT_CONVERSATION_ID): DocumentCreate => ({ + id, + kind: "source", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }); + + /** Opens a storage with its root conversation, runs `test` on it, closes it. */ + async function withStorage(name: string, test: (storage: IndexedDbStorage) => Promise): Promise { + const storage = await IndexedDbStorage.open(name, { indexedDB: new IDBFactory() }); + try { + await createRoot(storage); + await test(storage); + } finally { + await storage.close(context); + } + } + + it("stores the batches before a refused one, and the storage goes on", () => + withStorage("batch-prefix", async (storage) => { + const first = await storage.mintId(); + const after = await storage.mintId(); + const receipts: [number, Seq][] = []; + const result = await storage.commit( + [[entryWrite(first)], [entryWrite(first)], [entryWrite(after)]], + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${first} already belongs to entry`); + expect(receipts.map(([index]) => index)).toEqual([0]); + expect((await storage.entry(first, context))?.commitSeq).toBe(receipts[0]![1]); + expect(await storage.entry(after, context)).toBeUndefined(); + // The prefix advanced the metadata: the next commit takes the next seq. + const next = await commitWrites(storage, [entryWrite(after)], context); + expect(next).toBe(receipts[0]![1] + 1); + })); + + it("resolves a copy against the state an earlier batch of the call left", () => + withStorage("batch-copy", async (storage) => { + const sourceId = await storage.mintId(); + const childId = await storage.mintId(); + const copyId = await storage.mintId(); + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: sourceRecord(sourceId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + [ + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [ + { type: "conversation", value: { id: childId } }, + { + type: "document.copy", + record: sourceRecord(copyId, childId), + source: { id: sourceId, at: "current" }, + }, + ], + ], + () => {}, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copyId, "current", context))?.value).toEqual({ count: 2 }); + })); + + it("refuses a batch whose copy fails before it writes anything, keeping the batches before it", () => + withStorage("batch-copy-refused", async (storage) => { + const sourceId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: sourceRecord(sourceId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + context, + ); + const first = await storage.mintId(); + const taskId = await storage.mintId>(); + const copyId = await storage.mintId(); + const result = await storage.commit( + [ + [entryWrite(first)], + // The task comes before the copy: the refusal must leave it unwritten. + [ + { type: "task", value: pendingTask(taskId) }, + { + type: "document.copy", + record: { ...sourceRecord(copyId), kind: "mismatch" }, + source: { id: sourceId, at: "current" }, + }, + ], + ], + () => {}, + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain(`Document copy ${copyId} was rejected`); + expect(await storage.entry(first, context)).toBeDefined(); + expect(await storage.task(taskId, context)).toBeUndefined(); + expect(await storage.document(copyId, "current", context)).toBeUndefined(); + })); + + it("rejects with nothing stored when IndexedDB fails during a batch's checks", () => + withStorage("batch-check-failure", async (storage) => { + const first = await storage.mintId(); + const second = await storage.mintId(); + const get = FakeIDBObjectStore.prototype.get; + const failure = new DOMException("disk gone", "UnknownError"); + const spy = vi.spyOn(FakeIDBObjectStore.prototype, "get").mockImplementation(function ( + this: InstanceType, + key: Parameters[0], + ) { + if (this.name === "record_ids" && key === second) throw failure; + return get.call(this, key); + }); + const receipts: number[] = []; + try { + await expect( + storage.commit([[entryWrite(first)], [entryWrite(second)]], (index) => receipts.push(index), context), + ).rejects.toBe(failure); + } finally { + spy.mockRestore(); + } + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + await commitWrites(storage, [entryWrite(first)], context); + expect(await storage.entry(first, context)).toBeDefined(); + })); + + it("rejects with nothing stored when IndexedDB fails during a batch's writes", () => + withStorage("batch-write-failure", async (storage) => { + const first = await storage.mintId(); + const second = await storage.mintId(); + // A function cannot be stored: the put throws IndexedDB's `DataCloneError`. + const unstorable = { + type: "entry", + value: { id: second, conversationId: ROOT_CONVERSATION_ID, kind: "batch", data: { f: () => {} } }, + } as unknown as StorageWrite; + const receipts: number[] = []; + await expect( + storage.commit([[entryWrite(first)], [unstorable]], (index) => receipts.push(index), context), + ).rejects.toMatchObject({ name: "DataCloneError" }); + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + expect(await storage.entry(second, context)).toBeUndefined(); + })); +}); diff --git a/packages/durable/test/jsonl-storage.test.ts b/packages/durable/test/jsonl-storage.test.ts index 8f16ae547..0a30ecf01 100644 --- a/packages/durable/test/jsonl-storage.test.ts +++ b/packages/durable/test/jsonl-storage.test.ts @@ -10,7 +10,9 @@ import { NodeExecutionEnv } from "../src/env/node.ts"; import { idFromNumber } from "../src/ids.ts"; import { JsonlStorage } from "../src/storage/jsonl/index.ts"; import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -24,9 +26,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; type StoredTask = TaskRecord; const tempDirectories = new Set(); @@ -69,10 +68,14 @@ class ReopeningStorage implements Storage { this.directory = directory; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await JsonlStorage.open(this.directory, new NodeExecutionEnv({ cwd: this.directory }), context); @@ -163,7 +166,7 @@ function terminalTask(id: TaskId): StoredTask { } async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } function sessionDocument(id: DocumentId, kind = "test.document"): DocumentCreate { @@ -332,8 +335,9 @@ describe("Pico JsonlStorage publication and recovery", () => { await createRoot(storage); const taskId = await storage.mintId>(); const documentId = await storage.mintId(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); - await storage.commit( + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); + await commitWrites( + storage, [ { type: "document.create", @@ -341,15 +345,15 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [] } }], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -357,14 +361,13 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 1 } }, }, ], - adopt, context, ); expect(await readLines(join(directory, `task-${taskId}.jsonl`))).toHaveLength(1); expect(await readLines(join(directory, `doc-${documentId}.jsonl`))).toHaveLength(1); - await storage.commit([{ type: "document.retire", id: documentId }], adopt, context); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id: documentId }], context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); expect(await readLines(join(directory, "main.jsonl"))).toHaveLength(7); expect(await fileExists(join(directory, `task-${taskId}.jsonl`))).toBe(false); @@ -380,12 +383,12 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const taskId = await storage.mintId>(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, "first") }, { type: "task", value: pendingTask(taskId, "second") }, ], - adopt, context, ); expect(await readLines(join(directory, `task-${taskId}.jsonl`))).toHaveLength(2); @@ -402,12 +405,12 @@ describe("Pico JsonlStorage publication and recovery", () => { const taskId = await storage.mintId>(); env.fail({ operation: "append", call: 1, mode: "short" }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, "first") }, { type: "task", value: pendingTask(taskId, `second-${"x".repeat(512)}`) }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -420,6 +423,46 @@ describe("Pico JsonlStorage publication and recovery", () => { expect((await stat(join(directory, `task-${taskId}.jsonl`))).size).toBe(0); }); + it("poisons the storage when an append fails after its batches applied, before a waiting read goes on (review)", async () => { + const directory = await tempDirectory(); + const gate = Promise.withResolvers(); + class FailingJoin extends NodeExecutionEnv { + fail = false; + override async joinPath(parts: string[], callContext: Context): Promise> { + if (!this.fail) return super.joinPath(parts, callContext); + await gate.promise; + return err(new FileError("unknown", "injected path resolution failure")); + } + } + const fs = new FailingJoin({ cwd: directory }); + const storage = await JsonlStorage.open(directory, fs, context); + fs.fail = true; + const conversationId = idFromNumber(100); + const committing = storage.commit( + [ + [{ type: "conversation", value: { id: conversationId } }], + [ + { + type: "document.create", + record: { id: idFromNumber(101), kind: "state", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: {} }, + }, + ], + ], + () => {}, + context, + ); + // Started while the batches are in memory and the append waits. + const read = storage.conversation(conversationId, context); + gate.resolve(); + await expect(committing).rejects.toThrow("poisoned"); + await expect(read).rejects.toThrow("poisoned"); + await storage.close(context); + const reopened = await openStorage(directory); + expect(await reopened.conversation(conversationId, context)).toBeUndefined(); + await reopened.close(context); + }); + it("serializes the complete candidate before I/O and leaves preparation failures usable", async () => { const directory = await tempDirectory(); const env = new InstrumentedEnv({ cwd: directory }); @@ -428,7 +471,8 @@ describe("Pico JsonlStorage publication and recovery", () => { env.clear(); const id = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "entry", @@ -440,16 +484,15 @@ describe("Pico JsonlStorage publication and recovery", () => { }, }, ], - adopt, context, ), ).rejects.toThrow(); expect(env.operations).toEqual([]); expect(await storage.entry(id, context)).toBeUndefined(); expect( - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "good" } }], - adopt, context, ), ).toBe(2); @@ -474,7 +517,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const secondId = await storage.mintId(); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -487,7 +531,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { text: "β" } }, }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -519,7 +562,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const secondId = await storage.mintId(); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -532,7 +576,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -551,7 +594,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const firstId = await storage.mintId(); const secondId = await storage.mintId(); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -564,7 +608,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); const expected = [ @@ -576,7 +619,8 @@ describe("Pico JsonlStorage publication and recovery", () => { expect(env.operations).toEqual(expected); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -584,7 +628,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { checkpoint: true } }, }, ], - adopt, context, ); expect(env.operations).toEqual([ @@ -598,7 +641,8 @@ describe("Pico JsonlStorage publication and recovery", () => { ]); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -609,15 +653,14 @@ describe("Pico JsonlStorage publication and recovery", () => { }, }, ], - adopt, context, ); expect(env.operations).toEqual(["append:main.jsonl"]); const taskId = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); env.clear(); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); expect(env.operations).toEqual([ "append:main.jsonl", ...(fsync ? ["flush:main.jsonl"] : []), @@ -639,7 +682,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -647,10 +691,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -658,13 +702,13 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -672,7 +716,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(4); @@ -694,7 +737,8 @@ describe("Pico JsonlStorage publication and recovery", () => { await createRoot(storage); const taskId = await storage.mintId>(); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -703,11 +747,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 1 } }, }, ], - adopt, context, ); env.fail({ operation: "remove", call: 1, mode }); - await expect(storage.commit([{ type: "document.retire", id }], adopt, context)).resolves.toBe(3); + await expect(commitWrites(storage, [{ type: "document.retire", id }], context)).resolves.toBe(3); expect(await storage.document(id, "current", context)).toBeUndefined(); await storage.close(context); @@ -726,7 +769,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env, { fsync: true }); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -734,12 +778,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); env.fail({ operation: "flush", call: 2, mode }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -747,7 +791,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(3); @@ -782,7 +825,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env, { fsync: true }); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -790,12 +834,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); env.fail({ operation: "flush", call: 3, mode }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -803,12 +847,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(3); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -816,7 +860,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, }, ], - adopt, context, ); await storage.close(context); @@ -834,9 +877,9 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env); await createRoot(storage); const id = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(id) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(id) }], context); env.fail({ operation: "remove", call: 1, mode }); - await expect(storage.commit([{ type: "task", value: terminalTask(id) }], adopt, context)).resolves.toBe(3); + await expect(commitWrites(storage, [{ type: "task", value: terminalTask(id) }], context)).resolves.toBe(3); expect(await storage.task(id, context)).toEqual(terminalTask(id)); await storage.close(context); @@ -852,7 +895,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -860,10 +904,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -871,10 +915,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 10 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -882,7 +926,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 11]] }, }, ], - adopt, context, ); expect(await readLines(join(directory, `doc-${id}.jsonl`))).toHaveLength(2); @@ -902,12 +945,13 @@ describe("Pico JsonlStorage publication and recovery", () => { history: "rewindable" as const, fork: "asOf" as const, }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 0 } } }], - adopt, context, ); - const changedAt = await storage.commit( + const changedAt = await commitWrites( + storage, [ { type: "document.change", @@ -915,10 +959,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -926,10 +970,9 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(await readLines(join(directory, `doc-${id}.jsonl`))).toHaveLength(3); const reopened = await openStorage(directory); @@ -947,7 +990,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const sessionId = await storage.mintId(); const latestId = await storage.mintId(); const taskDocumentId = await storage.mintId(); - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -972,17 +1016,16 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); - const retiredAt = await storage.commit( + const retiredAt = await commitWrites( + storage, [ { type: "document.retire", id: sessionId }, { type: "document.retire", id: latestId }, { type: "document.retire", id: taskDocumentId }, { type: "task", value: terminalTask(taskId) }, ], - adopt, context, ); for (const file of [ @@ -1012,7 +1055,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1020,7 +1064,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { text: "kept" } }, }, ], - adopt, context, ); const sidecarPath = join(directory, `doc-${documentId}.jsonl`); @@ -1035,9 +1078,9 @@ describe("Pico JsonlStorage publication and recovery", () => { expect((await stat(sidecarPath)).size).toBe(sidecarSize); expect((await stat(mainPath)).size).toBe(mainSize); expect( - await reopened.commit( + await commitWrites( + reopened, [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [] } }], - adopt, context, ), ).toBe(3); @@ -1048,8 +1091,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const taskId = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); const sidecarPath = join(directory, `task-${taskId}.jsonl`); expect(await fileExists(sidecarPath)).toBe(false); await writeFile( @@ -1067,7 +1110,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const reopened = await openStorage(directory); expect(await fileExists(sidecarPath)).toBe(false); expect(await reopened.task(taskId, context)).toEqual(terminalTask(taskId)); - expect(await reopened.commit([], adopt, context)).toBe(4); + const next = await reopened.mintId(); + expect(await commitWrites(reopened, [{ type: "conversation", value: { id: next } }], context)).toBe(4); }); it("fails open when confirmed sidecar data is missing", async () => { @@ -1075,7 +1119,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1083,7 +1128,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); await writeFile(join(directory, `doc-${documentId}.jsonl`), ""); @@ -1097,7 +1141,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1105,10 +1150,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1116,7 +1161,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); const path = join(directory, `doc-${documentId}.jsonl`); @@ -1151,7 +1195,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1159,7 +1204,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); const path = join(directory, `doc-${documentId}.jsonl`); @@ -1192,3 +1236,68 @@ describe("Pico JsonlStorage publication and recovery", () => { ).rejects.toThrow("Malformed complete doc-99.jsonl"); }); }); + +describe("JsonlStorage batches", () => { + it("appends a batch's markers in one write and recovers a shorter prefix from a torn one", async () => { + const directory = await tempDirectory(); + const storage = await openStorage(directory); + await createRoot(storage); + const documentId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: sessionDocument(documentId), + content: { kind: "base", version: 1, value: { n: 0 } }, + }, + ], + context, + ); + const mainPath = join(directory, "main.jsonl"); + const before = (await readFile(mainPath, "utf8")).length; + const entries = [ + await storage.mintId(), + await storage.mintId(), + await storage.mintId(), + ]; + const batches: StorageWrite[][] = entries.map((id, index) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + { + type: "document.change", + id: documentId, + content: { kind: "delta", version: 1, ops: [["s", ["n"], index + 1]] }, + }, + ]); + // Reads wait for the append: a read started while it runs sees every batch durable. + const result = storage.commit(batches, () => {}, context); + const read = storage.entry(entries[2]!, context); + expect(await result).toEqual({ stored: 3 }); + expect(await read).toBeDefined(); + const main = await readFile(mainPath, "utf8"); + const markers = main + .slice(before) + .split("\n") + .filter((line) => line.length > 0); + expect(markers).toHaveLength(3); + await storage.close(context); + openStorages.delete(storage); + + // Torn inside the last marker: the two complete ones survive, and the third's sidecar record is unconfirmed. + const tornAt = main.length - Math.floor(markers[2]!.length / 2); + await writeFile(mainPath, main.slice(0, tornAt)); + const reopened = await openStorage(directory); + expect(await reopened.entry(entries[0]!, context)).toBeDefined(); + expect(await reopened.entry(entries[1]!, context)).toBeDefined(); + expect(await reopened.entry(entries[2]!, context)).toBeUndefined(); + expect((await reopened.document(documentId, "current", context))?.value).toEqual({ n: 2 }); + const seq = await commitWrites( + reopened, + [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [["s", ["n"], 9]] } }], + context, + ); + expect(seq).toBe(5); + const again = await openStorage(directory); + expect((await again.document(documentId, "current", context))?.value).toEqual({ n: 9 }); + }); +}); diff --git a/packages/durable/test/live-support.ts b/packages/durable/test/live-support.ts new file mode 100644 index 000000000..794da6beb --- /dev/null +++ b/packages/durable/test/live-support.ts @@ -0,0 +1,110 @@ +// What tests of renderers' state read: every `pi.live` value of a conversation, and the results of nested calls. +import { + type ConversationId, + type ConversationView, + type Harness, + type LiveState, + NestedCallDoc, + type NestedCallState, + type NestedToolExecutionResult, + type NestedToolSlot, + type TaskGraph, + type TaskGraphNode, + type TaskId, + type ToolSlot, +} from "@earendil-works/pi-durable"; +import { context, documentChanges, followCommits } from "./session-support.ts"; + +/** + * Every view of `conversationId` from now on, one per commit that changes it, unfolded: a synchronous observer, so no + * transition is skipped. `live()` lists their `pi.live` values. + */ +export async function viewHistory(harness: Harness, conversationId: ConversationId) { + const views: ConversationView[] = []; + const observation = await harness.observeConversation( + conversationId, + { advance: (call) => call.kind === "value" && views.push(call.value), end: () => {} }, + context, + ); + views.push(observation.value); + return { + views, + live: (): LiveState[] => views.map((view) => (view.docs["pi.live"] ?? {}) as LiveState), + detach: () => observation.detach(), + }; +} + +/** A model-issued or nested slot, read through the fields of both. */ +type Slot = ToolSlot & Partial>; + +/** The slots of tool call `callId`, model-issued or nested, in each `pi.live` value; `undefined` where it is absent. */ +export function slotsOf(history: readonly LiveState[], callId: string): (Slot | undefined)[] { + return history.map( + (live) => [...(live.tools ?? []), ...(live.nestedTools ?? [])].find((slot) => slot.callId === callId) as Slot, + ); +} + +/** The index of the first value where call `callId` ended: its slot is done, or gone after it was listed. */ +export function endedAt(history: readonly LiveState[], callId: string): number { + const slots = slotsOf(history, callId); + const listed = slots.findIndex((slot) => slot !== undefined); + if (listed < 0) return -1; + return slots.findIndex((slot, index) => index >= listed && (slot === undefined || slot.status === "done")); +} + +/** The calls whose slot went `running`, in the order they started. */ +export function startOrder(history: readonly LiveState[]): string[] { + const started: string[] = []; + for (const live of history) { + for (const slot of [...(live.tools ?? []), ...(live.nestedTools ?? [])]) { + if (slot.status === "running" && !started.includes(slot.callId)) started.push(slot.callId); + } + } + return started; +} + +/** + * The result of every nested call from now on, by its tool task, read from the `NestedCallDoc` its terminal commit + * writes (spec §7.3), the place a client reads a live caller's nested results from. + */ +export function nestedResults(harness: Harness): Map { + const results = new Map(); + followCommits(harness, (publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind !== NestedCallDoc.definition.kind || change.kind === "retire") continue; + const state = change.value as NestedCallState; + if (state.taskId !== undefined && state.result !== undefined) { + results.set(state.taskId, state.result as unknown as NestedToolExecutionResult); + } + } + }); + return results; +} + +/** The result of nested call `callId`, found by the task its slot names. */ +export function nestedResultOf( + history: readonly LiveState[], + results: ReadonlyMap, + callId: string, +): NestedToolExecutionResult | undefined { + const slot = slotsOf(history, callId).find((each) => each?.taskId !== undefined); + return slot?.taskId === undefined ? undefined : results.get(slot.taskId); +} + +/** Every task graph from now on, one per change, unfolded. */ +export async function graphHistory(harness: Harness) { + const graphs: TaskGraph[] = []; + const observation = await harness.observeTaskGraph( + { advance: (call) => call.kind === "value" && graphs.push(call.value), end: () => {} }, + context, + ); + graphs.push(observation.value); + return { graphs, detach: () => observation.detach() }; +} + +/** The live `pi.generation` nodes of `graph`, in task ID order. */ +export function generations(graph: TaskGraph): TaskGraphNode[] { + return Object.values(graph.tasks) + .filter((node) => node.kind === "pi.generation") + .sort((a, b) => a.id - b.id); +} diff --git a/packages/durable/test/memory-storage.test.ts b/packages/durable/test/memory-storage.test.ts index 6312e9db7..1b4bc9c5b 100644 --- a/packages/durable/test/memory-storage.test.ts +++ b/packages/durable/test/memory-storage.test.ts @@ -3,13 +3,14 @@ import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; import { describe, expect, it } from "vitest"; import { idFromNumber, seqFromNumber } from "../src/ids.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; -import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { type EntryId, ROOT_CONVERSATION_ID, type Seq, type StorageWrite } from "../src/types.ts"; registerStorageConformance({ describe, expect, it }, "MemoryStorage", (use) => use(new MemoryStorage())); it("does not expose retained state through a prepared commit", async () => { const storage = new MemoryStorage(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, BACKGROUND_CONTEXT); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); const entryId = idFromNumber(2); const prepared = storage.prepareCommit([ { @@ -30,3 +31,41 @@ it("does not expose retained state through a prepared commit", async () => { expect(prepared.apply()).toBe(seqFromNumber(2)); expect((await storage.entry(entryId, BACKGROUND_CONTEXT))?.entry.data).toEqual({ nested: [1] }); }); + +it("delivers the receipts of the batches applied before one whose apply throws, then rejects", async () => { + const failure = new Error("apply failed"); + class FailingApply extends MemoryStorage { + failAt: Seq | undefined; + override prepareCommit(writes: readonly StorageWrite[], seq?: Seq) { + const prepared = super.prepareCommit(writes, seq); + if (prepared.seq !== this.failAt) return prepared; + return { + ...prepared, + apply: (): Seq => { + throw failure; + }, + }; + } + } + const storage = new FailingApply(); + const root = await commitWrites( + storage, + [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], + BACKGROUND_CONTEXT, + ); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + storage.failAt = seqFromNumber(root + 2); + const receipts: [number, Seq][] = []; + await expect( + storage.commit( + [first, second].map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "m" } }, + ]), + (index, seq) => receipts.push([index, seq]), + BACKGROUND_CONTEXT, + ), + ).rejects.toBe(failure); + expect(receipts).toEqual([[0, root + 1]]); + expect((await storage.entry(first, BACKGROUND_CONTEXT))?.commitSeq).toBe(root + 1); + expect(await storage.entry(second, BACKGROUND_CONTEXT)).toBeUndefined(); +}); diff --git a/packages/durable/test/nested-images-bench.ts b/packages/durable/test/nested-images-bench.ts index f8c0e6d19..858f21672 100644 --- a/packages/durable/test/nested-images-bench.ts +++ b/packages/durable/test/nested-images-bench.ts @@ -10,6 +10,7 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-images-bench.ts [count] */ + import { randomBytes } from "node:crypto"; import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -18,7 +19,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import type { ImageContent } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { arr, lit, num, obj, opt, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -95,11 +96,11 @@ function noWrites(): Written { }; } -const ImageArgument = Type.Object({ - type: Type.Literal("image"), - mimeType: Type.String(), - data: Type.Optional(Type.String()), - source: Type.Optional(Type.Object({ taskId: Type.Number() })), +const ImageArgument = obj({ + type: lit("image"), + mimeType: str(), + data: opt(str()), + source: opt(obj({ taskId: num() })), }); function registryFor(scenario: Scenario, probe: Probe) { @@ -107,7 +108,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const readImage = defineTool({ name: "read_image", description: "Read an image", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), // One image, so programs get it as an `ImageContent`. Unique per call: base64 of random bytes. execute: async () => ({ output: [ @@ -118,7 +119,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const classify = defineTool({ name: "classify", description: "Classify images", - parameters: Type.Object({ images: Type.Array(ImageArgument) }), + parameters: obj({ images: arr(ImageArgument) }), execute: async (args, api, callContext) => { for (const image of args.images) { let data = image.data; @@ -141,7 +142,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const script = defineTool({ name: "script", description: "Read and classify images", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), callers: ["model"], execute: async (args, api, callContext) => { for (let n = 0; n < args.count; n++) { @@ -189,12 +190,12 @@ async function runScenario(scenario: Scenario): Promise { let commits = 0; const written = noWrites(); const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; // SQLite commits synchronously, which can starve the timer; sample here too. sample(); - for (const write of writes) written[write.type] += JSON.stringify(write).length; - return commit(writes, adopt, commitContext); + for (const write of batches.flat()) written[write.type] += JSON.stringify(write).length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/nested-tools-bench.ts b/packages/durable/test/nested-tools-bench.ts index 57561608c..002383fec 100644 --- a/packages/durable/test/nested-tools-bench.ts +++ b/packages/durable/test/nested-tools-bench.ts @@ -9,13 +9,14 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-tools-bench.ts [streaming|nesting] */ + import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { type Conversation, createRegistry, @@ -103,7 +104,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB const blob = defineTool({ name: "blob", description: "Stream a blob", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (_args, api) => { for (let n = 0; n < CHUNKS; n++) { api.output(chunk); @@ -118,7 +119,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB defineTool({ name, description: `Call ${target} in parallel`, - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const calls = Array.from({ length: args.count }, (_, n) => api.executeTool(target, { n, count: FAN_OUT }, callContext, options), @@ -133,10 +134,10 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB let commits = 0; const written = noWrites(); const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - for (const write of writes) written[write.type] += JSON.stringify(write).length; - return commit(writes, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; + for (const write of batches.flat()) written[write.type] += JSON.stringify(write).length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/replay-support.ts b/packages/durable/test/replay-support.ts deleted file mode 100644 index a79f5057d..000000000 --- a/packages/durable/test/replay-support.ts +++ /dev/null @@ -1,216 +0,0 @@ -import type { JsonValue } from "@earendil-works/chord"; -import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { idFromNumber, seqFromNumber } from "../src/ids.ts"; -import { - type ConversationId, - type DocumentCreate, - type DocumentId, - type DocumentPoint, - type DocumentRecord, - type EntryId, - type LogRecord, - type LogWrite, - type ReplayableStorage, - ROOT_CONVERSATION_ID, - type Storage, - type StorageWrite, - type SubmissionId, - type SubmissionRecord, - type TaskId, - type TaskRecord, -} from "../src/types.ts"; - -const context = BACKGROUND_CONTEXT; -type StoredTask = TaskRecord; - -export const conversationId = (id: number) => idFromNumber(id); -export const entryId = (id: number) => idFromNumber(id); -export const taskId = (id: number) => idFromNumber>(id); -export const documentId = (id: number) => idFromNumber(id); - -export function task(id: number, status: "pending" | "running" | "terminal", phase = "work"): StoredTask { - const base = { - id: taskId(id), - conversationId: ROOT_CONVERSATION_ID, - kind: "test.task", - version: 1, - input: { id }, - background: false, - abortRequested: false, - }; - return status === "terminal" - ? { ...base, state: { status, outcome: { status: "completed", result: { phase } } } } - : { ...base, state: { status, checkpoint: { phase } } }; -} - -function submission(id: number, status: "queued" | "placed", entry?: number): SubmissionRecord { - return status === "queued" - ? { id: idFromNumber(id), conversationId: ROOT_CONVERSATION_ID, type: "input", status } - : { - id: idFromNumber(id), - conversationId: ROOT_CONVERSATION_ID, - type: "input", - status, - entry: entryId(entry!), - }; -} - -export const entry = (id: number, conversation = ROOT_CONVERSATION_ID, extra: object = {}): LogWrite => ({ - type: "entry", - value: { id: entryId(id), conversationId: conversation, kind: "message", data: { id }, ...extra }, -}); - -export const sessionDoc = (id: number, kind = "test.session"): DocumentCreate => ({ - id: documentId(id), - kind, - scope: { kind: "session" }, -}); -export const rewindableDoc = ( - id: number, - conversation = ROOT_CONVERSATION_ID, - kind = "test.rewindable", -): DocumentCreate => ({ - id: documentId(id), - kind, - scope: { kind: "conversation", conversationId: conversation }, - history: "rewindable", - fork: "asOf", -}); -export const latestDoc = (id: number, conversation = ROOT_CONVERSATION_ID, kind = "test.latest"): DocumentCreate => ({ - id: documentId(id), - kind, - scope: { kind: "conversation", conversationId: conversation }, - history: "latest", - fork: "current", -}); - -export const create = (record: DocumentCreate, value: Record, version = 1): LogWrite => ({ - type: "document.create", - record, - content: { kind: "base", version, value }, -}); -export const delta = (id: number, path: string, value: JsonValue, version = 1): LogWrite => ({ - type: "document.change", - id: documentId(id), - content: { kind: "delta", version, ops: [["s", [path], value]] }, -}); -export const base = (id: number, value: Record, version = 1): LogWrite => ({ - type: "document.change", - id: documentId(id), - content: { kind: "base", version, value }, -}); - -/** - * A history that exercises everything export has to preserve: reclaimed current-only content, rewindable history - * across a version transition, task documents, fork copies, retire and recreate, an empty lifetime, upserted records, - * and a head that no stamp records. - */ -export function seedBatches(): StorageWrite[][] { - return [ - [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], - [ - entry(2), - entry(3), - { type: "task", value: task(4, "pending") }, - { type: "submission", value: submission(5, "queued") }, - create(sessionDoc(6), { n: 0 }), - create(rewindableDoc(7), { a: 1 }), - ], - [delta(6, "n", 1), delta(7, "b", 1), { type: "task", value: task(4, "running") }, entry(8)], - [base(6, { n: 5 }), base(7, { a: 2 }, 2), { type: "submission", value: submission(5, "placed", 8) }], - [ - create({ id: documentId(9), kind: "test.task-doc", scope: { kind: "task", taskId: taskId(4) } }, { t: 1 }), - create(latestDoc(10), { x: 1 }), - ], - [ - { - type: "conversation", - value: { id: conversationId(11), parent: { conversationId: ROOT_CONVERSATION_ID, at: entryId(8) } }, - }, - { - type: "document.copy", - record: rewindableDoc(12, conversationId(11)), - source: { id: documentId(7), at: seqFromNumber(3) }, - }, - { - type: "document.copy", - record: latestDoc(13, conversationId(11)), - source: { id: documentId(10), at: "current" }, - }, - entry(14, conversationId(11), { head: entryId(14) }), - ], - [base(10, { x: 2 }), delta(7, "c", 1, 2)], - [ - { type: "task", value: task(4, "terminal") }, - delta(9, "t", 2), - { type: "document.retire", id: documentId(9) }, - delta(10, "y", 1), - ], - [ - { type: "document.retire", id: documentId(7) }, - create(rewindableDoc(15), { a: 3 }), - create(latestDoc(16, ROOT_CONVERSATION_ID, "test.flash"), { gone: true }), - { type: "document.retire", id: documentId(16) }, - delta(13, "copied", true), - ], - [{ type: "task", value: task(17, "pending") }], - ]; -} - -export async function seed(storage: Storage): Promise { - for (const batch of seedBatches()) await storage.commit(batch, () => {}, context); -} - -type Reads = Record; - -/** Every read a Session can make, at every point of the history, as one comparable value. */ -export async function reads(storage: Storage & Pick): Promise { - const head = await storage.head(context); - const out: Reads = { head }; - const conversations = (await storage.scanConversations({}, 1000, undefined, context)).items; - out.conversations = conversations; - for (const conversation of conversations) { - const entries = ( - await storage.scanEntries({ conversationId: conversation.id, order: "ascending" }, 1000, undefined, context) - ).items; - out[`entries:${conversation.id}`] = await Promise.all(entries.map((item) => storage.entry(item.id, context))); - out[`head-marker:${conversation.id}`] = await storage.findLatestHeadMarker(conversation.id, undefined, context); - } - const tasks = (await storage.scanTasks({}, 1000, undefined, context)).items; - out.tasks = tasks; - out.submissions = (await storage.scanSubmissions({}, 1000, undefined, context)).items; - out.submissionByStatus = (await storage.scanSubmissions({ status: "placed" }, 1000, undefined, context)).items; - out.tasksByStatus = (await storage.scanTasks({ status: "terminal" }, 1000, undefined, context)).items; - - const scopes: DocumentRecord["scope"][] = [ - { kind: "session" }, - ...conversations.map((conversation) => ({ kind: "conversation" as const, conversationId: conversation.id })), - ...tasks.map((item) => ({ kind: "task" as const, taskId: item.id })), - ]; - const points: DocumentPoint[] = ["current"]; - for (let seq = 1; seq <= (head ?? 0); seq++) points.push(seqFromNumber(seq)); - for (const scope of scopes) { - for (const at of points) { - const documents = (await storage.scanDocuments({ scope, at }, 1000, undefined, context)).items; - out[`documents:${JSON.stringify(scope)}:${at}`] = documents; - for (const record of documents) { - out[`document:${record.id}:${at}`] = await storage - .document(record.id, at, context) - .catch((error: unknown) => ({ rejected: error instanceof Error ? error.name : "unknown" })); - const address = { - kind: record.kind, - scope: record.scope, - ...(record.key === undefined ? {} : { key: record.key }), - }; - out[`find:${record.id}:${at}`] = await storage.findDocument(address, at, context); - } - } - } - return out; -} - -export async function collect(log: AsyncIterable): Promise { - const records: LogRecord[] = []; - for await (const record of log) records.push(record); - return records; -} diff --git a/packages/durable/test/scheduler-fanout-bench.ts b/packages/durable/test/scheduler-fanout-bench.ts index ed3bb2eaa..2d2fc381f 100644 --- a/packages/durable/test/scheduler-fanout-bench.ts +++ b/packages/durable/test/scheduler-fanout-bench.ts @@ -5,10 +5,11 @@ * * node --conditions=source --experimental-strip-types test/scheduler-fanout-bench.ts [calls...] */ + import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../src/index.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; @@ -20,13 +21,13 @@ async function run(calls: number): Promise<{ ms: number; commits: number }> { const leaf = defineTool({ name: "leaf", description: "Return at once", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async () => ({ output: [{ type: "text", text: "ok" }] }), }); const fan = defineTool({ name: "fan", description: "Call leaf in parallel", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const results = await Promise.all( Array.from({ length: args.count }, (_, n) => @@ -41,9 +42,9 @@ async function run(calls: number): Promise<{ ms: number; commits: number }> { const storage = new MemoryStorage(); let commits = 0; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - return commit(writes, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/search.test.ts b/packages/durable/test/search.test.ts new file mode 100644 index 000000000..c4510cd46 --- /dev/null +++ b/packages/durable/test/search.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { rankDocuments, type SearchDocument, tokenize } from "../src/search.ts"; + +function doc(name: string, text: string, namespaceText?: string): SearchDocument { + return namespaceText === undefined ? { name, text } : { name, text, namespaceText }; +} + +describe("tokenize", () => { + it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { + expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); + expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t，。!? 😀 ")).toEqual([]); + }); +}); + +describe("rankDocuments", () => { + const documents = [ + doc("mcp__github__list_issues", "List issues in a repository. state open or closed"), + doc("mcp__github__create_pull_request", "Open a pull request."), + doc("mcp__linear__search_issues", "Search Linear issues by text."), + doc("mcp__docs__search", "Search the documentation."), + ]; + + it("ranks by term relevance and respects the limit", () => { + expect(rankDocuments("issue", documents, 8)).toEqual(["mcp__linear__search_issues", "mcp__github__list_issues"]); + expect(rankDocuments("pull requests", documents, 8)[0]).toBe("mcp__github__create_pull_request"); + expect(rankDocuments("search", documents, 1)).toHaveLength(1); + expect(rankDocuments("closed", documents, 8)).toEqual(["mcp__github__list_issues"]); + }); + + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (text, query) => { + expect(rankDocuments(query, [...documents, doc("localized", text)], 8)).toEqual(["localized"]); + }); + + it("returns nothing for unknown or empty queries", () => { + expect(rankDocuments("kubernetes", documents, 8)).toEqual([]); + expect(rankDocuments("the", documents, 8)).toEqual([]); + expect(rankDocuments("未知", documents, 8)).toEqual([]); + }); + + it("searches the namespace text", () => { + const namespaced = [doc("mcp__x__run", "Run it.", "mcp__x Kubernetes cluster tools")]; + expect(rankDocuments("kubernetes", namespaced, 8)).toEqual(["mcp__x__run"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query is its name", () => { + const named = [ + doc("mcp__app__queue_read", "Read a queued message of this thread."), + doc( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans. Continue with afterPosition until hasMore is false.", + ), + ]; + for (const query of ["thread_read", "dev_app__thread_read", "mcp__dev-app__thread_read", "Thread_Read"]) { + expect(rankDocuments(query, named, 1)).toEqual(["mcp__dev-app__thread_read"]); + } + // Partial names still rank by BM25. + expect(rankDocuments("read", named, 1)).toEqual(["mcp__app__queue_read"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a name match above mentions in descriptions", () => { + const ci = [ + doc("mcp__ci__status", "Deploy status: last deploy, deploy log, deploy errors."), + doc( + "mcp__ci__deploy_release", + "Start a release of the current branch to an environment, run the migrations, switch traffic over, and wait until the new version is live and healthy.", + ), + ]; + expect(rankDocuments("deploy", ci, 1)).toEqual(["mcp__ci__deploy_release"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query contains its whole name", () => { + const peekaboo = "mcp__peekaboo Peekaboo controls macOS apps by app name."; + const named = [ + doc("mcp__peekaboo__see", "Capture the app window of the peekaboo app and describe the app.", peekaboo), + doc("mcp__peekaboo__menu", "Click a menu item of an app.", peekaboo), + doc( + "mcp__peekaboo__app", + "Launch, quit, relaunch, focus, hide, or switch applications. Targets an application by name, bundle identifier, or process ID, and can wait until it is ready.", + peekaboo, + ), + doc("mcp__app__queue_read", "Read a queued message of this thread."), + doc( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans.", + ), + ]; + expect(rankDocuments("peekaboo app", named, 1)).toEqual(["mcp__peekaboo__app"]); + expect(rankDocuments("open the peekaboo app tool", named, 1)).toEqual(["mcp__peekaboo__app"]); + expect(rankDocuments("read thread_read state", named, 1)).toEqual(["mcp__dev-app__thread_read"]); + // Without the whole name, mentions in descriptions decide. + expect(rankDocuments("describe window", named, 1)).toEqual(["mcp__peekaboo__see"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool matching more query terms first, whatever the scores", () => { + const peekaboo = [ + doc("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier."), + doc("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app."), + doc("mcp__peekaboo__window", "Move, resize, or close a window of an app."), + doc("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window."), + ]; + // `window` and `app` are whole names, but each covers one of the two terms. + expect(rankDocuments("screenshot window", peekaboo, 1)).toEqual(["mcp__peekaboo__image"]); + expect(rankDocuments("describe app", peekaboo, 1)).toEqual(["mcp__peekaboo__see"]); + // An exact name still ranks first. + expect(rankDocuments("window", peekaboo, 1)).toEqual(["mcp__peekaboo__window"]); + }); +}); diff --git a/packages/durable/test/session-adoption.test.ts b/packages/durable/test/session-adoption.test.ts index 205c30066..a83da3181 100644 --- a/packages/durable/test/session-adoption.test.ts +++ b/packages/durable/test/session-adoption.test.ts @@ -1,9 +1,22 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { defineDoc, MemoryStorage, type Seq, type StorageWrite } from "@earendil-works/pi-durable"; +import { + type CommitPublication, + type CommitResult, + defineDoc, + MemoryStorage, + type Seq, + type StorageWrite, +} from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { SessionImpl } from "../src/session/session.ts"; +import type { FileError, Result } from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; import { Transaction } from "../src/session/transaction.ts"; -import { context } from "./session-support.ts"; +import { JsonlStorage } from "../src/storage/jsonl/index.ts"; +import { ControlledStorage, context, createConversation, documentValues } from "./session-support.ts"; const CounterDoc = defineDoc<{ count: number }>({ checkpointWhen: () => false, @@ -13,27 +26,42 @@ const CounterDoc = defineDoc<{ count: number }>({ initial: () => ({ count: 0 }), }); -/** Memory storage whose next commit, once durable, misbehaves as `fault` says. */ -class FaultyStorage extends MemoryStorage { +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const NoteDoc = defineDoc<{ a?: number; b?: number }>({ + checkpointWhen: () => false, + kind: "adoption.note", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), +}); + +/** Controlled storage that records `adopt` and its own resolution in `order`; its next commit misbehaves as `fault` says. */ +class FaultyStorage extends ControlledStorage { fault: "reject_after_adopt" | "skip_adopt" | undefined; readonly order: string[] = []; - override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, callContext: Context) { + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + callContext: Context, + ): Promise { const fault = this.fault; this.fault = undefined; - if (fault === "skip_adopt") return super.commit(writes, () => {}, callContext); - const seq = await super.commit( - writes, - (committed) => { + if (fault === "skip_adopt") return super.commit(batches, () => {}, callContext); + const result = await super.commit( + batches, + (index, committed) => { this.order.push("adopt"); - adopt(committed); + receive(index, committed); this.order.push("adopted"); }, callContext, ); this.order.push("resolve"); if (fault === "reject_after_adopt") throw new Error("reclaim failed"); - return seq; + return result; } } @@ -52,8 +80,8 @@ afterEach(() => { vi.restoreAllMocks(); }); -describe("Session adoption inside Storage.commit()", () => { - it("adopts synchronously inside the commit, before Storage resolves", async () => { +describe("Session adoption after Storage stored a commit", () => { + it("takes the durability receipt inside the commit, and adopts once Storage resolved", async () => { const { storage, session } = open(); const adopt = vi.spyOn(Transaction.prototype, "adopt"); await bump(session); @@ -70,7 +98,7 @@ describe("Session adoption inside Storage.commit()", () => { storage.fault = "reject_after_adopt"; await expect(bump(session)).rejects.toThrow("reclaim failed"); expect(discard).not.toHaveBeenCalled(); - expect(await session.closed).toMatchObject({ reason: "failed", error: new Error("reclaim failed") }); + expect(await session.closed).toMatchObject({ kind: "failed", error: new Error("reclaim failed") }); }); it("discards the staged changes when Storage rejects before adoption", async () => { @@ -81,7 +109,7 @@ describe("Session adoption inside Storage.commit()", () => { vi.spyOn(storage, "commit").mockRejectedValueOnce(new Error("disk full")); await expect(bump(session)).rejects.toThrow("disk full"); expect(discard).toHaveBeenCalledTimes(1); - expect(await session.closed).toMatchObject({ reason: "failed" }); + expect(await session.closed).toMatchObject({ kind: "failed" }); }); it("fails the Session with its own adoption error, which never reaches Storage", async () => { @@ -94,13 +122,288 @@ describe("Session adoption inside Storage.commit()", () => { await expect(bump(session)).rejects.toBe(failure); // Storage saw a normal adoption and resolved. expect(storage.order.slice(-3)).toEqual(["adopt", "adopted", "resolve"]); - expect(await session.closed).toEqual({ reason: "failed", error: failure }); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); }); it("fails the Session when Storage resolves without adopting", async () => { const { storage, session } = open(); storage.fault = "skip_adopt"; await expect(bump(session)).rejects.toThrow("without adopting"); - expect(await session.closed).toMatchObject({ reason: "failed" }); + expect(await session.closed).toMatchObject({ kind: "failed" }); + }); +}); + +describe("publication after storage", () => { + it("runs its own observers once Storage resolved, and host listeners one microtask later", async () => { + const { storage, session } = open(); + session.followCommits(() => { + storage.order.push(`observed adopting=${storage.adopting}`); + }); + session.observeCommits({ + advance: () => { + storage.order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, + }); + await bump(session); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "observed adopting=false", "host adopting=false"]); + await session.close(context); + }); + + it("calls the host listeners registered when its dispatch runs, and none for a commit published without one", async () => { + const { session } = open(); + const calls: string[] = []; + const removed = session.observeCommits({ + advance: () => { + calls.push("removed"); + }, + end: () => {}, + }); + session.observeCommits({ + advance: () => { + calls.push("kept"); + }, + end: () => {}, + }); + // During the publication, after the dispatch was queued. + let changed = false; + session.followCommits(() => { + if (changed) return; + changed = true; + removed.detach(); + session.observeCommits({ + advance: () => { + calls.push("added"); + }, + end: () => {}, + }); + }); + await bump(session); + expect(calls).toEqual(["kept", "added"]); + // A Session with no host listener at the publication queues no dispatch, so one added during it misses it. + const bare = open(); + const late: Seq[] = []; + let added = false; + bare.session.followCommits(() => { + if (added) return; + added = true; + bare.session.observeCommits({ + advance: (publication) => { + late.push(publication.seq); + }, + end: () => {}, + }); + }); + await bump(bare.session); + expect(late).toEqual([]); + await bump(bare.session); + expect(late).toHaveLength(1); + await session.close(context); + await bare.session.close(context); + }); + + it("fails the Session after Storage returned for a throw of its own observers; the others and hosts got the commit", async () => { + const { storage, session } = open(); + const bug = new Error("observer bug"); + session.followCommits(() => { + storage.order.push("observed"); + throw bug; + }); + session.followCommits(() => { + storage.order.push("observed after"); + }); + session.observeCommits({ + advance: () => { + storage.order.push(`host failed=${session.failure !== undefined}`); + }, + end: () => {}, + }); + session.subscribeClose(() => { + storage.order.push("close"); + }); + await expect(bump(session)).resolves.toBeUndefined(); + // Published once Storage resolved; the Session failed only after the hosts got the commit. + expect(storage.order).toEqual([ + "adopt", + "adopted", + "resolve", + "observed", + "observed after", + "host failed=false", + "close", + ]); + expect(await session.closed).toEqual({ kind: "failed", error: bug }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 1 }); + await reopened.close(context); + }); + + it("publishes a commit that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + await bump(session); + const stored: Seq[] = []; + session.followCommits(() => { + storage.order.push("observed"); + }); + session.observeCommits({ + advance: (publication) => { + storage.order.push("host"); + stored.push(publication.seq); + }, + end: () => {}, + }); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(bump(session)).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "observed", "host"]); + expect(stored).toHaveLength(1); + expect(await session.closed).toMatchObject({ kind: "failed" }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 2 }); + await reopened.close(context); + }); + + it("keeps what host listeners get: exactly what was stored, held changes included", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const observed: ObservedCommit[] = []; + const stored: CommitPublication[] = []; + session.followCommits((publication) => { + storage.order.push("observed"); + observed.push(publication); + }); + session.observeCommits({ + advance: (publication) => { + storage.order.push("host"); + stored.push(publication); + }, + end: () => {}, + }); + storage.order.length = 0; + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + // Held in memory: observed at once, nothing stored. + expect(storage.order).toEqual(["observed"]); + await session.commit(async (tx) => { + (await tx.doc(NoteDoc, conversationId)).b = 2; + }, context); + expect(storage.order).toEqual(["observed", "adopt", "adopted", "resolve", "observed", "host"]); + expect(documentValues(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); + expect(stored).toHaveLength(1); + expect(documentValues(stored[0]!).map((change) => change.ops)).toEqual([ + [ + ["s", ["a"], 1], + ["s", ["b"], 2], + ], + ]); + expect(stored[0]!.seq).toBe((observed[1] as Extract).seq); + await session.close(context); + }); + + it("publishes a store of held changes to host listeners once Storage resolved", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + session.observeCommits({ + advance: () => { + storage.order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + await session.unloadDocuments(); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "host adopting=false"]); + await session.close(context); + }); + + it("publishes a JSONL commit once the storage reclaimed its sidecar and resolved", async () => { + /** Node environment that records each sidecar rename, the last step of a reclaim. */ + class RenameLog extends NodeExecutionEnv { + readonly order: string[] = []; + + override renameFile( + source: string, + destination: string, + renameContext: Context, + ): Promise> { + this.order.push(`rename ${basename(destination)}`); + return super.renameFile(source, destination, renameContext); + } + } + // A base at the second count makes JSONL rewrite the document's sidecar after adopting the commit. + const Rebased = defineDoc<{ count: number }>({ + checkpointWhen: (value) => value.count === 2, + kind: "adoption.rebased", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), + }); + const directory = await mkdtemp(join(tmpdir(), "pi-durable-adoption-")); + const env = new RenameLog({ cwd: directory }); + const session = new SessionImpl(await JsonlStorage.open(directory, env, context)); + const count = () => + session.commit(async (tx) => { + (await tx.doc(Rebased)).count++; + }, context); + try { + await count(); + session.observeCommits({ + advance: () => { + env.order.push("host"); + }, + end: () => {}, + }); + await count(); + env.order.push("resolved"); + expect(env.order).toEqual([expect.stringMatching(/^rename doc-\d+\.jsonl$/), "host", "resolved"]); + } finally { + await session.close(context); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("publishes a store of held changes that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const stored: (readonly unknown[])[] = []; + session.observeCommits({ + advance: (publication) => { + stored.push(documentValues(publication).map((change) => change.ops)); + }, + end: () => {}, + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(session.unloadDocuments()).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "adopted", "resolve"]); + expect(stored).toEqual([[[["s", ["a"], 1]]]]); + expect(await session.closed).toMatchObject({ kind: "failed", error: new Error("reclaim failed") }); }); }); diff --git a/packages/durable/test/session-cached-storage.test.ts b/packages/durable/test/session-cached-storage.test.ts new file mode 100644 index 000000000..1abbb44f2 --- /dev/null +++ b/packages/durable/test/session-cached-storage.test.ts @@ -0,0 +1,183 @@ +import type { JsonValue } from "@earendil-works/chord"; +import { describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { CachedStorage } from "../src/session/cached-storage.ts"; +import { MemoryStorage } from "../src/storage/memory.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { ConversationId, StorageWrite, TaskId, TaskRecord } from "../src/types.ts"; +import { context } from "./session-support.ts"; + +function task(id: number, status: "pending" | "running" = "pending"): TaskRecord { + return { + id: idFromNumber>(id), + conversationId: idFromNumber(1), + kind: "test.task", + version: 1, + input: null, + background: false, + abortRequested: false, + state: { status, checkpoint: { phase: "run" } }, + }; +} + +class CountingStorage extends MemoryStorage { + taskReads = 0; + override task(...args: Parameters) { + this.taskReads++; + return super.task(...args); + } +} + +/** Stores `writes` as one batch, then publishes it, as the Session does. */ +async function commit(storage: CachedStorage, writes: StorageWrite[]): Promise { + await commitWrites(storage, writes, context); + storage.published(writes); +} + +describe("cached storage", () => { + it("serves a written record without a read and follows later writes", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + await commit(storage, [{ type: "task", value: task(2) }]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("pending"); + await commit(storage, [{ type: "task", value: task(2, "running") }]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect(backend.taskReads).toBe(0); + }); + + it("keeps a read only when no commit overlapped it", async () => { + const backend = new CountingStorage(); + await commitWrites( + backend, + [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + ], + context, + ); + const storage = new CachedStorage(backend); + const overlapped = storage.task(idFromNumber(2), context); + const written = commit(storage, [{ type: "task", value: task(2, "running") }]); + await Promise.all([overlapped, written]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + await storage.task(idFromNumber(2), context); + expect(backend.taskReads).toBe(1); + }); + + it("keeps a bounded number of ended tasks, and every live one until it ends", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + for (let id = 2; id < 600; id++) await commit(storage, [{ type: "task", value: task(id) }]); + // More live tasks than the ended table holds: each one's next change finds it without a read. + expect(storage.sizes).toMatchObject({ tasks: 0, liveTasks: 598 }); + for (let id = 2; id < 600; id++) await storage.task(idFromNumber(id), context); + expect(backend.taskReads).toBe(0); + for (let id = 2; id < 600; id++) { + const ended = { ...task(id), state: { status: "terminal", outcome: { status: "completed", result: null } } }; + await commit(storage, [{ type: "task", value: ended as TaskRecord }]); + } + expect(storage.sizes).toMatchObject({ tasks: 256, liveTasks: 0 }); + }); + + it("applies a batch stored with commit when it is published, not at its receipt", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + ]); + const batch: StorageWrite[] = [{ type: "task", value: task(2, "running") }]; + const receipts: number[] = []; + expect(await storage.commit([batch], (index) => receipts.push(index), context)).toEqual({ stored: 1 }); + expect(receipts).toEqual([0]); + // Stored, not yet published: the cache still shows what was published. + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("pending"); + storage.published(batch); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect(backend.taskReads).toBe(0); + }); + + it("drops a read a publication of its record overlapped, and keeps versions only for reads in flight", async () => { + const backend = new CountingStorage(); + await commitWrites( + backend, + [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + { type: "task", value: task(3) }, + ], + context, + ); + const storage = new CachedStorage(backend); + const overlapped = storage.task(idFromNumber(2), context); + const untouched = storage.task(idFromNumber(3), context); + expect(storage.readsInFlight).toBe(2); + // A publication of another record does not drop the read of task 3. + storage.published([{ type: "task", value: task(2, "running") }]); + await Promise.all([overlapped, untouched]); + expect(storage.readsInFlight).toBe(0); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect((await storage.task(idFromNumber(3), context))?.state.status).toBe("pending"); + expect(backend.taskReads).toBe(2); + }); + + it("keeps the ended tasks a task waits on until it moved on from its wait", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + const ended = (id: number) => + ({ + ...task(id), + state: { status: "terminal", outcome: { status: "completed", result: id } }, + }) as TaskRecord; + const waiting = (state: TaskRecord["state"]) => + ({ ...task(2), state }) as TaskRecord; + const on = [3, 4].map((id) => idFromNumber(id)); + await commit(storage, [ + { + type: "task", + value: waiting({ status: "waiting", checkpoint: { phase: "read" }, on, policy: "allSettled" }), + }, + ]); + await commit(storage, [ + { type: "task", value: ended(3) }, + { type: "task", value: ended(4) }, + ]); + // A burst of other ended tasks fills the bounded table. + for (let id = 10; id < 300; id++) await commit(storage, [{ type: "task", value: ended(id) }]); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + // Resumed: still held for the phase that reads the outcomes. + await commit(storage, [{ type: "task", value: waiting({ status: "running", checkpoint: { phase: "read" } }) }]); + for (const id of on) expect((await storage.task(id, context))?.state.status).toBe("terminal"); + expect(backend.taskReads).toBe(0); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + // Its next change ends the hold: the records go back to the bounded table, and no wait is left. + await commit(storage, [{ type: "task", value: waiting({ status: "running", checkpoint: { phase: "next" } }) }]); + expect(storage.sizes).toMatchObject({ awaited: 0, waits: 0, tasks: 256 }); + // A wait that ends with its task leaves nothing either. + await commit(storage, [ + { + type: "task", + value: waiting({ status: "waiting", checkpoint: { phase: "read" }, on, policy: "allSettled" }), + }, + ]); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + await commit(storage, [{ type: "task", value: ended(2) }]); + expect(storage.sizes).toMatchObject({ awaited: 0, waits: 0 }); + }); + + it("keeps the written record itself, frozen: no copy", async () => { + const storage = new CachedStorage(new MemoryStorage()); + const conversation = { id: idFromNumber(1) }; + const record = task(2); + storage.published([ + { type: "conversation", value: conversation }, + { type: "task", value: record }, + ]); + expect(await storage.task(idFromNumber(2), context)).toBe(record); + expect(await storage.conversation(idFromNumber(1), context)).toBe(conversation); + expect(Object.isFrozen(record.state)).toBe(true); + }); +}); diff --git a/packages/durable/test/session-checkpoints-migrations.test.ts b/packages/durable/test/session-checkpoints-migrations.test.ts index 98bb062ab..cc60b232e 100644 --- a/packages/durable/test/session-checkpoints-migrations.test.ts +++ b/packages/durable/test/session-checkpoints-migrations.test.ts @@ -7,7 +7,15 @@ import { type StorageWrite, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; -import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { + context, + createConversation, + documentChanges, + documentValues, + flush, + onValue, + openTestSession, +} from "./session-support.ts"; function documentWrites(writes: readonly StorageWrite[]): readonly StorageWrite[] { return writes.filter( @@ -94,7 +102,7 @@ describe("Session document checkpoints", () => { expect(falseCalls).toHaveLength(1); const snapshot = (await session.snapshot(BaseDoc, context))!; const deltaSnapshot = (await session.snapshot(DeltaDoc, context))!; - const [published, publishedDelta] = documentChanges(publications.at(-1)!); + const [published, publishedDelta] = documentValues(publications.at(-1)!); expect(calls[0]!.value).toBe(snapshot); expect(calls[0]!.ops).toBe(published!.ops); expect(falseCalls[0]!.value).toBe(deltaSnapshot); @@ -402,17 +410,21 @@ describe("Session document migrations", () => { // An observer of the older shape. const watch = (await session.watchDoc(Old, context))!; const frames: unknown[] = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); expect(await session.snapshot(Current, context)).toEqual({ count: 4 }); const snapshot = await session.snapshot(Current, context); // An observer of the new shape: the migration changes nothing it sees. const current = (await session.watchDoc(Current, context))!; const currentFrames: unknown[] = []; - current.start(async (value) => { - currentFrames.push(value); - }); + current.start( + onValue(async ({ value }) => { + currentFrames.push(value); + }), + ); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); @@ -488,7 +500,7 @@ describe("Session document migrations", () => { }), ]); await flush(); - const published = documentChanges(publications.at(-1)!)[0]!; + const published = documentValues(publications.at(-1)!)[0]!; const admitted = storage.admittedCommits.at(-1)![0]!; if (admitted.type !== "document.change" || admitted.content.kind !== "base") { throw new Error("Expected migration base"); @@ -836,10 +848,12 @@ describe("Session tracker cache across definition versions", () => { const watch = (await session.watchDoc(V1Doc, context))!; const frames: (readonly unknown[])[] = []; const delivered = new Promise((resolve) => { - watch.start(async (_value, ops) => { - frames.push(ops); - resolve(); - }); + watch.start( + onValue(async ({ ops }) => { + frames.push(ops); + resolve(); + }), + ); }); await session.commit(async (tx) => { (await tx.doc(V2Doc)).names.push("second"); diff --git a/packages/durable/test/session-conversation-tails.test.ts b/packages/durable/test/session-conversation-tails.test.ts new file mode 100644 index 000000000..9ed7e40e6 --- /dev/null +++ b/packages/durable/test/session-conversation-tails.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { ConversationTails, type HeadMarker } from "../src/session/conversation-tails.ts"; +import type { ConversationId, EntryId, EntryRecord, StorageWrite } from "../src/types.ts"; + +const conversation = (id: number) => idFromNumber(id); +const entryId = (id: number) => idFromNumber(id); + +function entry(id: number, conversationId: number, head?: number): StorageWrite { + const value: EntryRecord = { + id: entryId(id), + conversationId: conversation(conversationId), + kind: "note", + ...(head === undefined ? {} : { head: entryId(head) }), + }; + return { type: "entry", value }; +} + +const never = (): Promise => Promise.reject(new Error("read")); + +describe("conversation tails", () => { + it("reads once, then follows the commits", async () => { + const tails = new ConversationTails(); + let reads = 0; + const read = async () => { + reads++; + return entryId(5); + }; + expect(await tails.latest(conversation(1), read)).toBe(5); + expect(await tails.latest(conversation(1), read)).toBe(5); + expect(reads).toBe(1); + tails.apply([entry(9, 1)]); + expect(await tails.latest(conversation(1), never)).toBe(9); + }); + + it("knows a new root conversation is empty, and a fork only once read or written", async () => { + const tails = new ConversationTails(); + tails.apply([ + { type: "conversation", value: { id: conversation(2) } }, + { + type: "conversation", + value: { id: conversation(3), parent: { conversationId: conversation(2), at: entryId(4) } }, + }, + ]); + expect(await tails.latest(conversation(2), never)).toBeUndefined(); + expect(await tails.marker(conversation(2), undefined, never)).toBeUndefined(); + expect(await tails.latest(conversation(3), async () => entryId(4))).toBe(4); + }); + + it("answers a cutoff from the newest marker only at or after it", async () => { + const tails = new ConversationTails(); + tails.apply([{ type: "conversation", value: { id: conversation(1) } }, entry(10, 1), entry(11, 1, 10)]); + const marker = (await tails.marker(conversation(1), undefined, never)) as HeadMarker; + expect(marker.id).toBe(11); + expect((await tails.marker(conversation(1), entryId(12), never))?.id).toBe(11); + // An older cutoff may have an older marker: read, and not kept. + let reads = 0; + const older = async () => { + reads++; + return undefined; + }; + expect(await tails.marker(conversation(1), entryId(10), older)).toBeUndefined(); + expect(await tails.marker(conversation(1), entryId(10), older)).toBeUndefined(); + expect(reads).toBe(2); + expect((await tails.marker(conversation(1), undefined, never))?.id).toBe(11); + }); + + it("keeps a bounded number of conversations, fewer with markers", () => { + const tails = new ConversationTails(); + for (let id = 1; id <= 5000; id++) tails.apply([entry(2 * id, id), entry(2 * id + 1, id, 2 * id)]); + expect(tails.size).toBe(4096); + expect(tails.markers).toBe(256); + }); + + it("freezes the markers it keeps", () => { + const tails = new ConversationTails(); + const write = entry(2, 1, 1); + tails.apply([write]); + expect(Object.isFrozen((write as { value: object }).value)).toBe(true); + }); +}); diff --git a/packages/durable/test/session-deferred-commits.test.ts b/packages/durable/test/session-deferred-commits.test.ts new file mode 100644 index 000000000..7de47e659 --- /dev/null +++ b/packages/durable/test/session-deferred-commits.test.ts @@ -0,0 +1,661 @@ +// Deferred commits (`CommitPersistence`): what a deferred commit holds in memory, what stores it, and what Storage and +// each kind of listener then sees. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { JsonValue } from "@earendil-works/chord"; +import { applyImmutable } from "@earendil-works/chord/delta"; +import { + type CommitChange, + type CommitOptions, + type CommitPublication, + type ConversationId, + type DocumentId, + defineDoc, + type JsonObject, + ReadOnlySession, + type SetTimer, + type StorageWrite, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, createConversation, flush, onValue } from "./session-support.ts"; + +type Text = { text: string; base?: boolean; explode?: boolean; keys?: Record }; + +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const TextDoc = defineDoc({ + kind: "test.deferred.text", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + checkpointWhen: (value) => { + if (value.explode === true) throw new Error("checkpoint failed"); + return value.base === true; + }, +}); + +/** `TextDoc` at version 2, which a reload migrates to. */ +const TextDocV2 = defineDoc({ + kind: "test.deferred.text", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "", migrated: true }), + // `migrate()` gets the older version's untyped value: version 1's `Text`. + migrate: (value) => ({ ...(value as Text), migrated: true }), + checkpointWhen: () => false, +}); + +const RewindableDoc = defineDoc({ + kind: "test.deferred.rewindable", + version: 1, + scope: "conversation", + history: "rewindable", + fork: "asOf", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +const CurrentForkDoc = defineDoc({ + kind: "test.deferred.current-fork", + version: 1, + scope: "conversation", + history: "latest", + fork: "current", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +const SessionTextDoc = defineDoc({ + kind: "test.deferred.session", + version: 1, + scope: "session", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +/** Chord's `MAX_PENDING_OPS`: held operations past which a store writes the whole value. */ +const MAX_PENDING_OPS = 10_000; + +const DEFERRED: CommitOptions = { persistence: { kind: "deferred" } }; + +const IMMEDIATE: CommitOptions = { persistence: { kind: "immediate" } }; + +const within = (withinMs: number): CommitOptions => ({ persistence: { kind: "deferred", withinMs } }); + +/** One foreground timer the Session armed. */ +type Armed = { readonly callback: () => void; readonly delayMs: number; disarmed: boolean; fired: boolean }; + +/** + * A Session over controlled storage with a manual clock and manual foreground timers (background ones are declined, so + * commits evict), recording what `observeCommits()` and the Session's own observers get. + */ +async function open(options: { readonly retentionMs?: number; readonly persistWithinMs?: number } = {}) { + const clock = { now: 1_000 }; + const timers: Armed[] = []; + const setTimer: SetTimer = (callback, delayMs, { background }) => { + if (background) return undefined; + const timer: Armed = { callback, delayMs, disarmed: false, fired: false }; + timers.push(timer); + return () => { + timer.disarmed = true; + }; + }; + const storage = new ControlledStorage(); + const session = new SessionImpl( + storage, + () => clock.now, + () => options.retentionMs ?? 600_000, + setTimer, + false, + () => options.persistWithinMs, + ); + const stored: CommitPublication[] = []; + session.observeCommits({ + advance: (publication) => { + stored.push(publication); + }, + end: () => {}, + }); + const observed: ObservedCommit[] = []; + session.followCommits((publication) => { + observed.push(publication); + }); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(TextDoc, conversationId); + }, context); + const recordId = (await storage.findDocument(address(conversationId), "current", context))!.id; + return { + clock, + storage, + session, + stored, + observed, + conversationId, + recordId, + /** Timers armed and neither disarmed nor fired. */ + armed: () => timers.filter((timer) => !timer.disarmed && !timer.fired), + /** Fire every armed timer and let the line run what they queued. */ + async fire(): Promise { + for (const timer of timers) { + if (timer.disarmed || timer.fired) continue; + timer.fired = true; + timer.callback(); + } + await session.readOnLine(async () => {}); + }, + /** Batches admitted to Storage from here on. */ + mark(): () => (readonly StorageWrite[])[] { + const from = storage.commits.length; + return () => storage.commits.slice(from); + }, + async storedText(id: DocumentId = recordId): Promise { + return (await storage.document(id, "current", context))?.value; + }, + sizes: () => session.documentCacheSizes, + }; +} + +function address(conversationId: ConversationId) { + return { kind: TextDoc.definition.kind, scope: { kind: "conversation", conversationId } } as const; +} + +function append( + session: SessionImpl, + conversationId: ConversationId, + text: string, + options: CommitOptions = DEFERRED, +): Promise { + return session.commit( + async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += text; + }, + context, + options, + ); +} + +function changesOf( + batches: readonly (readonly StorageWrite[])[], +): Extract[] { + return batches.flat().filter((write) => write.type === "document.change"); +} + +describe("holding a deferred commit", () => { + it("adopts a commit that only changes an existing latest, initial-fork conversation document in memory, unstored", async () => { + const { session, storage, stored, observed, conversationId, sizes, storedText } = await open(); + const admitted = storage.commits.length; + const publications = stored.length; + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + expect(storage.commits.length).toBe(admitted); + expect(stored.length).toBe(publications); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "ab" }); + expect(await storedText()).toEqual({ text: "" }); + expect(sizes().held).toBe(1); + expect(observed.slice(-2).map((publication) => publication.kind)).toEqual(["memory", "memory"]); + }); + + it("stores a deferred commit at once when it writes anything else", async () => { + const { session, storage, conversationId, sizes } = await open(); + const stores = async (change: Parameters[0]): Promise => { + const admitted = storage.commits.length; + await session.commit(change, context, DEFERRED); + return storage.commits.length > admitted; + }; + // An entry next to the held document's change. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "x"; + await tx.appendEntry(conversationId, { kind: "test.note" }); + }), + ).toBe(true); + // Creation, then other document semantics. + expect(await stores(async (tx) => void (await tx.doc(RewindableDoc, conversationId)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(RewindableDoc, conversationId)).text = "r"; + }), + ).toBe(true); + expect(await stores(async (tx) => void (await tx.doc(CurrentForkDoc, conversationId)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(CurrentForkDoc, conversationId)).text = "c"; + }), + ).toBe(true); + expect(await stores(async (tx) => void (await tx.doc(SessionTextDoc)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(SessionTextDoc)).text = "s"; + }), + ).toBe(true); + // A held document next to an immediate one: the whole commit is stored. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "y"; + (await tx.doc(SessionTextDoc)).text += "s"; + }), + ).toBe(true); + // Retirement. + expect(await stores((tx) => tx.retireDoc(CurrentForkDoc, conversationId))).toBe(true); + expect(sizes().held).toBe(0); + // The control: the same change alone is held. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "z"; + }), + ).toBe(false); + }); + + it("stores a deferred commit that migrates its document at once, as a base", async () => { + const { session, conversationId, mark, sizes } = await open(); + const batches = mark(); + await session.commit( + async (tx) => { + (await tx.doc(TextDocV2, conversationId)).text = "v2"; + }, + context, + DEFERRED, + ); + expect(changesOf(batches()).map((write) => write.content.kind)).toEqual(["base"]); + expect(sizes().held).toBe(0); + }); + + it("rejects a deferred commit on a read-only Session like any commit with writes", async () => { + const { storage, conversationId } = await open(); + const viewer = new SessionImpl(storage, Date.now, () => 600_000, undefined, true); + await expect(append(viewer, conversationId, "a")).rejects.toBeInstanceOf(ReadOnlySession); + }); + + it("rejects a withinMs that is not a number of milliseconds, 0 or more", async () => { + const { session, conversationId } = await open(); + await expect(append(session, conversationId, "a", within(-1))).rejects.toBeInstanceOf(TypeError); + await expect(append(session, conversationId, "a", within(Number.NaN))).rejects.toBeInstanceOf(TypeError); + await expect(append(session, conversationId, "a", { persistence: { kind: "immediate" } })).resolves.toBe( + undefined, + ); + }); +}); + +describe("storing held changes", () => { + it("stores them first, folded with its own, in a commit that writes the document", async () => { + const { session, conversationId, mark, sizes, storedText } = await open(); + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + const batches = mark(); + await append(session, conversationId, "c", IMMEDIATE); + expect(batches()).toHaveLength(1); + expect(changesOf(batches()).map((write) => write.content)).toEqual([ + { version: 1, kind: "delta", ops: [["a", ["text"], "abc"]] }, + ]); + expect(await storedText()).toEqual({ text: "abc" }); + expect(sizes().held).toBe(0); + }); + + it("stores them in a commit that only reads the document, with its other writes", async () => { + const { session, conversationId, mark, storedText } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit(async (tx) => { + const text = (await tx.doc(TextDoc, conversationId)).text; + await tx.appendEntry(conversationId, { kind: "test.derived", data: text }); + }, context); + expect(batches()).toHaveLength(1); + expect( + batches()[0]! + .map((write) => write.type) + .sort(), + ).toEqual(["document.change", "entry"]); + expect(await storedText()).toEqual({ text: "held" }); + }); + + it("leaves them held through commits that neither read nor write the document, or write nothing", async () => { + const { session, conversationId, mark, sizes } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit((tx) => tx.appendEntry(conversationId, { kind: "test.unrelated" }).then(() => {}), context); + await session.commit(async (tx) => void (await tx.doc(TextDoc, conversationId)), context); + expect(changesOf(batches())).toEqual([]); + expect(sizes().held).toBe(1); + }); + + it("keeps them when a commit that reads the document fails before Storage", async () => { + const { session, conversationId, sizes, storedText, fire } = await open(); + await append(session, conversationId, "held"); + await expect( + session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "!"; + await tx.appendEntry(conversationId, { kind: "test.note" }); + throw new Error("callback failed"); + }, context), + ).rejects.toThrow("callback failed"); + // Its checkpoint predicate throws: settlement fails after every other step. + await expect( + session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).explode = true; + }, context), + ).rejects.toThrow("checkpoint failed"); + expect(sizes().held).toBe(1); + await fire(); + expect(await storedText()).toEqual({ text: "held" }); + }); + + it("stores every held document in one transaction when the earliest deadline comes", async () => { + const { session, conversationId, mark, armed, fire, stored, sizes, storage } = await open(); + const other = await createConversation(session); + await session.commit(async (tx) => void (await tx.doc(TextDoc, other)), context); + await append(session, conversationId, "one"); + await append(session, other, "two"); + expect(armed().map((timer) => timer.delayMs)).toEqual([1_000]); + const publications = stored.length; + const batches = mark(); + await fire(); + expect(batches()).toHaveLength(1); + expect(changesOf(batches())).toHaveLength(2); + expect(stored.length).toBe(publications + 1); + expect(sizes().held).toBe(0); + expect(armed()).toEqual([]); + const otherId = (await storage.findDocument(address(other), "current", context))!.id; + expect((await storage.document(otherId, "current", context))?.value).toEqual({ text: "two" }); + }); + + it("arms the earliest deadline of the held commits, each with its withinMs or the default", async () => { + const { session, conversationId, armed, clock } = await open({ persistWithinMs: 2_000 }); + await append(session, conversationId, "a"); + expect(armed().map((timer) => timer.delayMs)).toEqual([2_000]); + clock.now += 500; + // Earlier: the timer moves up. + await append(session, conversationId, "b", within(100)); + expect(armed().map((timer) => timer.delayMs)).toEqual([100]); + // Later: the deadline stays. + await append(session, conversationId, "c", within(60_000)); + expect(armed().map((timer) => timer.delayMs)).toEqual([100]); + }); + + it("arms no timer for an infinite window, and moves the timer on when a commit stores the held document", async () => { + const { session, conversationId, armed, sizes } = await open(); + await append(session, conversationId, "a", within(Number.POSITIVE_INFINITY)); + expect(armed()).toEqual([]); + expect(sizes().held).toBe(1); + await append(session, conversationId, "b"); + expect(armed().map((timer) => timer.delayMs)).toEqual([1_000]); + await append(session, conversationId, "c", IMMEDIATE); + expect(armed()).toEqual([]); + expect(sizes().held).toBe(0); + }); + + it("stores them at close, after the commits admitted before it, and closes Storage when that store fails", async () => { + const { session, storage, conversationId, recordId } = await open(); + await append(session, conversationId, "held"); + await session.close(context); + expect((await storage.reopen().document(recordId, "current", context))?.value).toEqual({ text: "held" }); + + const failing = await open(); + await append(failing.session, failing.conversationId, "lost"); + const failure = new Error("disk gone"); + failing.storage.failNextCommit(failure); + await expect(failing.session.close(context)).rejects.toBe(failure); + expect(await failing.session.closed).toEqual({ kind: "failed", error: failure }); + // Closed: the backend refuses calls until reopened. + await expect(failing.storage.document(failing.recordId, "current", context)).rejects.toThrow("closed"); + expect((await failing.storage.reopen().document(failing.recordId, "current", context))?.value).toEqual({ + text: "", + }); + }); + + it("stores nothing once the Session failed", async () => { + const { session, storage, conversationId, recordId, mark } = await open(); + await append(session, conversationId, "lost"); + const batches = mark(); + session.fail(new Error("crash")); + await session.closed; + expect(batches()).toEqual([]); + expect((await storage.reopen().document(recordId, "current", context))?.value).toEqual({ text: "" }); + }); + + it("stores them before unloadDocuments() drops the trackers", async () => { + const { session, conversationId, storedText, sizes } = await open(); + await append(session, conversationId, "held"); + await session.unloadDocuments(); + expect(await storedText()).toEqual({ text: "held" }); + expect(sizes()).toMatchObject({ cached: 0, held: 0 }); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "held" }); + }); + + it("stores them before a token of another definition version reloads the document", async () => { + const { session, conversationId, storedText, mark } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + expect(await session.snapshot(TextDocV2, conversationId, context)).toEqual({ text: "held", migrated: true }); + expect(changesOf(batches()).map((write) => write.content.kind)).toEqual(["delta"]); + expect(await storedText()).toEqual({ text: "held" }); + // The same inside a commit, which then migrates with a base. + await session.commit(async (tx) => void (await tx.doc(TextDoc, conversationId)), context); + await append(session, conversationId, "+"); + await session.commit(async (tx) => { + (await tx.doc(TextDocV2, conversationId)).text += "v2"; + }, context); + expect(await storedText()).toEqual({ text: "held+v2", migrated: true }); + }); + + it("stores them before the document's retirement, in the retiring commit", async () => { + const { session, conversationId, mark, sizes } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit((tx) => tx.retireDoc(TextDoc, conversationId), context); + expect(batches()[0]!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); + expect(changesOf(batches())[0]!.content).toEqual({ version: 1, kind: "delta", ops: [["a", ["text"], "held"]] }); + expect(sizes().held).toBe(0); + expect(await session.snapshot(TextDoc, conversationId, context)).toBeUndefined(); + }); +}); + +describe("bases and overflow", () => { + it("stores a base when a held commit selected one, and counts the store as one delta otherwise", async () => { + const { session, conversationId, mark, fire } = await open(); + const batches = mark(); + await session.commit( + async (tx) => { + const doc = await tx.doc(TextDoc, conversationId); + doc.text = "x"; + doc.base = true; + }, + context, + DEFERRED, + ); + await append(session, conversationId, "y"); + await fire(); + expect(changesOf(batches()).map((write) => write.content)).toEqual([ + { version: 1, kind: "base", value: { text: "xy", base: true } }, + ]); + }); + + it("stores the whole value once more than MAX_PENDING_OPS operations are held", async () => { + const { session, conversationId, mark, fire, stored } = await open(); + await overflow(session, conversationId); + const batches = mark(); + await fire(); + const value = (await session.snapshot(TextDoc, conversationId, context))!; + expect(changesOf(batches()).map((write) => write.content)).toEqual([{ version: 1, kind: "base", value }]); + // A durable listener replaces its copy with the stored value. + expect(stored.at(-1)!.changes).toMatchObject([{ type: "document", ops: [["r", value]] }]); + }); + + it("stores a base of the commit's own value when it stores an overflowed hold", async () => { + const { session, conversationId, mark } = await open(); + await overflow(session, conversationId); + await append(session, conversationId, "a"); + const batches = mark(); + await append(session, conversationId, "b", IMMEDIATE); + const value = (await session.snapshot(TextDoc, conversationId, context))!; + expect(value.text).toBe("ab"); + expect(changesOf(batches()).map((write) => write.content)).toEqual([{ version: 1, kind: "base", value }]); + }); +}); + +/** + * Hold more than `MAX_PENDING_OPS` operations: deferred commits of one set per key, each below the tracker's own limit, + * past which a commit is one root replacement. + */ +async function overflow(session: SessionImpl, conversationId: ConversationId): Promise { + await session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).keys = {}; + }, context); + for (let from = 0; from <= MAX_PENDING_OPS; from += 4_000) { + await session.commit( + async (tx) => { + const keys = (await tx.doc(TextDoc, conversationId)).keys!; + for (let index = from; index < from + 4_000; index++) keys[`k${index}`] = index; + }, + context, + DEFERRED, + ); + } +} + +describe("publications", () => { + it("give observeCommits() exactly what was stored, and nothing for a held commit", async () => { + const { session, storage, conversationId, recordId, stored, fire } = await open(); + const from = storage.commits.length; + const publications = stored.length; + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + await append(session, conversationId, "c", IMMEDIATE); + await append(session, conversationId, "d"); + await fire(); + await append(session, conversationId, "e"); + await session.commit(async (tx) => { + await tx.doc(TextDoc, conversationId); + await tx.appendEntry(conversationId, { kind: "test.note" }); + }, context); + await append(session, conversationId, "f"); + await session.close(context); + const batches = storage.commits.slice(from); + const published = stored.slice(publications); + expect(published).toHaveLength(batches.length); + // A replica built from the publications alone always equals what Storage holds. + let fromStorage: JsonValue = { text: "" }; + let fromPublications: JsonValue = { text: "" }; + for (const [index, batch] of batches.entries()) { + const write = changesOf([batch]).find((candidate) => candidate.id === recordId)!; + const change = published[index]!.changes.find( + (candidate): candidate is Extract => + candidate.type === "document" && candidate.kind === "change" && candidate.record.id === recordId, + )!; + fromStorage = + write.content.kind === "base" ? write.content.value : applyImmutable(fromStorage, write.content.ops); + fromPublications = applyImmutable(fromPublications, change.ops); + expect(fromPublications).toEqual(fromStorage); + expect(change.value).toEqual(fromStorage); + } + expect(fromStorage).toEqual({ text: "abcdef" }); + expect(published.map((publication) => publication.changes.map((change) => change.type).sort())).toEqual([ + ["document"], + ["document"], + ["document", "entry"], + ["document"], + ]); + }); + + it("show every held commit to watches and the Session's own observers, once", async () => { + const { session, conversationId, observed, fire } = await open(); + const watch = (await session.watchDoc(TextDoc, conversationId, context))!; + const values: string[] = []; + watch.start( + onValue(async ({ value }) => { + values.push(value.text); + }), + ); + const from = observed.length; + for (const text of ["a", "b", "c"]) { + await append(session, conversationId, text); + await flush(); + } + await append(session, conversationId, "d", IMMEDIATE); + await flush(); + await fire(); + expect(values).toEqual(["a", "ab", "abc", "abcd"]); + const seen = observed.slice(from); + expect(seen.map((publication) => publication.kind)).toEqual(["memory", "memory", "memory", "stored"]); + // The storing commit shows its own operations; the held ones were shown already. The timer showed nothing. + expect( + seen.map((publication) => + publication.changes.map((change) => change.type === "document" && change.kind === "change" && change.ops), + ), + ).toEqual([ + [[["a", ["text"], "a"]]], + [[["a", ["text"], "b"]]], + [[["a", ["text"], "c"]]], + [[["a", ["text"], "d"]]], + ]); + await watch.stop(); + }); +}); + +describe("pinning", () => { + it("keeps a held document cached with retention 0 until it is stored, then lets it go", async () => { + const { session, conversationId, sizes, fire } = await open({ retentionMs: 0 }); + await session.readOnLine(async () => {}); + expect(sizes()).toMatchObject({ cached: 0, held: 0, attached: 0 }); + await append(session, conversationId, "held"); + await session.readOnLine(async () => {}); + expect(sizes()).toMatchObject({ cached: 1, held: 1, attached: 1 }); + await fire(); + await session.commit(async () => {}, context); + expect(sizes()).toMatchObject({ cached: 0, held: 0, attached: 0 }); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "held" }); + }); + + it("retains a held revision only until it is stored", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const { session, conversationId, fire, stored, observed } = await open({ retentionMs: 0 }); + await append(session, conversationId, "x".repeat(1_000)); + const revision = new WeakRef((await session.snapshot(TextDoc, conversationId, context))!); + await collect(); + expect(revision.deref()).toBeDefined(); + await fire(); + await session.commit(async () => {}, context); + // The recorded publications would keep it. + stored.length = 0; + observed.length = 0; + await collect(); + expect(revision.deref()).toBeUndefined(); + }); +}); + +describe("crash", () => { + it("leaves Storage at the last stored state, which memory had, and nothing held after reopen", async () => { + const { session, storage, conversationId, recordId, fire } = await open(); + const states: string[] = []; + session.followCommits((publication) => { + for (const change of publication.changes) { + // Publications carry untyped document values; this one is `TextDoc`'s. + if (change.type === "document" && change.kind === "change" && change.record.id === recordId) + states.push((change.value as Text).text); + } + }); + await append(session, conversationId, "a"); + await fire(); + await append(session, conversationId, "b"); + await append(session, conversationId, "c"); + session.fail(new Error("crash")); + await session.closed; + const reopened = new SessionImpl(storage.reopen(), Date.now, () => 600_000); + const value = (await reopened.snapshot(TextDoc, conversationId, context))!; + expect(value).toEqual({ text: "a" }); + expect(states).toContain(value.text); + expect(reopened.documentCacheSizes.held).toBe(0); + await reopened.close(context); + }); +}); diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 6e2eed770..8d1b5e927 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -1,24 +1,24 @@ // The Session's document cache is bounded by use: unattached documents leave it after the retention period and load // from Storage again; documents with a watch or a document state never leave it. -import type { Context } from "@earendil-works/chord"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; + +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { - type AgentEvent, type ConversationId, defineDoc, defineDocFamily, defineTool, type HarnessSettings, + type LiveState, MemoryStorage, - type Seq, type StorageWrite, - watchEvents, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it, onTestFinished } from "vitest"; import { SessionImpl, setDefaultDocumentRetention } from "../src/session/session.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; -import { ControlledStorage, context, createConversation } from "./session-support.ts"; +import { viewHistory } from "./live-support.ts"; +import { ControlledStorage, context, createConversation, onValue } from "./session-support.ts"; import { deferred, eventually } from "./task-support.ts"; type Counter = { count: number }; @@ -155,14 +155,16 @@ describe("document cache retention", () => { const kept = (await session.watchDoc(CounterDoc, conversationId, context))!; kept.start(async () => {}); const seen: number[] = []; - watch.start(async (value) => { - seen.push(value?.count ?? -1); - }); + watch.start( + onValue(async ({ value }) => { + seen.push(value.count); + }), + ); clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); // Still cached: a read long after its retention passed loads nothing from Storage. - await eventually(() => seen.includes(2) && kept.value?.count === 2); + await eventually(() => seen.includes(2) && kept.value.count === 2); const reads = storage.documentReadCount; clock.now += 1_000_000; await session.commit(async (tx) => { @@ -175,7 +177,7 @@ describe("document cache retention", () => { clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); - await eventually(() => kept.value?.count === 3); + await eventually(() => kept.value.count === 3); await kept.stop(); clock.now += 1_000_000; await session.commit(async () => {}, context); @@ -216,9 +218,9 @@ describe("document cache retention", () => { const conversationId = await createConversation(session); const writes: StorageWrite[][] = []; const commit = storage.commit.bind(storage); - storage.commit = (batch: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) => { - writes.push([...batch]); - return commit(batch, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + for (const batch of batches) writes.push([...batch]); + return commit(batches, receive, commitContext); }; for (let n = 0; n < 5; n++) { await bump(session, conversationId); @@ -350,22 +352,27 @@ describe("document cache retention", () => { await session.close(context); }); - it("does not keep a replacement cached for an observer of the retired incarnation", async () => { + it("pins only what is at the address a watch follows: nothing while retired, then the new incarnation", async () => { const { session, sizes } = open(0); const conversationId = await createConversation(session); await bump(session, conversationId); const watch = (await session.watchDoc(CounterDoc, conversationId, context))!; watch.start(async () => {}); await session.commit((tx) => tx.retireDoc(CounterDoc, conversationId), context); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(watch.value).toBeNull(); - await bump(session, conversationId, 5); await eventually(async () => { await settleLine(session); return sizes().cached === 0; }); - expect(await session.snapshot(CounterDoc, conversationId, context)).toEqual({ count: 5 }); + expect(sizes().attached).toBe(0); + await bump(session, conversationId, 5); + await settleLine(session); + expect(sizes()).toMatchObject({ cached: 1, attached: 1 }); + await eventually(async () => watch.value.count === 5); await watch.stop(); + await eventually(async () => { + await settleLine(session); + return sizes().cached === 0; + }); await session.close(context); }); @@ -504,7 +511,7 @@ describe("document cache retention", () => { await harness.close(context); }); - it("gives the same events with documents evicted right after use as with documents kept", async () => { + it("gives the same views with documents evicted right after use as with documents kept", async () => { const run = async (documentRetentionMs: number): Promise => { const setup = chatSetup(); setup.settings.documentRetentionMs = documentRetentionMs; @@ -514,7 +521,7 @@ describe("document cache retention", () => { defineTool({ name: "steps", description: "Streams three steps", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (const step of ["a\n", "b\n", "c\n"]) { api.output(step); @@ -529,19 +536,18 @@ describe("document cache retention", () => { fauxAssistantMessage([fauxText("done")]), ]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const events: AgentEvent[] = []; - stream.start(async (batch) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); await harness.close(context); - // Ids and times differ between runs; the sequence of event types and tool outputs must not. - return events.map((event) => - event.type === "tool_execution_update" ? `update:${JSON.stringify(event.output)}` : event.type, - ); + // Ids and times differ between runs; the sequence of entries, run states, and tool outputs must not. + return history.views.map((view) => { + const live = (view.docs["pi.live"] ?? {}) as LiveState; + const tools = (live.tools ?? []).map((slot) => `${slot.callId}:${slot.status}:${slot.output ?? ""}`); + const kinds = view.entries.map((entry) => entry.kind).join(","); + return `${kinds} run:${live.run !== undefined} generation:${live.generation !== undefined} ${tools.join(" ")}`; + }); }; expect(await run(0)).toEqual(await run(600_000)); }); diff --git a/packages/durable/test/session-documents.test.ts b/packages/durable/test/session-documents.test.ts index b92f67776..0ae614d1d 100644 --- a/packages/durable/test/session-documents.test.ts +++ b/packages/durable/test/session-documents.test.ts @@ -8,7 +8,14 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { idFromNumber } from "../src/ids.ts"; -import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { + context, + createConversation, + documentChanges, + documentValues, + flush, + openTestSession, +} from "./session-support.ts"; type Live = { message?: string; items: string[]; nested: { count: number }; other: { label: string } }; @@ -103,7 +110,7 @@ describe("Session document transactions", () => { expect(snapshot).toEqual({ message: "hello", items: [], nested: { count: 0 }, other: { label: "x" } }); await flush(); const publication = publications.at(-1)!; - const published = documentChanges(publication)[0]!; + const published = documentValues(publication)[0]!; expect(published.record.createdAt).toBe(publication.seq); expect(published.value).toBe(snapshot); expect(published.conversationId).toBe(conversationId); @@ -148,7 +155,7 @@ describe("Session document transactions", () => { }, context); await flush(); const snapshot = (await session.snapshot(LiveDoc, conversationId, context))!; - const published = documentChanges(publications.at(-1)!)[0]!; + const published = documentValues(publications.at(-1)!)[0]!; expect(published.value).toBe(snapshot); const admitted = storage.admittedCommits.at(-1)!.find((write) => write.type === "document.change")!; expect(admitted.content.kind).toBe("delta"); @@ -206,7 +213,7 @@ describe("Session document transactions", () => { const after = (await session.snapshot(LiveDoc, conversationId, context))!; expect(after).toEqual(before); expect(after).not.toBe(before); - expect(documentChanges(publications.at(-1)!)[0]!.ops.length).toBeGreaterThan(0); + expect(documentValues(publications.at(-1)!)[0]!.ops.length).toBeGreaterThan(0); }); it("revokes escaped drafts when the callback settles", async () => { @@ -506,12 +513,14 @@ describe("Session document transactions", () => { await flush(); const publication = publications.at(-1)!; const [retired, created] = documentChanges(publication); - expect(retired).toMatchObject({ record: { id: oldId }, value: null, ops: [] }); - expect(retired!.record.retiredAt).toBe(publication.seq); + expect(retired).toEqual({ type: "document", kind: "retire", record: retired!.record, conversationId }); + expect(retired!.record).toMatchObject({ id: oldId, retiredAt: publication.seq }); expect(created!.record.id).not.toBe(oldId); expect(created!.record.createdAt).toBe(publication.seq); - expect(created).toMatchObject({ value: { message: "new", items: [] }, ops: [] }); - expect(await session.snapshot(LiveDoc, conversationId, context)).toBe(created!.value); + expect(created).toMatchObject({ kind: "change", value: { message: "new", items: [] }, ops: [] }); + expect(await session.snapshot(LiveDoc, conversationId, context)).toBe( + created?.kind === "change" ? created.value : undefined, + ); await session.commit((tx) => tx.retireDoc(LiveDoc, conversationId), context); expect(await session.snapshot(LiveDoc, conversationId, context)).toBeUndefined(); @@ -613,8 +622,11 @@ describe("Session document transactions", () => { const { session, publications, conversationId } = await setupLive(); const published = publications.length; let listenerContext: typeof context | undefined; - const unsubscribe = session.subscribeCommits((_publication, deliveredContext) => { - listenerContext = deliveredContext; + const observation = session.observeCommits({ + advance: (_publication, deliveredContext) => { + listenerContext = deliveredContext; + }, + end: () => {}, }); const result = await session.commit(async (tx) => { (await tx.doc(LiveDoc, conversationId)).message = "m"; @@ -623,7 +635,7 @@ describe("Session document transactions", () => { expect(result).toBe("done"); expect(listenerContext).toBe(context); expect(publications.length).toBe(published + 1); - unsubscribe(); + observation.detach(); }); it("publishes close synchronously and supports unsubscription", async () => { diff --git a/packages/durable/test/session-forks.test.ts b/packages/durable/test/session-forks.test.ts index 3c541b548..0755abe57 100644 --- a/packages/durable/test/session-forks.test.ts +++ b/packages/durable/test/session-forks.test.ts @@ -595,7 +595,7 @@ describe("Session conversation document forks", () => { session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context), ).rejects.toMatchObject({ name: "SessionFailed", cause: rejection }); // The failed Session closed its Storage; reopened, it holds nothing of the batch. - expect(await session.closed).toEqual({ reason: "failed", error: rejection }); + expect(await session.closed).toEqual({ kind: "failed", error: rejection }); expect(await storage.reopen().conversation(rejectedChildId, context)).toBeUndefined(); }); diff --git a/packages/durable/test/session-group-commit.test.ts b/packages/durable/test/session-group-commit.test.ts new file mode 100644 index 000000000..d135c4740 --- /dev/null +++ b/packages/durable/test/session-group-commit.test.ts @@ -0,0 +1,1017 @@ +// Group commit on the Session line (docs/group-commit.md): batching, gates, outcome fences, publication, failures per +// part, upkeep through the writer, backpressure, bounds, and a randomized interleaving against an oracle. +import type { Context } from "@earendil-works/chord"; +import { + type CommitOptions, + type CommitResult, + type ConversationId, + defineDoc, + type EntryId, + type Seq, + SessionFailed, + type StorageWrite, + type TaskId, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { CachedStorage } from "../src/session/cached-storage.ts"; +import type { ObservedCommit } from "../src/session/session.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { Transaction } from "../src/session/transaction.ts"; +import { ControlledStorage, context, flush, refusingAt } from "./session-support.ts"; +import { type Deferred, deferred, eventually, settled } from "./task-support.ts"; + +/** A latest, initial-fork conversation document: one key per conversation, and one a deferred commit may hold. */ +const Note = defineDoc<{ n: number }>({ + checkpointWhen: () => false, + kind: "gc.note", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ n: 0 }), +}); + +const Shared = defineDoc<{ n: number }>({ + checkpointWhen: () => false, + kind: "gc.shared", + version: 1, + scope: "session", + initial: () => ({ n: 0 }), +}); + +const DEFERRED = { persistence: { kind: "deferred", withinMs: Number.POSITIVE_INFINITY } } as const; + +/** + * Memory storage whose `commit()` calls are recorded and can be held in flight, and whose batches can be refused + * (failed before adoption) or whose calls can reject after their receipts. + */ +class BatchStorage extends ControlledStorage { + readonly calls: (readonly (readonly StorageWrite[])[])[] = []; + /** Refuse the next batch this matches, with `refusal.value`. */ + refuse: ((writes: readonly StorageWrite[]) => boolean) | undefined; + refusal: { readonly value: unknown } = { value: new Error("refused") }; + /** Reject the call with this index after its batches are stored, with `rejection.value`. */ + rejectCall: number | undefined; + rejection: { readonly value: unknown } = { value: new Error("connection lost") }; + /** Delay each call by up to this many milliseconds, at random. */ + latency: (() => number) | undefined; + #held: { readonly entered: Deferred; readonly gate: Deferred } | undefined; + + /** Hold the next call in flight until released. */ + hold(): { readonly entered: Promise; release(): void } { + const held = { entered: deferred(), gate: deferred() }; + this.#held = held; + return { entered: held.entered.promise, release: () => held.gate.resolve() }; + } + + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + const call = this.calls.length; + this.calls.push(batches); + const held = this.#held; + this.#held = undefined; + if (held !== undefined) { + held.entered.resolve(); + await held.gate.promise; + } + const delay = this.latency?.(); + if (delay !== undefined) await new Promise((resolve) => setTimeout(resolve, delay)); + const refuses = (writes: readonly StorageWrite[]): boolean => { + if (this.refuse?.(writes) !== true) return false; + this.refuse = undefined; + return true; + }; + const result = await refusingAt( + (...args) => super.commit(...args), + batches, + receive, + commitContext, + refuses, + this.refusal.value, + ); + if (this.rejectCall === call) throw this.rejection.value; + return result; + } +} + +async function open(conversations: number): Promise<{ + readonly storage: BatchStorage; + readonly session: SessionImpl; + readonly ids: ConversationId[]; + readonly published: ObservedCommit[]; +}> { + const storage = new BatchStorage(); + const session = new SessionImpl(storage, Date.now, () => 600_000); + const ids = await session.commit(async (tx) => { + const created: ConversationId[] = []; + for (let n = 0; n < conversations; n++) { + created.push((await tx.createConversation({ ownership: { kind: "ownerless" } })).id); + } + return created; + }, context); + const published: ObservedCommit[] = []; + session.followCommits((publication) => { + published.push(publication); + }); + return { storage, session, ids, published }; +} + +/** Increment conversation `id`'s note; resolves with the value it read. */ +function bump(session: SessionImpl, id: ConversationId, options?: CommitOptions): Promise { + return session.commit( + async (tx) => { + const note = await tx.doc(Note, id); + return note.n++; + }, + context, + options, + ); +} + +/** Record the order in which `promises` settle, by name; a rejection as `name!`. */ +function settleOrder(promises: Record>): string[] { + const order: string[] = []; + for (const [name, promise] of Object.entries(promises)) { + promise.then( + () => order.push(name), + () => order.push(`${name}!`), + ); + } + return order; +} + +/** The note values a reopened Session over `storage` reads, per conversation. */ +async function notes(storage: BatchStorage, ids: readonly ConversationId[]): Promise { + const reopened = new SessionImpl(storage.reopen()); + const values: number[] = []; + for (const id of ids) values.push((await reopened.snapshot(Note, id, context))?.n ?? 0); + await reopened.close(context); + return values; +} + +/** The kinds of the entries `published` holds, in order. */ +function entryKinds(published: readonly ObservedCommit[]): string[] { + return published.flatMap((publication) => + publication.kind === "stored" + ? publication.changes.flatMap((change) => (change.type === "entry" ? [change.value.kind] : [])) + : [], + ); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("group commit: batching", () => { + it("stores the commits admitted while a call is in flight in one call, each with its seq, published in order", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const ran: string[] = []; + const second = session.commit(async (tx) => { + (await tx.doc(Note, b)).n++; + ran.push("b"); + }, context); + const third = session.commit(async (tx) => { + (await tx.doc(Note, c)).n++; + ran.push("c"); + }, context); + // Their callbacks run while the first commit is in flight: they touch nothing it writes. + await eventually(() => ran.length === 2); + expect(published).toHaveLength(0); + const order = settleOrder({ a: first, b: second, c: third }); + held.release(); + await Promise.all([first, second, third]); + expect(order).toEqual(["a", "b", "c"]); + expect(storage.calls.slice(-2).map((call) => call.length)).toEqual([1, 2]); + const seqs = published.map((publication) => (publication.kind === "stored" ? publication.seq : 0)); + expect(seqs).toEqual([...seqs].sort((x, y) => x - y)); + expect(new Set(seqs).size).toBe(3); + await session.close(context); + }); +}); + +describe("group commit: gates", () => { + it("makes a read of an entry ID leaked by an unpublished commit wait for its publication", async () => { + const { storage, session, ids } = await open(1); + const held = storage.hold(); + let leaked: EntryId | undefined; + const appending = session.commit(async (tx) => { + leaked = (await tx.appendEntry(ids[0]!, { kind: "note" })).id; + }, context); + await held.entered; + const reading = session.commit((tx) => tx.entry(leaked!), context); + expect(await settled(reading)).toBe(false); + held.release(); + await appending; + expect((await reading)?.id).toBe(leaked); + await session.close(context); + }); + + it("makes a job wait for an unpublished creation at a document address instead of creating it again", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + const first = session.commit(async (tx) => { + (await tx.doc(Shared)).n++; + }, context); + await held.entered; + const second = session.commit(async (tx) => { + const shared = await tx.doc(Shared); + shared.n++; + return shared.n; + }, context); + expect(await settled(second)).toBe(false); + held.release(); + await first; + expect(await second).toBe(2); + const creations = storage.calls + .flat(2) + .filter((write) => write.type === "document.create" && write.record.kind === Shared.definition.kind); + expect(creations).toHaveLength(1); + await session.close(context); + }); + + it("makes a write to a conversation an unpublished commit creates wait for it", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + let leaked: ConversationId | undefined; + const creating = session.commit(async (tx) => { + leaked = (await tx.createConversation({ ownership: { kind: "ownerless" } })).id; + }, context); + await held.entered; + const appending = session.commit((tx) => tx.appendEntry(leaked!, { kind: "note" }), context); + expect(await settled(appending)).toBe(false); + held.release(); + await creating; + expect((await appending).conversationId).toBe(leaked); + await session.close(context); + }); + + it("makes each read of the entries of a conversation an unpublished commit creates wait for its creation", async () => { + // The Session's own helpers (newest entry, head marker), a transaction's scan, and the line's reads, each alone + // behind the creation, of a conversation Storage does not have yet. + const reads: Record Promise> = { + latestEntryId: (session, id) => session.commit((tx) => tx.latestEntryId(id), context), + latestHeadMarker: (session, id) => session.commit((tx) => tx.latestHeadMarker(id), context), + "tx.scanEntries": (session, id) => + session.commit(async (tx) => (await tx.scanEntries({ conversationId: id }, 1)).items, context), + "line scanEntries": (session, id) => + session.readOnLine( + async (line) => (await line.scanEntries({ conversationId: id }, 1, undefined, context)).items, + ), + "line findLatestHeadMarker": (session, id) => + session.readOnLine((line) => line.findLatestHeadMarker(id, undefined, context)), + }; + for (const [name, read] of Object.entries(reads)) { + const { storage, session } = await open(0); + const held = storage.hold(); + let leaked: ConversationId | undefined; + const creating = session.commit(async (tx) => { + leaked = (await tx.createConversation({ ownership: { kind: "ownerless" } })).id; + }, context); + await held.entered; + const reading = read(session, leaked!); + await flush(); + held.release(); + await creating; + const result = await reading.then( + (value) => ({ value }), + (error: unknown) => ({ error }), + ); + expect(result, name).toEqual({ value: Array.isArray((result as { value?: unknown }).value) ? [] : undefined }); + await session.close(context); + } + }); + + it("makes a scan that cannot be keyed wait for every unpublished commit", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + const creating = session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await held.entered; + const scanning = session.commit( + async (tx) => (await tx.scanConversations({}, 10)).items.map((c) => c.id), + context, + ); + expect(await settled(scanning)).toBe(false); + held.release(); + expect(await scanning).toContain(await creating); + await session.close(context); + }); + + it("makes a line read after an un-awaited commit see it", async () => { + const { storage, session, ids } = await open(1); + const held = storage.hold(); + let leaked: EntryId | undefined; + void session.commit(async (tx) => { + leaked = (await tx.appendEntry(ids[0]!, { kind: "note" })).id; + }, context); + await held.entered; + const reading = session.readOnLine((storage) => storage.entry(leaked!, context)); + await flush(); + held.release(); + expect((await reading)?.entry.id).toBe(leaked); + await session.close(context); + }); + + it("makes a reader of a document whose held changes an unpublished commit stores wait, and store them once", async () => { + const { storage, session, ids } = await open(1); + const id = ids[0]!; + await bump(session, id); + await bump(session, id, DEFERRED); + // The first commit only reads the held document: storing its held changes is its write of it. + const held = storage.hold(); + const storing = session.commit(async (tx) => { + await tx.doc(Note, id); + await tx.appendEntry(id, { kind: "a" }); + }, context); + await held.entered; + let read = false; + const reading = session.commit(async (tx) => { + await tx.doc(Note, id); + read = true; + await tx.appendEntry(id, { kind: "b" }); + }, context); + await flush(); + expect(read).toBe(false); + held.release(); + await Promise.all([storing, reading]); + const changes = storage.calls + .slice(-2) + .map((call) => call.flat().filter((write) => write.type === "document.change")); + expect(changes.map((writes) => writes.length)).toEqual([1, 0]); + const record = (await storage.findDocument( + { kind: Note.definition.kind, scope: { kind: "conversation", conversationId: id } }, + "current", + context, + ))!; + expect((await storage.document(record.id, "current", context))?.deltasSinceBase).toBe(1); + await session.close(context); + }); + + it("makes a reader of a document held by a deferred commit behind a call wait for it, and store it", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + const held = storage.hold(); + const first = bump(session, b); + await held.entered; + const deferredCommit = session.commit( + async (tx) => { + (await tx.doc(Note, a)).n = 5; + }, + context, + DEFERRED, + ); + const reader = bump(session, a); + await flush(); + // Nothing is published while the first commit is in flight, the deferred one neither. + expect(published).toHaveLength(1); + held.release(); + await Promise.all([first, deferredCommit]); + expect(await reader).toBe(5); + // The reader's own batch stored what the deferred commit held. + const stored = storage.calls.at(-1)!.flat(); + expect(stored.filter((write) => write.type === "document.change")).toHaveLength(1); + await session.close(context); + expect(await notes(storage, [a, b])).toEqual([6, 1]); + }); +}); + +describe("group commit: outcomes", () => { + it("delivers a read, a commit without writes, a callback error, and a deferred commit after an earlier commit", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const order = settleOrder({ + first, + read: session.readOnLine(async () => "read"), + none: session.commit(async () => "none", context), + error: session.commit(async () => { + throw new Error("own"); + }, context), + deferred: bump(session, b, DEFERRED), + }); + await flush(); + expect(order).toEqual([]); + held.release(); + await eventually(() => order.length === 5); + // A callback's own error affects only its job. + expect(order).toEqual(["first", "read", "none", "error!", "deferred"]); + await session.close(context); + }); + + it("makes every outcome after a fatal failure SessionFailed, and keeps the earlier ones", async () => { + const { storage, session, ids } = await open(4); + const [a, b, c, d] = ids as [ConversationId, ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const stored = bump(session, b); + const refused = session.commit((tx) => tx.appendEntry(c, { kind: "refuse" }), context); + const after = bump(session, d); + const read = session.readOnLine(async () => "read"); + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + // Every callback has run, so the three are admitted while the first call is in flight. + await flush(); + held.release(); + expect(await first).toBe(0); + expect(await stored).toBe(0); + await expect(refused).rejects.toThrow("refused"); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + await expect(read).rejects.toBeInstanceOf(SessionFailed); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("refused") }); + // One call stored the first two of the three batches it got; the refused one and the one after it are not stored. + expect(storage.calls.at(-1)).toHaveLength(3); + expect(await notes(storage, [a, b, d])).toEqual([1, 1, 0]); + }); +}); + +describe("group commit: failures per part", () => { + it("publishes the parts received before a call rejects, then rejects them with its error", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const second = bump(session, b); + const third = bump(session, c); + storage.rejectCall = storage.calls.length; + await flush(); + held.release(); + await first; + await expect(second).rejects.toThrow("connection lost"); + await expect(third).rejects.toThrow("connection lost"); + // Durable, so published. + expect(published).toHaveLength(3); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("connection lost") }); + expect(await notes(storage, [a, b, c])).toEqual([1, 1, 1]); + }); + + it("fails every job after the first part of a call that rejects, except its other parts, with their storage error", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const first = session.commit((tx) => tx.appendEntry(id, { kind: "a" }), context); + await held.entered; + // Between the two parts of the failing call, a job without writes. + const before = session.commit((tx) => tx.appendEntry(id, { kind: "b" }), context); + const between = session.commit(async () => "no write", context); + const after = session.commit((tx) => tx.appendEntry(id, { kind: "d" }), context); + const outcomes = Promise.allSettled([first, before, between, after]); + await flush(); + storage.rejectCall = storage.calls.length; + held.release(); + const [a, b, n, d] = await outcomes; + expect(a.status).toBe("fulfilled"); + expect(b).toMatchObject({ status: "rejected", reason: new Error("connection lost") }); + expect((n as PromiseRejectedResult).reason).toBeInstanceOf(SessionFailed); + expect(d).toMatchObject({ status: "rejected", reason: new Error("connection lost") }); + // Both parts were received, so they were published before they rejected. + expect(storage.calls.at(-1)).toHaveLength(2); + expect(published).toHaveLength(3); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("connection lost") }); + }); + + it("reports SessionFailed with the first error for a call that rejects after another failure failed the Session", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const inFlight = session.commit((tx) => tx.appendEntry(id, { kind: "a" }), context); + await held.entered; + // A later job's read fails first, and fails the Session; then the call in flight rejects with another error. + const first = new Error("first failure: read"); + vi.spyOn(storage, "task").mockRejectedValueOnce(first); + const reading = session.commit((tx) => tx.task(idFromNumber(999)), context); + reading.catch(() => {}); + await flush(); + expect(session.failure).toEqual({ error: first }); + storage.rejectCall = storage.calls.length - 1; + held.release(); + await expect(reading).rejects.toBe(first); + const outcome = await inFlight.then( + () => undefined, + (error: unknown) => error, + ); + expect(outcome).toBeInstanceOf(SessionFailed); + expect((outcome as SessionFailed).cause).toBe(first); + // Nothing of the call is published once the Session failed. + expect(published).toEqual([]); + expect(await session.closed).toEqual({ kind: "failed", error: first }); + }); + + for (const value of [undefined, null]) { + it(`rejects every part of a call with its rejection value, ${value} included`, async () => { + const { storage, session, ids } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const settle = (promise: Promise) => + promise.then( + () => ({ kind: "resolved" }), + (error: unknown) => ({ kind: "rejected", error }), + ); + const parts = [settle(bump(session, b)), settle(bump(session, c))]; + await flush(); + storage.rejection = { value }; + storage.rejectCall = storage.calls.length; + held.release(); + await first; + expect(await Promise.all(parts)).toEqual([ + { kind: "rejected", error: value }, + { kind: "rejected", error: value }, + ]); + expect(await session.closed).toEqual({ kind: "failed", error: value }); + }); + + it(`rejects a refused part with its refusal, ${value} included, and the parts after it with SessionFailed`, async () => { + const { storage, session, ids } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const refused = session + .commit((tx) => tx.appendEntry(b, { kind: "refuse" }), context) + .then( + () => ({ kind: "resolved" }), + (error: unknown) => ({ kind: "rejected", error }), + ); + const after = bump(session, c).then( + () => undefined, + (error: unknown) => error, + ); + await flush(); + storage.refusal = { value }; + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + held.release(); + await first; + expect(await refused).toEqual({ kind: "rejected", error: value }); + expect(await after).toBeInstanceOf(SessionFailed); + expect(await session.closed).toEqual({ kind: "failed", error: value }); + }); + } + + it("leaves the parts after one whose adoption throws unpublished, rejected with SessionFailed", async () => { + const { storage, session, ids, published } = await open(4); + const [a, b, c, d] = ids as [ConversationId, ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const before = bump(session, b); + const failing = bump(session, c); + const after = bump(session, d); + const failure = new Error("adoption failed"); + const adopt = Transaction.prototype.adopt; + let adoptions = 0; + vi.spyOn(Transaction.prototype, "adopt").mockImplementation(function (this: Transaction, seq: Seq) { + if (++adoptions === 3) throw failure; + return adopt.call(this, seq); + }); + await flush(); + held.release(); + await first; + await before; + await expect(failing).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toHaveLength(2); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + // All four are durable: reopening recovers the unpublished one. + vi.restoreAllMocks(); + expect(await notes(storage, [a, b, c, d])).toEqual([1, 1, 1, 1]); + }); +}); + +describe("group commit: a publication that throws", () => { + it("fails the Session for a deferred commit memory cannot adopt, and still closes", async () => { + const { session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b); + published.length = 0; + const failure = new Error("held adoption failed"); + vi.spyOn(Transaction.prototype, "adoptHeld").mockImplementationOnce(() => { + throw failure; + }); + const held = bump(session, a, DEFERRED); + const after = bump(session, b); + await expect(held).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toEqual([]); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + await session.close(context); + }); + + it("settles a deferred commit whose adoption throws, alone, and closes", async () => { + // Astra's: the only commit is the deferred one, on a storage that runs nothing else. + const storage = new ControlledStorage(); + const session = new SessionImpl(storage); + const id = await session.commit(async (tx) => { + const conversation = await tx.createConversation({ ownership: { kind: "ownerless" } }); + await tx.doc(Note, conversation.id); + return conversation.id; + }, context); + const failure = new Error("injected held adoption failure"); + vi.spyOn(Transaction.prototype, "adoptHeld").mockImplementationOnce(() => { + throw failure; + }); + await expect(bump(session, id, DEFERRED)).rejects.toBe(failure); + expect(session.failure).toEqual({ error: failure }); + await session.close(context); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + }); + + it("fails the Session for a throw in the read caches' write-through, and leaves the later parts unpublished", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const failing = bump(session, b); + const after = bump(session, c); + await flush(); + const failure = new Error("cache write failed"); + const write = CachedStorage.prototype.published; + let writes = 0; + vi.spyOn(CachedStorage.prototype, "published").mockImplementation(function (this: CachedStorage, batch) { + if (++writes === 2) throw failure; + return write.call(this, batch); + }); + held.release(); + await first; + await expect(failing).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toHaveLength(1); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + await session.close(context); + }); +}); + +describe("group commit: deferred commits", () => { + it("publishes a deferred commit only once the commit before it is published", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + // Only a change of an existing document is held. + await bump(session, b); + published.length = 0; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, DEFERRED); + await flush(); + expect(published).toHaveLength(0); + held.release(); + await Promise.all([first, later]); + expect(published.map((publication) => publication.kind)).toEqual(["stored", "memory"]); + await session.close(context); + }); + + it("never publishes a deferred commit behind a commit that fails", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, b); + published.length = 0; + const held = storage.hold(); + const first = session.commit((tx) => tx.appendEntry(a, { kind: "refuse" }), context); + await held.entered; + const later = bump(session, b, DEFERRED); + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + held.release(); + await expect(first).rejects.toThrow("refused"); + await expect(later).rejects.toBeInstanceOf(SessionFailed); + expect(published).toEqual([]); + }); +}); + +describe("group commit: publication", () => { + it("publishes the parts of one call a macrotask apart: what one resolves runs before the next is published", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const second = bump(session, b); + const third = bump(session, c); + let seenAtSecond: number | undefined; + void second.then(() => { + seenAtSecond = published.length; + }); + await flush(); + held.release(); + await Promise.all([first, second, third]); + expect(storage.calls.at(-1)).toHaveLength(2); + expect(seenAtSecond).toBe(2); + expect(published).toHaveLength(3); + await session.close(context); + }); +}); + +describe("group commit: the storage's publication scope", () => { + it("publishes each commit inside the storage's scope, the next only once one it runs later ran", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const scopes: string[] = []; + let depth = 0; + // Runs each publication a macrotask later, inside a scope of its own. + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = (publish) => { + setTimeout(() => { + depth++; + scopes.push(`open ${published.length}`); + publish(); + scopes.push(`close ${published.length}`); + depth--; + }, 0); + }; + let inside: boolean[] = []; + session.followCommits(() => { + inside.push(depth === 1); + }); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const rest = [bump(session, b), bump(session, c)]; + await flush(); + inside = []; + held.release(); + await Promise.all([first, ...rest]); + expect(inside).toEqual([true, true, true]); + expect(scopes).toEqual(["open 0", "close 1", "open 1", "close 2", "open 2", "close 3"]); + await session.close(context); + }); + + it("reports a scope that throws instead of running the publication, and publishes the commit", async () => { + const { storage, session, ids, published } = await open(1); + const reports: unknown[] = []; + session.report = (error) => { + reports.push(error); + }; + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = () => { + throw new Error("no scope"); + }; + expect(await bump(session, ids[0]!)).toBe(0); + expect(published).toHaveLength(1); + expect(reports).toEqual([new Error("no scope")]); + await session.close(context); + }); +}); + +describe("group commit: a failure between two publications", () => { + it("publishes no part of a call after a failure that a continuation of an earlier part caused", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + const two = session.commit((tx) => tx.appendEntry(id, { kind: "two" }), context); + two.catch(() => {}); + // What `one` resolves runs in the hop before `two` is published: a read there fails the Session. + const read = new Error("read failed between publications"); + vi.spyOn(storage, "task").mockRejectedValue(read); + const failing = one.then(() => session.commit((tx) => tx.task(idFromNumber(999)), context)); + failing.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one, two, failing]); + expect(entryKinds(published)).toEqual(["lead", "one"]); + const outcome = await two.then( + () => undefined, + (error: unknown) => error, + ); + expect(outcome).toBeInstanceOf(SessionFailed); + expect((outcome as SessionFailed).cause).toBe(read); + }); + + it("publishes no part of a call after the Session fails in a continuation of an earlier part", async () => { + // Sol's: `one`'s continuation fails the Session itself. + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const failure = new Error("failed between publications"); + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + void one.then(() => session.fail(failure)); + const two = session.commit((tx) => tx.appendEntry(id, { kind: "two" }), context); + two.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one, two]); + expect(entryKinds(published)).toEqual(["lead", "one"]); + await expect(two).rejects.toBeInstanceOf(SessionFailed); + }); + + it("checks the failure when a storage's scope runs a publication it deferred, not when it was asked to", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const failure = new Error("failed before a deferred publication ran"); + let calls = 0; + // The second publication's scope is entered before the failure; it runs the publication only after it. + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = (publish) => { + if (++calls === 2) session.fail(failure); + setTimeout(publish, 0); + }; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + one.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one]); + expect(entryKinds(published)).toEqual(["lead"]); + await expect(one).rejects.toBeInstanceOf(SessionFailed); + }); +}); + +describe("group commit: upkeep through the writer", () => { + it("stores held changes for an unload after the call in flight", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b, DEFERRED); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const unloading = session.unloadDocuments(); + expect(await settled(unloading)).toBe(false); + held.release(); + await Promise.all([first, unloading]); + expect(storage.calls.at(-1)!.flat()).toMatchObject([{ type: "document.change" }]); + expect(await session.snapshot(Note, b, context)).toEqual({ n: 1 }); + await session.close(context); + expect(await notes(storage, [a, b])).toEqual([2, 1]); + }); + + it("stores held changes for a version reload inside a callback after the call in flight", async () => { + const NoteV2 = defineDoc<{ n: number; migrated: true }>({ + checkpointWhen: () => false, + kind: "gc.note", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ n: 0, migrated: true }), + migrate: (value) => ({ ...(value as { n: number }), migrated: true }), + }); + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b); + await bump(session, b, DEFERRED); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const reloading = session.commit(async (tx) => (await tx.doc(NoteV2, b)).n, context); + expect(await settled(reloading)).toBe(false); + held.release(); + await first; + // The reload found what memory held, stored first. + expect(await reloading).toBe(2); + await session.close(context); + }); + + it("stores held changes at their deadline after the call in flight", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, { persistence: { kind: "deferred", withinMs: 0 } }); + await flush(); + const calls = storage.calls.length; + held.release(); + await Promise.all([first, later]); + await eventually(() => storage.calls.length === calls + 1); + expect(storage.calls.at(-1)!.flat()).toMatchObject([{ type: "document.create" }]); + await session.close(context); + }); + + it("drains the calls in flight and stores held changes when it closes", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, DEFERRED); + await flush(); + const closing = session.close(context); + held.release(); + await Promise.all([first, later]); + await closing; + expect(await session.closed).toEqual({ kind: "closed" }); + expect(await notes(storage, [a, b])).toEqual([1, 1]); + }); +}); + +describe("group commit: backpressure and bounds", () => { + it("starts no job past 256 without a delivered outcome, and returns its line state to zero when idle", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + let ran = 0; + const appends: Promise[] = []; + for (let n = 0; n < 300; n++) { + appends.push( + session.commit(async (tx) => { + ran++; + await tx.appendEntry(b, { kind: "note" }); + }, context), + ); + } + await eventually(() => ran === 255); + await flush(); + expect(ran).toBe(255); + expect(session.lineSizes.outstanding).toBe(256); + held.release(); + await Promise.all([first, ...appends]); + expect(ran).toBe(300); + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + await session.close(context); + }); +}); + +describe("group commit: randomized interleavings", () => { + /** A small deterministic generator, so a failing seed reproduces. */ + function random(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + return state / 2_147_483_648; + }; + } + + for (const seed of [1, 7, 42, 1_234, 99_991]) { + it(`stores, publishes, and reads only what an order of the commits implies (seed ${seed})`, async () => { + const next = random(seed); + const { storage, session, ids } = await open(4); + const log = ids[0]!; + storage.latency = () => Math.floor(next() * 3); + const refuseAt = seed % 2 === 0 ? Math.floor(next() * 60) : -1; + storage.refuse = (writes) => + writes.some( + (write) => write.type === "entry" && (write.value.data as { index?: number })?.index === refuseAt, + ); + // Per commit, in line order: the notes it increments and the values it read. + const plans: ConversationId[][] = []; + const reads: (number[] | undefined)[] = []; + const outcomes: Promise[] = []; + for (let index = 0; index < 60; index++) { + const touched = ids.filter(() => next() < 0.4); + plans.push(touched); + const outcome = session.commit(async (tx) => { + const values: number[] = []; + for (const id of touched) values.push((await tx.doc(Note, id)).n++); + reads[index] = values; + await tx.appendEntry(log, { kind: "log", data: { index } }); + }, context); + outcome.catch(() => {}); + outcomes.push(outcome); + const pause = next(); + if (pause < 0.3) await outcome.catch(() => {}); + else if (pause < 0.5) await flush(); + } + const results = await Promise.allSettled(outcomes); + if (refuseAt >= 0) await session.closed; + else await session.close(context); + // Outcomes match storage: a commit resolved exactly when its log entry is stored. + const reopened = new SessionImpl(storage.reopen()); + const logged = new Set(); + const entries = await reopened.commit((tx) => tx.scanEntries({ conversationId: log }, 1_000), context); + for (const entry of entries.items) logged.add((entry.data as { index: number }).index); + const stored = results.map((result) => result.status === "fulfilled"); + for (const [index, ok] of stored.entries()) expect(logged.has(index), `commit ${index}`).toBe(ok); + // Each callback read every earlier stored increment of what it touched, and nothing else. + for (const [index, values] of reads.entries()) { + if (values === undefined) continue; + for (const [position, id] of plans[index]!.entries()) { + const expected = plans + .slice(0, index) + .filter((touched, earlier) => stored[earlier] && touched.includes(id)); + expect(values[position], `commit ${index} read of ${id}`).toBe(expected.length); + } + } + // Storage equals the stored commits replayed in order. + for (const id of ids) { + const increments = plans.filter((touched, index) => stored[index] && touched.includes(id)).length; + expect((await reopened.snapshot(Note, id, context))?.n ?? 0).toBe(increments); + } + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + await reopened.close(context); + }); + } +}); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts new file mode 100644 index 000000000..cdc847b78 --- /dev/null +++ b/packages/durable/test/session-observers.test.ts @@ -0,0 +1,541 @@ +// Tests for `Session.observeDoc()`, the synchronous observer of one document address that watches are built on. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchCall, WatchEnd } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type CommitObserver, + type ConversationId, + defineDoc, + MemoryStorage, + type Observation, + type Observer, + type ReportInfo, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, createConversation, flush } from "./session-support.ts"; + +type Counter = { count: number }; + +const SessionCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.session-counter", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), +}); + +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const ConversationCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.conversation-counter", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ count: 0 }), +}); + +/** A Session that keeps its reports, which a plain Session drops. */ +class ReportingSession extends SessionImpl { + readonly reports: { readonly error: unknown; readonly info: ReportInfo }[] = []; + + override report(error: unknown, info: ReportInfo = { kind: "listener_error" }): void { + this.reports.push({ error, info }); + } +} + +/** An observer that records each call, and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array< + { readonly advance: WatchCall>; readonly adopting: boolean } | { readonly end: WatchEnd } + > = []; + const observer: Observer> = { + advance: (change) => { + calls.push({ advance: change, adopting: storage.adopting }); + }, + end: (end) => { + calls.push({ end }); + }, + }; + return { calls, observer }; +} + +function open() { + const storage = new ControlledStorage(); + const session = new ReportingSession(storage); + return { storage, session }; +} + +const add = (session: SessionImpl, by = 1) => + session.commit(async (tx) => { + (await tx.doc(SessionCounter)).count += by; + }, context); + +describe("observeDoc", () => { + it("returns undefined for an absent address and attaches nothing", async () => { + const { storage, session } = open(); + const { calls, observer } = recorder(storage); + expect(await session.observeDoc(SessionCounter, observer, context)).toBeUndefined(); + await add(session); + expect(calls).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("starts at the committed value and advances at the publication of each stored commit with its ops", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(observation.value).toEqual({ count: 1 }); + const committing = add(session, 2); + await committing; + expect(calls).toEqual([ + { + advance: { kind: "value", value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, + adopting: false, + }, + ]); + observation.detach(); + await session.close(context); + }); + + it("advances when memory adopts a held commit, before anything is stored", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + const stored = storage.commits.length; + await session.commit( + async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 5; + }, + context, + { persistence: { kind: "deferred", withinMs: Number.POSITIVE_INFINITY } }, + ); + expect(storage.commits).toHaveLength(stored); + expect(calls).toEqual([ + { + advance: { kind: "value", value: { count: 5 }, ops: [["s", ["count"], 5]], retired: false }, + adopting: false, + }, + ]); + observation.detach(); + await session.close(context); + }); + + it("follows a conversation document's address: a retirement with the last value, then the new incarnation", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + // The address is absent now: a new attachment finds nothing, the existing one goes on. + expect(await session.observeDoc(ConversationCounter, conversationId, recorder(storage).observer, context)).toBe( + undefined, + ); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 9; + }, context); + expect(calls).toEqual([ + { advance: { kind: "value", value: observation.value, ops: [], retired: true }, adopting: false }, + { + advance: { kind: "value", value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, + adopting: false, + }, + ]); + expect((calls[0] as { advance: Extract, { kind: "value" }> }).advance.value).toBe( + observation.value, + ); + observation.detach(); + await session.close(context); + }); + + it("gets nothing after detach, which is idempotent and unpins the document", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(session.documentCacheSizes.attached).toBe(1); + observation.detach(); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await add(session); + await session.close(context); + expect(calls).toEqual([]); + }); + + it("ends once when the Session closes, `failed` (S1: an adapter follows the next Session), a watch `ended`", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + const watch = (await session.watchDoc(SessionCounter, context))!; + await session.close(context); + expect(calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("ends each close's observers with an error of that close, not one shared by every Session", async () => { + const errors: unknown[] = []; + for (let i = 0; i < 2; i++) { + const { session } = open(); + await add(session); + await session.observeDoc( + SessionCounter, + { advance: () => {}, end: (end) => errors.push(end.kind === "failed" ? end.error : end) }, + context, + ); + await session.close(context); + } + expect(errors).toEqual([new Error("session closed"), new Error("session closed")]); + expect(errors[0]).not.toBe(errors[1]); + }); + + it("ends with the error that failed the Session", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(add(session)).rejects.toThrow("disk gone"); + await session.closed; + expect(calls).toEqual([{ end: { kind: "failed", error: failure } }]); + }); + + it("reports a throwing observer, detaches it and ends it with the error; the commit and other observers go on", async () => { + const { storage, session } = open(); + await add(session); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + const throwing: Observer> = { + advance: () => { + throw bug; + }, + end: (end) => ends.push(end), + }; + await session.observeDoc(SessionCounter, throwing, context); + const healthy = recorder(storage); + await session.observeDoc(SessionCounter, healthy.observer, context); + await add(session); + await add(session); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(healthy.calls).toHaveLength(2); + expect(await session.snapshot(SessionCounter, context)).toEqual({ count: 3 }); + await session.close(context); + // Detached at its throw: the close does not end it again. + expect(ends).toHaveLength(1); + expect(session.failure).toBeUndefined(); + }); + + it("attaches atomically on the line: a commit queued before the attachment is in its value", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const committing = add(session, 10); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + await committing; + expect(observation.value).toEqual({ count: 11 }); + expect(calls).toEqual([]); + observation.detach(); + await session.close(context); + }); + + it("rejects an attachment whose context is cancelled, and attaches nothing", async () => { + const { storage, session } = open(); + await add(session); + const controller = new AbortController(); + controller.abort(new Error("gone")); + const cancelled = withAbortSignal(controller.signal, context); + await expect(session.observeDoc(SessionCounter, recorder(storage).observer, cancelled)).rejects.toThrow("gone"); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("keeps no detached observer alive", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + await add(session); + /** Attach an observer only the Session references; `detach` says whether to detach it before returning. */ + const attach = async (detach: boolean) => { + const observer: Observer> = { advance: () => {}, end: () => {} }; + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + if (detach) observation.detach(); + return { observation, observer: new WeakRef(observer) }; + }; + // Control: an attached observer stays, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.observer.deref()).toBeDefined(); + attached.observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.observer.deref()).toBeUndefined(); + await flush(); + await session.close(context); + }); + + it("rejects, never throws, on a closed Session or a missing scope ID", async () => { + const { storage, session } = open(); + const missing = session.observeDoc( + ConversationCounter, + undefined as unknown as ConversationId, + recorder(storage).observer, + context, + ); + await expect(missing).rejects.toThrow(TypeError); + await session.close(context); + const closed = session.observeDoc(SessionCounter, recorder(storage).observer, context); + await expect(closed).rejects.toThrow("Session is closed"); + }); + it("keeps no value of a document retired for good, while still attached", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + /** Attach an observer that keeps nothing, change the document, and retire it: only a WeakRef to its last value. */ + const retire = async () => { + let last: WeakRef | undefined; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + if (change.kind === "value" && !change.retired) last = new WeakRef(change.value); + }, + end: () => {}, + }, + context, + ))!; + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 7; + }, context); + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + await session.unloadDocuments(); + return { observation, last: last! }; + }; + const { observation, last } = await retire(); + await collect(); + expect(last.deref()).toBeUndefined(); + observation.detach(); + await session.close(context); + }); +}); + +describe("observeDoc throwing asynchronously", () => { + it("reports an observer whose call returns a rejected promise, then detaches it and ends it `listener_error`", async () => { + const { session } = open(); + await add(session); + const bug = new Error("async observer bug"); + const ends: WatchEnd[] = []; + let calls = 0; + const observer = { + advance: async () => { + calls++; + throw bug; + }, + end: (end: WatchEnd) => ends.push(end), + }; + await session.observeDoc(SessionCounter, observer, context); + await add(session); + await flush(); + await add(session); + expect(calls).toBe(1); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + await session.close(context); + expect(ends).toHaveLength(1); + }); +}); + +describe("observeCommits", () => { + /** A commit observer recording each seq and its end. */ + function commits() { + const calls: Array<{ readonly seq: number } | { readonly end: WatchEnd }> = []; + const observer: CommitObserver = { + advance: (publication) => calls.push({ seq: publication.seq }), + end: (end) => calls.push({ end }), + }; + return { calls, observer }; + } + + it("gets every stored commit, then `failed` once the Session closed (S1), after its last publication", async () => { + const { session } = open(); + const { calls, observer } = commits(); + session.observeCommits(observer); + await add(session); + await add(session); + await session.close(context); + expect(calls).toEqual([ + { seq: expect.any(Number) }, + { seq: expect.any(Number) }, + { end: { kind: "failed", error: new Error("session closed") } }, + ]); + }); + + it("ends `failed` with the error that failed the Session", async () => { + const { storage, session } = open(); + const { calls, observer } = commits(); + session.observeCommits(observer); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(add(session)).rejects.toThrow("disk gone"); + await session.closed; + expect(calls).toEqual([{ end: { kind: "failed", error: failure } }]); + }); + + it("gets nothing after detach, which is idempotent, not even its end", async () => { + const { session } = open(); + const { calls, observer } = commits(); + const observation = session.observeCommits(observer); + await add(session); + await flush(); + observation.detach(); + observation.detach(); + await add(session); + await session.close(context); + expect(calls).toHaveLength(1); + }); + + it("reports a throwing observer, detaches it and ends it `listener_error`; the others go on", async () => { + const { session } = open(); + const bug = new Error("commit observer bug"); + const ends: WatchEnd[] = []; + let thrown = 0; + session.observeCommits({ + advance: () => { + thrown++; + throw bug; + }, + end: (end) => ends.push(end), + }); + const healthy = commits(); + session.observeCommits(healthy.observer); + await add(session); + await add(session); + await flush(); + expect(thrown).toBe(1); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(healthy.calls).toHaveLength(2); + await session.close(context); + expect(ends).toHaveLength(1); + }); +}); + +describe("observeDoc detached during a commit", () => { + /** A Session with a conversation counter at 1, and the commit that retires it and creates it again at 2. */ + async function retiring() { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 1; + }, context); + const recreate = () => + session.commit(async (tx) => { + await tx.retireDoc(ConversationCounter, conversationId); + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + return { storage, session, conversationId, recreate }; + } + + it("gets nothing more of the commit once it throws on the retirement, and pins nothing for the recreation", async () => { + const { session, conversationId, recreate } = await retiring(); + const bug = new Error("observer bug"); + const changes: WatchCall>[] = []; + const ends: unknown[] = []; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + throw bug; + }, + end: (end) => ends.push(end), + }, + context, + ))!; + await recreate(); + expect(changes).toEqual([{ kind: "value", value: observation.value, ops: [], retired: true }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(session.documentCacheSizes.attached).toBe(0); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("gets nothing more of the commit once it detaches in the retirement call, and pins nothing", async () => { + const { session, conversationId, recreate } = await retiring(); + const changes: WatchCall>[] = []; + const ends: unknown[] = []; + let observation: Observation> | undefined; + observation = await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + observation?.detach(); + }, + end: (end) => ends.push(end), + }, + context, + ); + await recreate(); + expect(changes).toEqual([{ kind: "value", value: observation!.value, ops: [], retired: true }]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + expect(ends).toEqual([]); + }); + + it("gets nothing of the commit when another observer's call detached it first, and pins nothing", async () => { + const { storage, session, conversationId, recreate } = await retiring(); + let second: Observation> | undefined; + const first = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: () => second?.detach(), + end: () => {}, + }, + context, + ))!; + const detached = recorder(storage); + second = await session.observeDoc(ConversationCounter, conversationId, detached.observer, context); + await recreate(); + expect(detached.calls).toEqual([]); + // Only the first observer pins the new incarnation; its detach leaves nothing pinned. + first.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); +}); diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index b5c4df8b6..8fb1a015c 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -1,7 +1,9 @@ import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { + type CommitChange, type CommitPublication, + type CommitResult, type ConversationId, type DocumentAddress, type DocumentCommitChange, @@ -11,12 +13,27 @@ import { type Id, MemoryStorage, type Seq, + type Session, + type Storage, type StorageWrite, } from "@earendil-works/pi-durable"; -import { SessionImpl } from "../src/session/session.ts"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; export const context: Context = BACKGROUND_CONTEXT; +/** + * A watch listener for value calls: Durable's watches never send a `failed` call (their failures end them), so one here + * throws, which ends the watch `listener_error`. + */ +export function onValue( + listener: (call: Extract, context: Context) => Promise, +): (call: C, context: Context) => Promise { + return (call, deliveryContext) => { + if (call.kind !== "value") throw new Error(`Unexpected ${call.kind} watch call`); + return listener(call as Extract, deliveryContext); + }; +} + type Deferred = { readonly promise: Promise; readonly resolve: () => void }; function deferred(): Deferred { @@ -33,6 +50,24 @@ export type Gate = { release(): void; }; +/** + * Commits `batches` through `commit` up to the first one `refuses` matches, which it refuses with `error`: what a + * storage does whose own check refuses that batch. + */ +export async function refusingAt( + commit: Storage["commit"], + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + refuses: (writes: readonly StorageWrite[]) => boolean, + error: unknown, +): Promise { + const at = batches.findIndex(refuses); + if (at < 0) return commit(batches, receive, commitContext); + const result = await commit(batches.slice(0, at), receive, commitContext); + return result.stored < at ? result : { stored: at, refused: error }; +} + /** Memory storage with observable commits, held calls, and injected commit failures. */ export class ControlledStorage extends MemoryStorage { /** Exact borrowed batches admitted by Session. */ @@ -41,6 +76,10 @@ export class ControlledStorage extends MemoryStorage { readonly commits: (readonly StorageWrite[])[] = []; mintCount = 0; documentReadCount = 0; + /** Set while a receipt runs, the storage's synchronous step: code that sees it set runs inside `commit()`'s `receive`. */ + adopting = false; + /** Runs once, in the next commit's synchronous step right after its first receipt returned. */ + afterAdopt: (() => void) | undefined; #commitGate: { gate: Deferred; entered: Deferred } | undefined; #findGate: { gate: Deferred; entered: Deferred } | undefined; #commitFailure: Error | undefined; @@ -78,24 +117,41 @@ export class ControlledStorage extends MemoryStorage { held.gate.resolve(); } + /** + * Records and holds the call, then fails its first batch if a failure is set (a refusal: nothing stored), or commits + * it in memory, `adopting` set while each receipt runs and `afterAdopt` run after the first one. + */ override async commit( - writes: readonly StorageWrite[], - adopt: (seq: Seq) => void, + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, commitContext: Context, - ): Promise { - this.admittedCommits.push(writes); - this.commits.push(structuredClone(writes)); + ): Promise { const held = this.#commitGate; if (held !== undefined) { held.entered.resolve(); await held.gate.promise; } const failure = this.#commitFailure; + for (const writes of failure === undefined ? batches : batches.slice(0, 1)) { + this.admittedCommits.push(writes); + this.commits.push(structuredClone(writes)); + } if (failure !== undefined) { this.#commitFailure = undefined; - throw failure; + return { stored: 0, refused: failure }; } - return super.commit(writes, adopt, commitContext); + const step = (index: number, seq: Seq): void => { + this.adopting = true; + try { + receive(index, seq); + } finally { + this.adopting = false; + } + const after = this.afterAdopt; + this.afterAdopt = undefined; + after?.(); + }; + return super.commit(batches, step, commitContext); } override mintId>(): Promise { @@ -122,6 +178,44 @@ export class ControlledStorage extends MemoryStorage { } } +/** + * Controlled storage whose commits resolve a macrotask after they adopted the batch, as a storage with work after + * durability does (JSONL reclaims sidecars): code woken inside `adopt` runs before the commit resolves. `gap` runs in + * that work, a macrotask after adoption, and `rejectAfterAdopt` makes a commit it matches reject there, durable and + * adopted, as a failed reclaim does; each fires once. + */ +export class LateResolvingStorage extends ControlledStorage { + /** Set from a commit's start until it resolves. */ + committing = false; + gap: ((writes: readonly StorageWrite[]) => boolean) | undefined; + rejectAfterAdopt: ((writes: readonly StorageWrite[]) => boolean) | undefined; + + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + this.committing = true; + try { + const result = await super.commit(batches, receive, commitContext); + const stored = batches.slice(0, result.stored); + await flush(); + if (stored.some((writes) => this.gap?.(writes) === true)) { + this.gap = undefined; + // What the gap started runs before the commit resolves. + await flush(); + } + if (stored.some((writes) => this.rejectAfterAdopt?.(writes) === true)) { + this.rejectAfterAdopt = undefined; + throw new Error("reclaim failed"); + } + return result; + } finally { + this.committing = false; + } + } +} + /** Session kernel plus its controlled storage and every committed publication. */ export function openTestSession(): { readonly storage: ControlledStorage; @@ -132,20 +226,45 @@ export function openTestSession(): { // These tests pin the cache's identities and loads, so documents stay as long as the default keeps them. const session = new SessionImpl(storage, Date.now, () => 600_000); const publications: CommitPublication[] = []; - session.subscribeCommits((publication) => { - publications.push(publication); + session.observeCommits({ + advance: (publication) => { + publications.push(publication); + }, + end: () => {}, }); return { storage, session, publications }; } -export function documentChanges( - publication: CommitPublication, -): readonly Extract[] { +export function documentChanges(publication: { + readonly changes: readonly CommitChange[]; +}): readonly Extract[] { return publication.changes.filter( (change): change is Extract => change.type === "document", ); } +/** + * Follow every commit as memory adopts it, deferred ones before they are stored, as views and document watches do; + * `observeCommits()` sees only what was stored. + */ +export function followCommits( + session: Session, + listener: (publication: ObservedCommit, context: Context) => void, +): () => void { + // `followCommits()` is internal, so the public type lacks it; every Session and Harness is a `SessionImpl`. + return (session as unknown as SessionImpl).followCommits(listener); +} + +/** The document changes of `publication` that carry a value: creations and updates, no retirements. */ +export function documentValues(publication: { + readonly changes: readonly CommitChange[]; +}): readonly Extract[] { + return publication.changes.filter( + (change): change is Extract => + change.type === "document" && change.kind === "change", + ); +} + export function documentCopyChanges( publication: CommitPublication, ): readonly Extract[] { diff --git a/packages/durable/test/session-tables.test.ts b/packages/durable/test/session-tables.test.ts index fe017504f..3713b5cb9 100644 --- a/packages/durable/test/session-tables.test.ts +++ b/packages/durable/test/session-tables.test.ts @@ -432,8 +432,7 @@ describe("Session transaction tables", () => { const publication = publications.at(-1)!; const documents = documentChanges(publication); expect(publication.changes.filter((change) => change.type === "task")).toHaveLength(1); - expect(documents.map((document) => document.value)).toEqual([null, null, null]); - for (const document of documents) expect(document.ops).toEqual([]); + expect(documents.map((document) => document.kind)).toEqual(["retire", "retire", "retire"]); for (const document of documents) expect(document.conversationId).toBe(conversationId); expect(await session.snapshot(ProgressDoc, taskId, context)).toBeUndefined(); expect(await session.snapshot(StepDoc, taskId, "committed", context)).toBeUndefined(); diff --git a/packages/durable/test/session-watches.test.ts b/packages/durable/test/session-watches.test.ts index 79a7f5b39..256c27b51 100644 --- a/packages/durable/test/session-watches.test.ts +++ b/packages/durable/test/session-watches.test.ts @@ -1,6 +1,6 @@ import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { Context } from "@earendil-works/chord"; +import type { Context, WatchCall, WatchEnd } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, createContextKey, @@ -9,10 +9,10 @@ import { withContextValue, } from "@earendil-works/chord/context"; import { applyImmutable } from "@earendil-works/chord/delta"; -import { type DocumentWatch, defineDoc, MemoryStorage } from "@earendil-works/pi-durable"; +import { type DocumentWatch, defineDoc, defineDocFamily, MemoryStorage } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { SessionImpl } from "../src/session/session.ts"; -import { context, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { context, createConversation, documentValues, flush, onValue, openTestSession } from "./session-support.ts"; type State = { value: number; items: string[]; retained: { label: string } }; @@ -44,6 +44,15 @@ function deferred(): Deferred { return { promise, resolve }; } +/** The end of `closed`, or undefined while the watch is open. */ +function endOf(watch: { readonly closed: Promise }): { readonly end: WatchEnd | undefined } { + const settled: { end: WatchEnd | undefined } = { end: undefined }; + void watch.closed.then((end) => { + settled.end = end; + }); + return settled; +} + async function createState(): Promise> { const harness = openTestSession(); await harness.session.commit((tx) => tx.doc(StateDoc).then(() => undefined), context); @@ -67,21 +76,23 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 1; }, context); await flush(); - const firstPublished = documentChanges(publications.at(-1)!)[0]!; + const firstPublished = documentValues(publications.at(-1)!)[0]!; await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 2; }, context); await flush(); - const secondPublished = documentChanges(publications.at(-1)!)[0]!; + const secondPublished = documentValues(publications.at(-1)!)[0]!; expect(watch.value).toBe(initial); let inline = true; - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { - expect(inline).toBe(false); - expect(watch.value).toBe(value); - deliveries.push({ value, ops }); - }); + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start( + onValue(async ({ value, ops }) => { + expect(inline).toBe(false); + expect(watch.value).toBe(value); + deliveries.push({ value, ops }); + }), + ); inline = false; expect(deliveries).toHaveLength(0); await flush(); @@ -96,7 +107,7 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 3; }, context); await flush(); - const thirdPublished = documentChanges(publications.at(-1)!)[0]!; + const thirdPublished = documentValues(publications.at(-1)!)[0]!; expect(deliveries).toHaveLength(2); expect(deliveries[1]!.value).toBe(thirdPublished.value); expect(deliveries[1]!.ops).toBe(thirdPublished.ops); @@ -112,17 +123,19 @@ describe("Session document watches", () => { const batches: unknown[] = []; let active = 0; let maxActive = 0; - watch.start(async (value, ops) => { - active++; - maxActive = Math.max(maxActive, active); - values.push(value?.value ?? -1); - batches.push(ops); - if (values.length === 1) { - entered.resolve(); - await release.promise; - } - active--; - }); + watch.start( + onValue(async ({ value, ops }) => { + active++; + maxActive = Math.max(maxActive, active); + values.push(value.value); + batches.push(ops); + if (values.length === 1) { + entered.resolve(); + await release.promise; + } + active--; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -146,17 +159,19 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const completed = deferred(); const values: number[] = []; - watch.start(async (value) => { - const current = value?.value ?? -1; - values.push(current); - if (current === 1) { - await session.commit(async (tx) => { - (await tx.doc(StateDoc)).value = 2; - }, context); - } else { - completed.resolve(); - } - }); + watch.start( + onValue(async ({ value }) => { + const current = value.value; + values.push(current); + if (current === 1) { + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 2; + }, context); + } else { + completed.resolve(); + } + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -174,9 +189,11 @@ describe("Session document watches", () => { }, context); } const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); + }), + ); await flush(); expect(deliveries).toEqual([{ value: 101, ops: [["s", ["value"], 101]] }]); await watch.stop(); @@ -197,13 +214,15 @@ describe("Session document watches", () => { for (let index = from; index < to; index++) state.keys[`k${index}`] = index; }, context); } - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value, ops }); - }); + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value, ops }); + }), + ); await flush(); expect(deliveries).toEqual([{ value: watch.value, ops: [["r", watch.value]] }]); - expect(Object.keys(watch.value?.keys ?? {})).toHaveLength(10_001); + expect(Object.keys(watch.value.keys)).toHaveLength(10_001); await watch.stop(); }); @@ -213,13 +232,15 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); - if (deliveries.length === 1) { - entered.resolve(); - await release.promise; - } - }); + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); + if (deliveries.length === 1) { + entered.resolve(); + await release.promise; + } + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -237,7 +258,7 @@ describe("Session document watches", () => { await watch.stop(); }); - it("folds retirement over pending commits and then closes", async () => { + it("folds a retirement into the pending commits' call, which reports it with the last value, and goes on", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; for (let value = 1; value <= 100; value++) { @@ -245,13 +266,27 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = value; }, context); } + const last = (await session.snapshot(StateDoc, context))!; await session.commit((tx) => tx.retireDoc(StateDoc), context); - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value, ops }); + const changes: WatchCall>[] = []; + watch.start(async (change) => { + changes.push(change); }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(deliveries).toEqual([{ value: null, ops: [["r", null]] }]); + await flush(); + expect(changes).toEqual([{ kind: "value", value: last, ops: [["s", ["value"], 100]], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(last); + expect(watch.value).toBe(last); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 7; + }, context); + await flush(); + expect(changes[1]).toEqual({ + kind: "value", + value: { ...last, value: 7 }, + ops: [["r", { ...last, value: 7 }]], + retired: false, + }); + await watch.stop(); }); it("delivers replayable structural no-op commits instead of suppressing them", async () => { @@ -267,10 +302,12 @@ describe("Session document watches", () => { expect(newest).not.toBe(initial); expect(newest).toEqual(initial); const batches: unknown[] = []; - watch.start(async (value, ops) => { - expect(value).toBe(newest); - batches.push(ops); - }); + watch.start( + onValue(async ({ value, ops }) => { + expect(value).toBe(newest); + batches.push(ops); + }), + ); await flush(); expect(batches).toHaveLength(1); expect((batches[0] as readonly unknown[]).length).toBeGreaterThan(0); @@ -290,7 +327,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -303,7 +340,7 @@ describe("Session document watches", () => { expect(deliveryContext!.abortSignal).toBeUndefined(); parentController.abort("caller finished"); const stopped = watch.stop(); - expect(await stopped).toEqual({ reason: "stopped" }); + expect(await stopped).toEqual({ kind: "stopped" }); expect(deliveryContext!.abortSignal).toBeUndefined(); release.resolve(); }); @@ -312,44 +349,67 @@ describe("Session document watches", () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; const initial = watch.value; - let delivered: Readonly | null | undefined; - watch.start(async (value) => { - delivered = value; - }); + let delivered: Readonly | undefined; + watch.start( + onValue(async ({ value }) => { + delivered = value; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); await flush(); expect(delivered).toBe(watch.value); expect(delivered).not.toBe(initial); - expect(delivered?.retained).toBe(initial?.retained); - expect(initial?.value).toBe(0); + expect(delivered?.retained).toBe(initial.retained); + expect(initial.value).toBe(0); await watch.stop(); }); - it("delivers retirement and does not follow recreation", async () => { + it("reports a change and a retirement in one commit as the retirement, with the value it was given before", async () => { + const { session, storage } = await createState(); + const watch = (await session.watchDoc(StateDoc, context))!; + const before = watch.value; + const changes: WatchCall>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 5; + await tx.retireDoc(StateDoc); + }, context); + await flush(); + // Storage got the change, then the retirement; the commit publishes the retirement only. + expect(storage.commits.at(-1)!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); + expect(changes).toEqual([{ kind: "value", value: before, ops: [], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(before); + await watch.stop(); + }); + + it("follows its address across a retirement and a recreation in one commit, in one call", async () => { const { session } = await createState(); - const oldWatch = (await session.watchDoc(StateDoc, context))!; - const values: Array | null> = []; - oldWatch.start(async (value) => { - values.push(value); + const watch = (await session.watchDoc(StateDoc, context))!; + const settled = endOf(watch); + const changes: WatchCall>[] = []; + watch.start(async (change) => { + changes.push(change); }); await session.commit(async (tx) => { await tx.retireDoc(StateDoc); (await tx.doc(StateDoc)).value = 10; }, context); - expect(await oldWatch.closed).toEqual({ reason: "retired" }); - expect(values).toEqual([null]); - expect(oldWatch.value).toBeNull(); - - const replacement = (await session.watchDoc(StateDoc, context))!; - expect(replacement.value?.value).toBe(10); + await flush(); + const recreated = (await session.snapshot(StateDoc, context))!; + expect(changes).toHaveLength(1); + expect(changes[0]).toEqual({ kind: "value", value: recreated, ops: [["r", recreated]], retired: true }); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 11; }, context); await flush(); - expect(oldWatch.value).toBeNull(); - await replacement.stop(); + expect(changes[1]).toEqual({ kind: "value", value: watch.value, ops: [["s", ["value"], 11]], retired: false }); + expect(watch.value.value).toBe(11); + expect(settled.end).toBeUndefined(); + await watch.stop(); }); it("Session close discards retirement buffered before start", async () => { @@ -358,7 +418,7 @@ describe("Session document watches", () => { const baseline = watch.value; await session.commit((tx) => tx.retireDoc(StateDoc), context); await session.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value).toBe(baseline); expect(() => watch.start(async () => undefined)).toThrow("stopped"); }); @@ -368,19 +428,21 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const entered = deferred(); const release = deferred(); - const values: Array = []; - watch.start(async (value) => { - values.push(value?.value ?? null); - entered.resolve(); - await release.promise; - }); + const values: number[] = []; + watch.start( + onValue(async ({ value }) => { + values.push(value.value); + entered.resolve(); + await release.promise; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); await entered.promise; await session.commit((tx) => tx.retireDoc(StateDoc), context); await session.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); release.resolve(); await flush(); expect(values).toEqual([1]); @@ -394,7 +456,7 @@ describe("Session document watches", () => { const first = started.stop(); const second = started.stop(); expect(second).toBe(first); - expect(await first).toEqual({ reason: "stopped" }); + expect(await first).toEqual({ kind: "stopped" }); const stopped = (await session.watchDoc(StateDoc, context))!; await stopped.stop(); @@ -421,7 +483,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -431,7 +493,7 @@ describe("Session document watches", () => { }, context); await entered.promise; child.cancel(); - expect(await watch.closed).toEqual({ reason: "cancelled" }); + expect(await watch.closed).toEqual({ kind: "aborted" }); expect(deliveryContext!.abortSignal).toBeUndefined(); release.resolve(); }); @@ -442,7 +504,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -453,7 +515,7 @@ describe("Session document watches", () => { await entered.promise; await session.close(context); expect(deliveryContext!.abortSignal).toBeUndefined(); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); release.resolve(); }); @@ -472,8 +534,8 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 1; }, context); const end = await failed.closed; - expect(end.reason).toBe("listener_error"); - if (end.reason === "listener_error") expect(end.error.message).toBe("listener failed"); + expect(end.kind).toBe("listener_error"); + if (end.kind === "listener_error") expect(end.error.message).toBe("listener failed"); await flush(); expect(healthyCalls).toBe(1); await healthy.stop(); @@ -503,9 +565,11 @@ describe("Session document watches", () => { expect(watch.value).toEqual({ value: 3, migrated: true }); expect(storage.commits).toHaveLength(commits); const values: number[] = []; - watch.start(async (value) => { - values.push(value?.value ?? -1); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value.value); + }), + ); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); expect(values).toEqual([]); @@ -520,11 +584,13 @@ describe("Session document watches", () => { it("can replay every delivered exact operation batch from the acquisition revision", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; - let replica: Readonly | null = watch.value; - watch.start(async (value, ops) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - }); + let replica: Readonly = watch.value; + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + }), + ); await session.commit(async (tx) => { const state = await tx.doc(StateDoc); state.value = 3; @@ -550,10 +616,196 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 7; }, context); await flush(); - expect(watch.value?.value).toBe(7); + expect(watch.value.value).toBe(7); await watch.stop(); }); + it("ignores the same kind in another conversation and under another family key", async () => { + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.scoped-notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const Members = defineDocFamily<{ text: string }, null>({ + checkpointWhen: () => false, + kind: "watch.members", + version: 1, + family: true, + scope: "session", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const watched = await createConversation(session); + const other = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(Notes, watched)).text = "a"; + (await tx.doc(Members, "watched", null)).text = "a"; + }, context); + const notes = (await session.watchDoc(Notes, watched, context))!; + const member = (await session.watchDoc(Members, "watched", context))!; + const changes: WatchCall>[] = []; + notes.start(async (change) => { + changes.push(change); + }); + member.start(async (change) => { + changes.push(change); + }); + // The same kinds at other addresses: created, changed, retired, and created again. + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "other"; + (await tx.doc(Members, "other", null)).text = "other"; + }, context); + await session.commit(async (tx) => { + await tx.retireDoc(Notes, other); + await tx.retireDoc(Members, "other"); + }, context); + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "again"; + (await tx.doc(Members, "other", null)).text = "again"; + }, context); + await flush(); + expect(changes).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(2); + await notes.stop(); + await member.stop(); + await session.close(context); + }); + + it("takes the version of a recreation written by a newer definition, then its ops", async () => { + const Old = defineDoc<{ value: number }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 1, + scope: "session", + initial: () => ({ value: 1 }), + }); + const Current = defineDoc<{ value: number; migrated: boolean }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 2, + scope: "session", + initial: () => ({ value: 0, migrated: false }), + migrate: (value) => ({ value: value.value as number, migrated: true }), + }); + const { session } = openTestSession(); + await session.commit((tx) => tx.doc(Old).then(() => undefined), context); + const watch = (await session.watchDoc(Old, context))!; + const changes: WatchCall[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + await tx.retireDoc(Current); + (await tx.doc(Current)).value = 5; + }, context); + await flush(); + await session.commit(async (tx) => { + (await tx.doc(Current)).value = 6; + }, context); + await flush(); + expect(changes).toEqual([ + { + kind: "value", + value: { value: 5, migrated: false }, + ops: [["r", { value: 5, migrated: false }]], + retired: true, + }, + { kind: "value", value: { value: 6, migrated: false }, ops: [["s", ["value"], 6]], retired: false }, + ]); + await watch.stop(); + await session.close(context); + }); + + it("keeps no earlier incarnation alive after a retirement and a recreation, while it goes on", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + /** Watch the document, retire and recreate it, and deliver the new value; `drop` then unloads the cache. */ + const follow = async (session: SessionImpl, drop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).retained = { label: "first incarnation" }; + }, context); + const watch = (await session.watchDoc(StateDoc, context))!; + const first = new WeakRef(watch.value.retained); + watch.start(async () => {}); + await session.commit(async (tx) => { + await tx.retireDoc(StateDoc); + (await tx.doc(StateDoc)).value = 2; + }, context); + await flush(); + if (drop) await session.unloadDocuments(); + return { watch, first }; + }; + // Retention 0 evicts the retired incarnation at once; the default keeps the cache until it is unloaded. + for (const [session, drop] of [ + [new SessionImpl(new MemoryStorage(), Date.now, () => 0), false], + [new SessionImpl(new MemoryStorage(), Date.now, () => 600_000), true], + ] as const) { + const { watch, first } = await follow(session, drop); + await collect(); + expect(watch.value.value).toBe(2); + expect(watch.value.retained).toEqual({ label: "stable" }); + expect(first.deref()).toBeUndefined(); + await watch.stop(); + await session.close(context); + } + }); + + it("keeps a watch of a retired conversation document until it is stopped, then nothing of it", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const conversationId = await createConversation(session); + /** A started watch of a document that is then retired; `stop` says whether to stop it before returning. */ + const acquire = async (stop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(Notes, conversationId)).text = "kept"; + }, context); + const watch = (await session.watchDoc(Notes, conversationId, context))!; + const listener = async (): Promise => {}; + watch.start(listener); + await session.commit((tx) => tx.retireDoc(Notes, conversationId), context); + await flush(); + if (stop) await watch.stop(); + return { watch: new WeakRef(watch), listener: new WeakRef(listener) }; + }; + // A watch follows its address, so a forgotten one stays until it is stopped or the Session ends. + const forgotten = await acquire(false); + await collect(); + expect(forgotten.watch.deref()).toBeDefined(); + await forgotten.watch.deref()!.stop(); + + const stopped = await acquire(true); + await collect(); + expect(stopped.watch.deref()).toBeUndefined(); + expect(stopped.listener.deref()).toBeUndefined(); + await session.close(context); + }); + it("retains neither its Session nor its listener once stopped, while still referenced", async () => { setFlagsFromString("--expose-gc"); const gc = runInNewContext("gc") as () => void; @@ -595,7 +847,7 @@ describe("Session document watches", () => { expect(stopped.session.deref()).toBeUndefined(); expect(stopped.listener.deref()).toBeUndefined(); // What it delivered last stays readable. - expect(stopped.watch.value?.value).toBe(1); - expect(await stopped.watch.closed).toEqual({ reason: "stopped" }); + expect(stopped.watch.value.value).toBe(1); + expect(await stopped.watch.closed).toEqual({ kind: "stopped" }); }); }); diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index 489f48204..b1896bf66 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -3,10 +3,13 @@ * the names the spec leaves to the application declared below. Keep the two in sync; `test/examples/` runs the same * patterns end to end. */ + import type { Context, Draft } from "@earendil-works/chord"; -import { type AssistantMessage, type Models, type ToolCall, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, Models, ToolCall } from "@earendil-works/pi-ai"; import { + type Conversation, type ConversationId, + ConversationTranscript, configure, createRegistry, defineDoc, @@ -27,13 +30,16 @@ import { type TaskId, type ToolExecutionApi, ToolTask, + type TranscriptView, type Tx, type UserInput, wrapTool, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { expect, expectTypeOf, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { CodingTools, createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; +import { onValue } from "./session-support.ts"; // ─── Names the spec leaves to the application ──────────────────────────────── @@ -75,6 +81,8 @@ declare const manager: SettingsManager; declare const Anchor: ReturnType>; declare const Reporter: ReturnType>; declare const subagentTool: ReturnType; +declare const conversation: Conversation; +declare function render(view: TranscriptView): void; /** Never called: the declared names above have no values. */ function examples() { @@ -163,7 +171,7 @@ function examples() { defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", execute: async (args, api, context) => { const { task } = args; @@ -351,6 +359,21 @@ function examples() { }); }, + // Section 9.3: a client's transcript over the windowed view. + transcript: async () => { + const transcript = new ConversationTranscript((query, limit, ctx) => + conversation.entries(query, limit, undefined, ctx), + ); + const watch = await conversation.watch(context); + render(transcript.merge(watch.value)); + watch.start(onValue(async ({ value }) => render(transcript.merge(value)))); + // A marker scrolled into view: + render(await transcript.loadEarlier(context)); + // Reconnect after a drop: + const resumed = await conversation.watch(context, transcript.resume); + return resumed; + }, + // Section 4: table reads before the first table write; documents stay usable. tableRules: async () => { await session.commit(async (tx) => { diff --git a/packages/durable/test/sqlite-cloudflare.test.ts b/packages/durable/test/sqlite-cloudflare.test.ts index 58b78a4e0..14608a3b1 100644 --- a/packages/durable/test/sqlite-cloudflare.test.ts +++ b/packages/durable/test/sqlite-cloudflare.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { durableObjectStorage } from "../../chord/test/sqlite/helpers.ts"; import { idFromNumber } from "../src/ids.ts"; import { DurableObjectSqliteDatabase, openDurableObjectSqliteStorage } from "../src/storage/sqlite/cloudflare.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; registerStorageConformance({ describe, expect, it }, "SqliteStorage on a prefixed Durable Object", async (use) => @@ -15,7 +16,7 @@ describe("Durable Object SQLite adapter", () => { it("keeps schema names unprefixed by default", async () => { const sqlite = new DatabaseSync(":memory:"); await openDurableObjectSqliteStorage(durableObjectStorage(sqlite)); - expect(sqlite.prepare("SELECT version FROM durable_schema").get()).toEqual({ version: 3 }); + expect(sqlite.prepare("SELECT version FROM durable_schema").get()).toEqual({ version: 2 }); expect(sqlite.prepare("SELECT name FROM sqlite_master WHERE name = 'pi_durable_schema'").get()).toBeUndefined(); }); @@ -48,20 +49,20 @@ describe("Durable Object SQLite adapter", () => { ]); const entryId = idFromNumber(2); await Promise.all([ - first.commit( + commitWrites( + first, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "first" } }, ], - () => {}, BACKGROUND_CONTEXT, ), - second.commit( + commitWrites( + second, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "second" } }, ], - () => {}, BACKGROUND_CONTEXT, ), ]); diff --git a/packages/durable/test/sqlite-facade.test.ts b/packages/durable/test/sqlite-facade.test.ts index ab8b9da8e..d6c6995c0 100644 --- a/packages/durable/test/sqlite-facade.test.ts +++ b/packages/durable/test/sqlite-facade.test.ts @@ -11,11 +11,9 @@ import { idFromNumber } from "../src/ids.ts"; import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "../src/storage/sqlite/index.ts"; import { SqliteStorage } from "../src/storage/sqlite/index.ts"; import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type DocumentId, type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - type SettlementMode = "immediate" | "delay" | "reject"; class ControlledSettlementDatabase implements SqliteDatabase { @@ -35,6 +33,10 @@ class ControlledSettlementDatabase implements SqliteDatabase { return this.delegate.run(sql, ...params); } + runCounted(sql: string, ...params: SqliteValue[]): Promise { + return this.delegate.runCounted(sql, ...params); + } + get(sql: string, ...params: SqliteValue[]): Promise { return this.delegate.get(sql, ...params); } @@ -97,29 +99,29 @@ describe("portable SQLite facade settlement", () => { it("prepares each storage statement once per connection and reuses it across transactions", async () => { const connection = new PrepareCountingDatabaseSync(":memory:"); const storage = await SqliteStorage.open(new NodeSqliteDatabase(connection)); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); - await storage.commit( + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); + await commitWrites( + storage, Array.from({ length: 100 }, (_, index) => ({ type: "entry" as const, value: { id: idFromNumber(index + 2), conversationId: ROOT_CONVERSATION_ID, kind: "cached" }, })), - adopt, BACKGROUND_CONTEXT, ); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(102), conversationId: ROOT_CONVERSATION_ID, kind: "cached-again" }, }, ], - adopt, BACKGROUND_CONTEXT, ); expect((await storage.entry(idFromNumber(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached"); expect((await storage.entry(idFromNumber(102), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached-again"); await expect( - storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT), + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT), ).rejects.toThrow("ID 1 already belongs to conversation"); expect((await storage.entry(idFromNumber(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached"); expect(connection.repeatedPrepares()).toEqual([]); @@ -129,9 +131,9 @@ describe("portable SQLite facade settlement", () => { it("reads storage commits in call order", async () => { const database = await openNodeSqliteDatabase(":memory:"); const storage = await SqliteStorage.open(database); - const commit = storage.commit( + const commit = commitWrites( + storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], - adopt, BACKGROUND_CONTEXT, ); const read = storage.conversation(ROOT_CONVERSATION_ID, BACKGROUND_CONTEXT); @@ -143,12 +145,12 @@ describe("portable SQLite facade settlement", () => { it("lets admitted multi-query reads finish before storage closes", async () => { const storage = await SqliteStorage.open(await openNodeSqliteDatabase(":memory:")); const entryId = idFromNumber(2); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "probe" } }, ], - adopt, BACKGROUND_CONTEXT, ); const scan = storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, BACKGROUND_CONTEXT); @@ -180,14 +182,14 @@ describe("portable SQLite facade settlement", () => { const storage = await opening; database.controlNextSettlement("delay"); - const committing = storage.commit( + const committing = commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(100), conversationId: ROOT_CONVERSATION_ID, kind: "settled" }, }, ], - adopt, BACKGROUND_CONTEXT, ); let committed = false; @@ -202,14 +204,14 @@ describe("portable SQLite facade settlement", () => { expect(await storage.mintId()).toBe(101); database.controlNextSettlement("reject"); - const rejected = storage.commit( + const rejected = commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(200), conversationId: ROOT_CONVERSATION_ID, kind: "rejected" }, }, ], - adopt, BACKGROUND_CONTEXT, ); expect(await storage.mintId()).toBe(102); @@ -225,7 +227,8 @@ describe("portable SQLite facade settlement", () => { // Each yield count starts the commit at a different point of the read's record and revision queries. for (let yields = 0; yields < 16; yields++) { const storage = await SqliteStorage.open(await openNodeSqliteDatabase(":memory:")); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -233,14 +236,13 @@ describe("portable SQLite facade settlement", () => { content: { kind: "base", version: 1, value: { value: 1 } }, }, ], - adopt, BACKGROUND_CONTEXT, ); const read = storage.document(id, "current", BACKGROUND_CONTEXT); for (let index = 0; index < yields; index++) await Promise.resolve(); - const replace = storage.commit( + const replace = commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { value: 2 } } }], - adopt, BACKGROUND_CONTEXT, ); const [stored] = await Promise.all([read, replace]); diff --git a/packages/durable/test/sqlite-migrations.test.ts b/packages/durable/test/sqlite-migrations.test.ts index 93d25ce5b..64224808d 100644 --- a/packages/durable/test/sqlite-migrations.test.ts +++ b/packages/durable/test/sqlite-migrations.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { afterEach, describe, expect, it } from "vitest"; -import { idFromNumber, seqFromNumber } from "../src/ids.ts"; +import { idFromNumber } from "../src/ids.ts"; import { applySqliteMigrations, CURRENT_SQLITE_SCHEMA_VERSION, @@ -11,6 +11,7 @@ import { type SqliteMigration, } from "../src/storage/sqlite/index.ts"; import { openNodeSqliteDatabase, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; const directories = new Set(); @@ -44,56 +45,6 @@ describe("durable SQLite migrations", () => { } }); - it("stamps records stored before version 3 with sequence 0, which only a complete log reads", async () => { - const path = await databasePath(); - const database = await openNodeSqliteDatabase(path); - await applySqliteMigrations(database, SQLITE_MIGRATIONS.slice(0, 2)); - const task = { - id: 2, - conversationId: 1, - kind: "test.task", - version: 1, - input: null, - background: false, - abortRequested: false, - state: { status: "pending", checkpoint: { phase: "run" } }, - }; - await database.exec(` - INSERT INTO record_ids (id, record_type) VALUES (1, 'conversation'), (2, 'task'), (3, 'entry'); - INSERT INTO conversations (id, record) VALUES (1, '{"id":1}'); - INSERT INTO tasks (id, conversation_id, kind, status, abort_requested, background, record) - VALUES (2, 1, '"test.task"', 'pending', 0, 0, '${JSON.stringify(task)}'); - INSERT INTO entries (id, conversation_id, commit_seq, record) - VALUES (3, 1, 2, '{"id":3,"conversationId":1,"kind":"message"}'); - UPDATE durable_metadata SET next_id = '4', next_seq = 3 WHERE singleton = 1; - `); - await database.close(); - - const storage = await openNodeSqliteStorage(path); - try { - const log = []; - for await (const record of storage.log(undefined, BACKGROUND_CONTEXT)) log.push(record); - expect(log).toEqual([ - { - seq: 2, - writes: [ - { type: "conversation", value: { id: 1 } }, - { type: "task", value: task }, - { type: "entry", value: { id: 3, conversationId: 1, kind: "message" } }, - ], - }, - ]); - // A replica holding sequence 1 already holds them. - const after = []; - for await (const record of storage.log(seqFromNumber(1), BACKGROUND_CONTEXT)) after.push(record); - expect(after).toEqual([ - { seq: 2, writes: [{ type: "entry", value: { id: 3, conversationId: 1, kind: "message" } }] }, - ]); - } finally { - await storage.close(BACKGROUND_CONTEXT); - } - }); - it("migrates a version 1 database to store sleeping tasks, keeping its tasks and indexes", async () => { const path = await databasePath(); const database = await openNodeSqliteDatabase(path); @@ -123,7 +74,7 @@ describe("durable SQLite migrations", () => { await storage.close(BACKGROUND_CONTEXT); const migrated = await openNodeSqliteDatabase(path); try { - expect(await migrated.get("SELECT version FROM durable_schema WHERE singleton = 1")).toEqual({ version: 3 }); + expect(await migrated.get("SELECT version FROM durable_schema WHERE singleton = 1")).toEqual({ version: 2 }); expect(await migrated.get("SELECT status, record FROM tasks WHERE id = 5")).toEqual({ status: "waiting", record: JSON.stringify(waiting), @@ -139,7 +90,6 @@ describe("durable SQLite migrations", () => { "tasks_by_background", "tasks_by_conversation", "tasks_by_kind", - "tasks_by_seq", "tasks_by_status", ]); expect(await migrated.get("SELECT count(*) AS count FROM sqlite_schema WHERE name = 'tasks_v2'")).toEqual({ @@ -166,7 +116,8 @@ describe("durable SQLite migrations", () => { it("rolls a failed migration back and preserves stored data for a successful retry", async () => { const path = await databasePath(); const storage = await openNodeSqliteStorage(path); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { @@ -179,7 +130,6 @@ describe("durable SQLite migrations", () => { }, }, ], - () => {}, BACKGROUND_CONTEXT, ); await storage.close(BACKGROUND_CONTEXT); diff --git a/packages/durable/test/sqlite-storage.test.ts b/packages/durable/test/sqlite-storage.test.ts index 811862dec..369768245 100644 --- a/packages/durable/test/sqlite-storage.test.ts +++ b/packages/durable/test/sqlite-storage.test.ts @@ -4,12 +4,19 @@ import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import type { Context, JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import type { SqliteDatabase } from "@earendil-works/chord/sqlite"; import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; import { afterEach, describe, expect, it } from "vitest"; import { idFromNumber } from "../src/ids.ts"; -import type { SqliteStorage } from "../src/storage/sqlite/index.ts"; -import { type NodeSqliteStorageOptions, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { type SqliteStorage, SqliteStorage as SqliteStorageClass } from "../src/storage/sqlite/index.ts"; +import { + type NodeSqliteStorageOptions, + openNodeSqliteDatabase, + openNodeSqliteStorage, +} from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -24,9 +31,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; const openStorages = new Set(); const tempDirectories = new Set(); @@ -60,13 +64,13 @@ class ReopeningStorage implements Storage { } async commit( - writes: readonly StorageWrite[], - adopt: Parameters[1], - commitContext: Parameters[2], - ): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await openNodeSqliteStorage(this.path, this.options); @@ -140,7 +144,7 @@ function entry(id: EntryId, conversationId: ConversationId, data?: JsonValue): E } async function createRoot(storage: Storage): Promise { - return storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + return commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } function pendingTask(id: TaskId): TaskRecord { @@ -179,7 +183,7 @@ describe("Pico SqliteStorage", () => { expect(await createRoot(storage)).toBe(1); const entryId = await storage.mintId(); expect( - await storage.commit([{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], adopt, context), + await commitWrites(storage, [{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], context), ).toBe(2); await storage.close(context); openStorages.delete(storage); @@ -192,9 +196,9 @@ describe("Pico SqliteStorage", () => { }); expect(await reopened.mintId()).toBe(entryId + 1); expect( - await reopened.commit( + await commitWrites( + reopened, [{ type: "task", value: pendingTask(await reopened.mintId>()) }], - adopt, context, ), ).toBe(3); @@ -257,7 +261,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -265,7 +270,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { retained: true } }, }, ], - adopt, context, ); const database = new DatabaseSync(path); @@ -283,7 +287,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -291,10 +296,10 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { nested: { value: 1 }, rows: [] } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -306,10 +311,10 @@ describe("Pico SqliteStorage", () => { }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -325,10 +330,10 @@ describe("Pico SqliteStorage", () => { }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -336,7 +341,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["s", ["nested", "value"], 4]] }, }, ], - adopt, context, ); @@ -367,7 +371,8 @@ describe("Pico SqliteStorage", () => { const circular: { self?: unknown } = {}; circular.self = circular; await expect( - storage.commit( + commitWrites( + storage, [ { type: "entry", value: entry(transientId, ROOT_CONVERSATION_ID) }, { @@ -375,7 +380,6 @@ describe("Pico SqliteStorage", () => { value: { ...pendingTask(transientTaskId), input: circular as unknown as JsonValue }, }, ], - adopt, context, ), ).rejects.toThrow("circular structure"); @@ -383,7 +387,7 @@ describe("Pico SqliteStorage", () => { expect(await storage.task(transientTaskId, context)).toBeUndefined(); const committedId = await storage.mintId(); expect( - await storage.commit([{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], adopt, context), + await commitWrites(storage, [{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], context), ).toBe(2); const db = new DatabaseSync(path, { readOnly: true }); @@ -406,15 +410,16 @@ describe("Pico SqliteStorage", () => { history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 0 } } }], - adopt, context, ); let ancientAt = createdAt; let recentAt = createdAt; for (let count = 1; count <= 40; count++) { - recentAt = await storage.commit( + recentAt = await commitWrites( + storage, [ { type: "document.change", @@ -425,7 +430,6 @@ describe("Pico SqliteStorage", () => { : { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, context, ); if (count === 5) ancientAt = recentAt; @@ -500,7 +504,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -508,13 +513,12 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); for (let count = 1; count <= 10; count++) { - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] } }], - adopt, context, ); } @@ -530,13 +534,14 @@ describe("Pico SqliteStorage", () => { } finally { revisions.close(); } - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { count: 11 } } }], - adopt, context, ); expect(revisionCount(path, id)).toBe(1); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -544,11 +549,10 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["r", { count: 12 }]] }, }, ], - adopt, context, ); expect(revisionCount(path, id)).toBe(2); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(revisionCount(path, id)).toBe(0); }); @@ -556,7 +560,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage({ walAutoCheckpointPages: 1 }); await createRoot(storage); for (let index = 0; index < 20; index++) { - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -566,7 +571,6 @@ describe("Pico SqliteStorage", () => { }), }, ], - adopt, context, ); } @@ -588,7 +592,8 @@ describe("Pico SqliteStorage", () => { await createRoot(storage); const id = await storage.mintId(); const large = "x".repeat(512 * 1024); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -596,12 +601,11 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { text: large } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: "small" } } }], - adopt, context, ); const before = new DatabaseSync(path, { readOnly: true }); @@ -614,9 +618,9 @@ describe("Pico SqliteStorage", () => { before.close(); } expect(freeAfterDelete).toBeGreaterThan(0); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: large } } }], - adopt, context, ); const after = new DatabaseSync(path, { readOnly: true }); @@ -634,7 +638,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); for (let index = 0; index < 100; index++) { - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -644,12 +649,12 @@ describe("Pico SqliteStorage", () => { }), }, ], - adopt, context, ); } const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -663,11 +668,11 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); for (let count = 1; count <= 100; count++) { - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -675,7 +680,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, context, ); } @@ -684,3 +688,130 @@ describe("Pico SqliteStorage", () => { expect((await stat(path)).size).toBeLessThan(1024 * 1024); }); }); + +describe("SqliteStorage batches", () => { + for (const mode of ["undefined rejection", "synchronous throw"] as const) { + it(`rejects the whole call, the batches before it included, when a later batch's check read fails with a ${mode}`, async () => { + // Sol's: the first batch passes its checks and is written; the second batch's check read fails. + const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-")); + tempDirectories.add(directory); + const database = await openNodeSqliteDatabase(join(directory, "storage.sqlite")); + let failCheck = false; + let reads = 0; + const faulty: SqliteDatabase = { + exec: (sql) => database.exec(sql), + run: (sql, ...params) => database.run(sql, ...params), + runCounted: (sql, ...params) => database.runCounted(sql, ...params), + get: (sql, ...params) => database.get(sql, ...params), + all: (sql, ...params) => database.all(sql, ...params), + close: () => database.close(), + transaction: (callback) => + database.transaction((transaction) => + callback({ + exec: (sql) => transaction.exec(sql), + run: (sql, ...params) => transaction.run(sql, ...params), + runCounted: (sql, ...params) => transaction.runCounted(sql, ...params), + all: (sql, ...params) => transaction.all(sql, ...params), + get: (sql, ...params) => { + if (failCheck && sql.includes("record_type") && ++reads === 2) { + if (mode === "synchronous throw") throw new Error("sync driver failure"); + return Promise.reject(undefined); + } + return transaction.get(sql, ...params); + }, + }), + ), + }; + const storage = await SqliteStorageClass.open(faulty); + openStorages.add(storage); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const ids = [await storage.mintId(), await storage.mintId()]; + const batches: StorageWrite[][] = ids.map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + ]); + failCheck = true; + const receipts: number[] = []; + const outcome = await storage + .commit(batches, (index) => receipts.push(index), context) + .then( + () => "resolved", + () => "rejected", + ); + failCheck = false; + expect(outcome).toBe("rejected"); + expect(receipts).toEqual([]); + expect(await storage.entry(ids[0]!, context)).toBeUndefined(); + }); + } + + /** How a check's read of record types fails in the database. */ + const failures: Record Promise> = { + "an Error rejection": () => Promise.reject(new Error("disk I/O error")), + "a synchronous throw": () => { + throw new Error("disk I/O error"); + }, + "a primitive rejection": () => Promise.reject("disk I/O error"), + "an undefined rejection": () => Promise.reject(undefined), + }; + for (const [name, fail] of Object.entries(failures)) { + it(`rejects a batch call whose check read fails in the database with ${name}, and refuses only for its own checks`, async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-")); + tempDirectories.add(directory); + const database = await openNodeSqliteDatabase(join(directory, "storage.sqlite")); + // A driver failure in a check's read, armed for the next batch call. + let failCheck = false; + const faulty: SqliteDatabase = { + exec: (sql) => database.exec(sql), + run: (sql, ...params) => database.run(sql, ...params), + runCounted: (sql, ...params) => database.runCounted(sql, ...params), + get: (sql, ...params) => database.get(sql, ...params), + all: (sql, ...params) => database.all(sql, ...params), + close: () => database.close(), + transaction: (callback) => + database.transaction((transaction) => + callback({ + exec: (sql) => transaction.exec(sql), + run: (sql, ...params) => transaction.run(sql, ...params), + runCounted: (sql, ...params) => transaction.runCounted(sql, ...params), + all: (sql, ...params) => transaction.all(sql, ...params), + get: (sql, ...params) => { + if (failCheck && sql.includes("record_type")) return fail(); + return transaction.get(sql, ...params); + }, + }), + ), + }; + const storage = await SqliteStorageClass.open(faulty); + openStorages.add(storage); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const first = await storage.mintId(); + const second = await storage.mintId(); + const batches: StorageWrite[][] = [first, second].map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + ]); + failCheck = true; + const receipts: number[] = []; + const outcome = await storage + .commit(batches, (index) => receipts.push(index), context) + .then( + (result) => ({ kind: "resolved", result }), + () => ({ kind: "rejected" }), + ); + failCheck = false; + // Unknown outcome, rolled back whole: nothing received, nothing stored. + expect(outcome).toEqual({ kind: "rejected" }); + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + // Durable's own check still refuses, keeping the batches before it. + const refused = await storage.commit( + [batches[0]!, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }]], + (index) => receipts.push(index), + context, + ); + expect(refused.stored).toBe(1); + expect(String(refused.refused)).toContain("already belongs to conversation"); + expect(receipts).toEqual([0]); + expect(await storage.entry(first, context)).toBeDefined(); + }); + } +}); diff --git a/packages/durable/test/storage-memory.ts b/packages/durable/test/storage-memory.ts index 0e105fdc8..49828a4b0 100644 --- a/packages/durable/test/storage-memory.ts +++ b/packages/durable/test/storage-memory.ts @@ -17,8 +17,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; type MemorySnapshot = { @@ -125,11 +126,13 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma storage = new MemoryStorage(); } else { directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-memory-`)); - path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); storage = backend === "sqlite" ? await openNodeSqliteStorage(path) - : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); + : backend === "store" + ? await openStoreStorage(path) + : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } try { @@ -151,7 +154,7 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma collectGarbage(); const postRead = snapshot(); let disk: DiskFootprint | undefined; - if (path !== undefined && backend === "sqlite") { + if (path !== undefined && backend !== "jsonl") { disk = { ...sqliteMetrics(path), mainBytes: await fileSize(path), diff --git a/packages/durable/test/storage-replay.test.ts b/packages/durable/test/storage-replay.test.ts deleted file mode 100644 index c839441a9..000000000 --- a/packages/durable/test/storage-replay.test.ts +++ /dev/null @@ -1,298 +0,0 @@ -/// - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb"; -import { afterEach, describe, expect, it } from "vitest"; -import { seqFromNumber } from "../src/ids.ts"; -import { IndexedDbStorage } from "../src/storage/indexeddb.ts"; -import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; -import { MemoryStorage } from "../src/storage/memory.ts"; -import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; -import { type LogRecord, type ReplayableStorage, ROOT_CONVERSATION_ID, type StorageWrite } from "../src/types.ts"; -import { - base, - collect, - create, - delta, - documentId, - entry, - entryId, - latestDoc, - reads, - rewindableDoc, - seed, - seedBatches, - task, -} from "./replay-support.ts"; - -Object.defineProperty(globalThis, "IDBKeyRange", { configurable: true, value: FakeIDBKeyRange }); - -const context = BACKGROUND_CONTEXT; - -const cleanups: (() => Promise)[] = []; - -afterEach(async () => { - for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); -}); - -async function temporaryDirectory(): Promise { - const directory = await mkdtemp(join(tmpdir(), "pi-durable-replay-")); - cleanups.push(() => rm(directory, { recursive: true, force: true })); - return directory; -} - -type Backend = { - readonly name: string; - open(): Promise<{ readonly storage: ReplayableStorage; reopen(): Promise }>; -}; - -function track(storage: ReplayableStorage): ReplayableStorage { - cleanups.push(() => storage.close(context).catch(() => {})); - return storage; -} - -const backends: readonly Backend[] = [ - { - name: "memory", - async open() { - const storage = new MemoryStorage(); - return { storage, reopen: async () => storage }; - }, - }, - { - name: "sqlite", - async open() { - const path = join(await temporaryDirectory(), "session.sqlite"); - let storage = track(await openNodeSqliteStorage(path)); - return { - storage, - reopen: async () => { - await storage.close(context); - storage = track(await openNodeSqliteStorage(path)); - return storage; - }, - }; - }, - }, - { - name: "jsonl", - async open() { - const directory = await temporaryDirectory(); - let storage = track(await openNodeJsonlStorage(directory, context)); - return { - storage, - reopen: async () => { - await storage.close(context); - storage = track(await openNodeJsonlStorage(directory, context)); - return storage; - }, - }; - }, - }, - { - name: "indexeddb", - async open() { - const indexedDB = new IDBFactory(); - let storage = track(await IndexedDbStorage.open("session", { indexedDB })); - return { - storage, - reopen: async () => { - await storage.close(context); - storage = track(await IndexedDbStorage.open("session", { indexedDB })); - return storage; - }, - }; - }, - }, -]; - -/** Bring `replica` up to `source` the way a sync flush does: the source's log after the replica's head. */ -async function catchUp(source: ReplayableStorage, replica: ReplayableStorage): Promise { - const records = await collect(source.log(await replica.head(context), context)); - for (const record of records) await replica.replay(record, context); - return records; -} - -describe("log and replay", () => { - for (const source of backends) { - for (const target of backends) { - it(`rebuilds ${source.name} in ${target.name} with the same reads`, async () => { - const opened = await source.open(); - await seed(opened.storage); - const reopened = await opened.reopen(); - const log = await collect(reopened.log(undefined, context)); - expect(log.at(-1)?.seq).toBe(await reopened.head(context)); - for (const record of log) { - expect(record.writes.some((write) => (write as StorageWrite).type === "document.copy")).toBe(false); - } - - const replica = (await target.open()).storage; - for (const record of log) await replica.replay(record, context); - expect(await reads(replica)).toEqual(await reads(reopened)); - // The replica mints past every replayed ID, as the source does. - expect(await replica.mintId()).toBe(await reopened.mintId()); - }); - - it(`catches ${target.name} up with ${source.name} after every commit`, async () => { - const { storage } = await source.open(); - const replica = (await target.open()).storage; - for (const batch of seedBatches()) { - await storage.commit(batch, () => {}, context); - await catchUp(storage, replica); - expect(await reads(replica)).toEqual(await reads(storage)); - } - }); - } - - it(`catches up from ${source.name} at every pace, keeping only the surviving changes`, async () => { - for (const every of [2, 3, 4]) { - const { storage } = await source.open(); - const replica = new MemoryStorage(); - const batches = seedBatches(); - for (let index = 0; index < batches.length; index++) { - await storage.commit(batches[index]!, () => {}, context); - if ((index + 1) % every === 0 || index === batches.length - 1) await catchUp(storage, replica); - } - expect(await reads(replica), `every ${every}`).toEqual(await reads(storage)); - } - }); - } - - for (const backend of backends) { - describe(backend.name, () => { - it("logs nothing from an empty storage, and nothing after the head", async () => { - const { storage } = await backend.open(); - expect(await storage.head(context)).toBeUndefined(); - expect(await collect(storage.log(undefined, context))).toEqual([]); - await seed(storage); - expect(await collect(storage.log(seqFromNumber(10), context))).toEqual([]); - }); - - it("rejects a log after a sequence past the head", async () => { - const { storage } = await backend.open(); - await seed(storage); - await expect(collect(storage.log(seqFromNumber(11), context))).rejects.toThrow("past the head"); - }); - - it("logs each record at its own stamp and ends at the head", async () => { - const { storage } = await backend.open(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, context); - await storage.commit([{ type: "task", value: task(2, "pending") }], () => {}, context); - await storage.commit([entry(3)], () => {}, context); - await storage.commit([{ type: "task", value: task(2, "running") }], () => {}, context); - await storage.commit( - [ - { type: "task", value: task(4, "pending") }, - { type: "task", value: task(4, "running") }, - ], - () => {}, - context, - ); - await storage.commit([entry(5)], () => {}, context); - expect(await collect(storage.log(undefined, context))).toEqual([ - { seq: 1, writes: [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }] }, - { seq: 3, writes: [entry(3)] }, - { seq: 4, writes: [{ type: "task", value: task(2, "running") }] }, - { seq: 5, writes: [{ type: "task", value: task(4, "running") }] }, - { seq: 6, writes: [entry(5)] }, - ]); - expect(await collect(storage.log(seqFromNumber(4), context))).toEqual([ - { seq: 5, writes: [{ type: "task", value: task(4, "running") }] }, - { seq: 6, writes: [entry(5)] }, - ]); - await storage.commit([{ type: "task", value: task(2, "terminal") }], () => {}, context); - expect(await collect(storage.log(seqFromNumber(6), context))).toEqual([ - { seq: 7, writes: [{ type: "task", value: task(2, "terminal") }] }, - ]); - }); - - it("logs only what followed the newest base of a current-only document", async () => { - const { storage } = await backend.open(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, context); - await storage.commit([create(latestDoc(2), { n: 0 })], () => {}, context); - await storage.commit([delta(2, "n", 1)], () => {}, context); - await storage.commit([delta(2, "n", 2)], () => {}, context); - await storage.commit([base(2, { n: 3 })], () => {}, context); - await storage.commit([delta(2, "n", 4)], () => {}, context); - expect(await collect(storage.log(seqFromNumber(2), context))).toEqual([ - { seq: 5, writes: [base(2, { n: 3 })] }, - { seq: 6, writes: [delta(2, "n", 4)] }, - ]); - // Created after the sequence with its base reclaimed: the placeholder, then what survived. - expect(await collect(storage.log(seqFromNumber(1), context))).toEqual([ - { - seq: 2, - writes: [ - { - type: "document.create", - record: latestDoc(2), - content: { kind: "base", version: 1, value: {} }, - }, - ], - }, - { seq: 5, writes: [base(2, { n: 3 })] }, - { seq: 6, writes: [delta(2, "n", 4)] }, - ]); - }); - - it("replays at explicit sequences with gaps and continues after them", async () => { - const { storage } = await backend.open(); - await storage.replay( - { seq: seqFromNumber(3), writes: [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }] }, - context, - ); - await storage.replay({ seq: seqFromNumber(7), writes: [entry(9)] }, context); - expect(await storage.head(context)).toBe(7); - expect((await storage.entry(entryId(9), context))?.commitSeq).toBe(7); - expect(await storage.mintId()).toBe(10); - expect(await storage.commit([entry(10)], () => {}, context)).toBe(8); - await expect(storage.replay({ seq: seqFromNumber(8), writes: [entry(11)] }, context)).rejects.toThrow( - "does not strictly increase", - ); - expect(await storage.entry(entryId(11), context)).toBeUndefined(); - }); - - it("refuses document copies in a log record", async () => { - const { storage } = await backend.open(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, context); - const copy = { - type: "document.copy", - record: latestDoc(3), - source: { id: documentId(2), at: "current" }, - } as unknown as LogRecord["writes"][number]; - await expect(storage.replay({ seq: seqFromNumber(2), writes: [copy] }, context)).rejects.toThrow( - "cannot contain a document copy", - ); - }); - - it("rejects a log that a commit interrupts", async () => { - const { storage } = await backend.open(); - await seed(storage); - const iterator = storage.log(undefined, context)[Symbol.asyncIterator](); - await iterator.next(); - await storage.commit([entry(100)], () => {}, context); - await expect(async () => { - while (!(await iterator.next()).done); - }).rejects.toThrow("changed while its log was read"); - }); - }); - } - - it("streams a SQLite log across pages", async () => { - const sqlite = backends.find((backend) => backend.name === "sqlite")!; - const { storage } = await sqlite.open(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, context); - await storage.commit([create(rewindableDoc(2), { n: 0 })], () => {}, context); - for (let id = 3; id < 1000; id += 2) await storage.commit([entry(id), delta(2, "n", id)], () => {}, context); - const log = await collect(storage.log(undefined, context)); - expect(log).toHaveLength(501); - - const replica = new MemoryStorage(); - for (const record of log) await replica.replay(record, context); - expect(await reads(replica)).toEqual(await reads(storage)); - expect(await collect(storage.log(seqFromNumber(400), context))).toHaveLength(101); - }); -}); diff --git a/packages/durable/test/storage.bench.ts b/packages/durable/test/storage.bench.ts index f24e915eb..68fdb06a3 100644 --- a/packages/durable/test/storage.bench.ts +++ b/packages/durable/test/storage.bench.ts @@ -14,8 +14,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { EntryId, Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; const READ_OPTIONS = { time: 300, iterations: 10, warmupTime: 75, warmupIterations: 3 } as const; @@ -38,9 +39,10 @@ async function createFixture(backend: StorageBenchmarkBackend): Promise } const directory = await mkdtemp(join(tmpdir(), "pi-durable-benchmark-")); directories.push(directory); - if (backend === "sqlite") { + if (backend === "sqlite" || backend === "store") { const path = join(directory, "storage.sqlite"); - const fixture = { backend, storage: await openNodeSqliteStorage(path) } satisfies Fixture; + const storage = backend === "sqlite" ? await openNodeSqliteStorage(path) : await openStoreStorage(path); + const fixture = { backend, storage } satisfies Fixture; fixtures.push(fixture); return fixture; } @@ -123,11 +125,12 @@ type ReopenFixture = { }; async function openPersistentStorage(backend: PersistentBackend, path: string): Promise { + if (backend === "store") return openStoreStorage(path); return backend === "sqlite" ? openNodeSqliteStorage(path) : openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } async function copyPersistentStorage(backend: PersistentBackend, source: string, destination: string): Promise { - if (backend === "sqlite") await copyFile(source, destination); + if (backend !== "jsonl") await copyFile(source, destination); else await cp(source, destination, { recursive: true }); } @@ -136,7 +139,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { if (backend === "memory") continue; const seedDirectory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-benchmark-`)); directories.push(seedDirectory); - const seedPath = join(seedDirectory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const seedPath = join(seedDirectory, backend === "jsonl" ? "storage" : "storage.sqlite"); const seed = await openPersistentStorage(backend, seedPath); const dataset = await seedStorageBenchmark(seed); await seed.close(BACKGROUND_CONTEXT); @@ -144,7 +147,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { Array.from({ length: REOPEN_OPTIONS.iterations + REOPEN_OPTIONS.warmupIterations }, async () => { const directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-sample-`)); directories.push(directory); - const path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); await copyPersistentStorage(backend, seedPath, path); return path; }), diff --git a/packages/durable/test/store-storage-commit.test.ts b/packages/durable/test/store-storage-commit.test.ts new file mode 100644 index 000000000..30f79a573 --- /dev/null +++ b/packages/durable/test/store-storage-commit.test.ts @@ -0,0 +1,769 @@ +// StoreStorage's commit path (`Storage.commit` over `ChordStore.commitSeries`), p3's primary storage: per call one +// backend transaction with one seq per batch; Durable's checks of a batch refuse it in its turn, the batches before it +// stored; a failed statement rolls the call back and rejects, the store going on; a failed `COMMIT` stops the store; +// receipts after `COMMIT`; each commit published in a host batch of its own; other users of the store wait their turn. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { BACKGROUND_CONTEXT, withTelemetry } from "@earendil-works/chord/context"; +import { type ChordStore, createHost, openStore, sqliteBackend } from "@earendil-works/chord/next"; +import type { SqliteDatabase, SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { MemoryStorage } from "../src/index.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { + CommitResult, + ConversationId, + DocumentCreate, + DocumentId, + EntryId, + Seq, + Storage, + StorageWrite, +} from "../src/types.ts"; +import { ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +const context = BACKGROUND_CONTEXT; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A fresh SQLite file in this test's directory. */ +async function storePath(): Promise { + directory ??= await mkdtemp(join(tmpdir(), "pi-durable-store-commit-")); + return join(directory, `store-${Math.random().toString(36).slice(2)}.sqlite`); +} + +/** + * What the next backend transaction does besides its work: `commit` fails it after its work (a failed `COMMIT`, an + * unknown outcome); `fail` fails a statement it matches (by its SQL and its index, from 1), as a backend error; `hold` + * holds a statement until the promise it returns settles. + */ +type Faults = { + commit: boolean; + fail: ((sql: string, index: number) => boolean) | undefined; + hold: ((sql: string, index: number) => Promise | undefined) | undefined; +}; + +/** + * A storage with its root conversation, over a store with an outbox on the SQLite file at `path`, whose backend counts + * its transactions and the statements of the last one, and takes `faults` for its next transaction. + */ +async function counted(path = ":memory:") { + const db = await openNodeSqliteDatabase(path); + const counts = { transactions: 0, statements: 0 }; + const faults: Faults = { commit: false, fail: undefined, hold: undefined }; + const backend: SqliteDatabase = { + exec: (sql) => db.exec(sql), + run: (sql, ...params) => db.run(sql, ...params), + runCounted: (sql, ...params) => db.runCounted(sql, ...params), + get: (sql, ...params) => db.get(sql, ...params), + all: (sql, ...params) => db.all(sql, ...params), + close: () => db.close(), + transaction: (work) => { + counts.transactions++; + counts.statements = 0; + const { commit, fail, hold } = faults; + faults.commit = false; + faults.fail = undefined; + faults.hold = undefined; + return db.transaction(async (executor) => { + const call = async (sql: string, run: () => Promise): Promise => { + const index = ++counts.statements; + await hold?.(sql, index); + if (fail?.(sql, index) === true) throw new Error(`injected: statement ${index} failed`); + return run(); + }; + const result = await work({ + exec: (sql) => call(sql, () => executor.exec(sql)), + run: (sql, ...params) => call(sql, () => executor.run(sql, ...params)), + runCounted: (sql, ...params) => call(sql, () => executor.runCounted(sql, ...params)), + get: (sql, ...params) => call(sql, () => executor.get(sql, ...params)), + all: (sql, ...params) => call(sql, () => executor.all(sql, ...params)), + } satisfies SqliteExecutor); + if (commit) throw new Error("COMMIT failed"); + return result; + }); + }, + }; + const store = (await openStore(sqliteBackend(backend), context, { outbox: true })) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + return { store, storage, counts, faults }; +} + +/** The same storage opened again on the file at `path`, as another process would. */ +async function reopen(path: string) { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + return { store, storage: await StoreStorage.open(store, "harness", context) }; +} + +const entryWrite = (id: EntryId, kind = "m"): StorageWrite => ({ + type: "entry", + value: { id, conversationId: ROOT_CONVERSATION_ID, kind }, +}); + +/** One macrotask: what can run before it has run. */ +const macrotask = (): Promise => new Promise((resolve) => setTimeout(resolve, 0)); + +/** The store's outbox commits, by seq. */ +async function outboxSeqs(store: ChordStore): Promise { + const rows = await store.read((executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox"), context); + return rows.map((row) => row.seq); +} + +describe("one transaction per call", () => { + it("stores the batches in one backend transaction, one seq and one outbox commit each, receipts in order", async () => { + const { store, storage, counts } = await counted(); + const ids = [await storage.mintId(), await storage.mintId(), await storage.mintId()]; + const before = counts.transactions; + const last = store.lastSeq; + const receipts: [number, Seq][] = []; + const result = await storage.commit( + ids.map((id) => [entryWrite(id)]), + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(result).toEqual({ stored: 3 }); + expect(counts.transactions - before).toBe(1); + expect(receipts).toEqual([ + [0, last + 1], + [1, last + 2], + [2, last + 3], + ]); + for (const [index, id] of ids.entries()) { + expect((await storage.entry(id, context))?.commitSeq).toBe(last + 1 + index); + } + const outbox = await store.read( + (executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox WHERE seq > ?", last), + context, + ); + expect(outbox.map((row) => row.seq)).toEqual([last + 1, last + 2, last + 3]); + await store.close(context); + }); + + it("writes the metadata once per call: three statements per entry append, plus one (cost guard)", async () => { + const { store, storage, counts } = await counted(); + const telemetry = new InMemoryTelemetryContext(); + const span = await telemetry.startSpan({ name: "test" }, (each) => each); + const metered = withTelemetry(span, context); + const statements = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.store.statements") + .reduce((total, { value }) => total + value, 0); + // Alone, an append is its id check, its row and claim, its outbox commit, and the metadata. + await commitWrites(storage, [entryWrite(await storage.mintId())], metered); + expect(statements()).toBe(5); + const ids = [await storage.mintId(), await storage.mintId(), await storage.mintId()]; + const before = counts.transactions; + await storage.commit( + ids.map((id) => [entryWrite(id)]), + () => {}, + metered, + ); + expect([counts.transactions - before, statements() - 5]).toEqual([1, 3 * 4 + 1]); + await store.close(context); + }); +}); + +describe("refusals: the batches before the refused one stand", () => { + it("stores the batches before a refused one with Durable's message, none after it, and goes on", async () => { + const { store, storage } = await counted(); + const first = await storage.mintId(); + const after = await storage.mintId(); + const last = store.lastSeq; + const receipts: number[] = []; + const result = await storage.commit( + [[entryWrite(first)], [entryWrite(first, "again")], [entryWrite(after)]], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${first} already belongs to entry`); + expect(receipts).toEqual([0]); + expect((await storage.entry(first, context))?.entry.kind).toBe("m"); + expect(await storage.entry(after, context)).toBeUndefined(); + expect(store.lastSeq).toBe(last + 1); + expect(await commitWrites(storage, [entryWrite(after)], context)).toBe(last + 2); + await store.close(context); + }); + + it("refuses a batch its staging fails in that batch's turn, keeping the batches before it (review)", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const document = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [entryWrite(entry)], + // Staging refuses it: one document retired twice. + [ + { type: "document.retire", id: document }, + { type: "document.retire", id: document }, + ], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: Document ${document} is retired more than once`); + expect(receipts).toEqual([0]); + expect(await storage.entry(entry, context)).toBeDefined(); + await store.close(context); + }); + + it.each([ + [ + "a duplicate retirement", + [ + { type: "document.retire", id: 101 }, + { type: "document.retire", id: 101 }, + ], + ], + [ + "a duplicate content command", + [ + { type: "document.change", id: 101, content: { kind: "base", version: 1, value: {} } }, + { type: "document.change", id: 101, content: { kind: "base", version: 1, value: {} } }, + ], + ], + ] as const)( + "refuses a later batch with %s as Memory does, keeping the prefix (review)", + async (_name, malformed) => { + const batches: StorageWrite[][] = [ + [{ type: "conversation", value: { id: idFromNumber(100) } }], + malformed.map((write) => ({ ...write, id: idFromNumber(write.id) })) as StorageWrite[], + ]; + const outcomes = []; + for (const storage of [new MemoryStorage(), await openStoreStorage(":memory:")]) { + const receipts: number[] = []; + const result = await storage.commit(batches, (index) => receipts.push(index), context); + const conversation = await storage.conversation(idFromNumber(100), context); + outcomes.push({ stored: result.stored, refused: String(result.refused), receipts, conversation }); + await storage.close(context); + } + expect(outcomes[0]).toMatchObject({ stored: 1, receipts: [0], conversation: { id: 100 } }); + expect(outcomes[1]).toEqual(outcomes[0]); + }, + ); + + it.each([ + [ + "a change of a document an earlier batch retired", + (id: DocumentId): StorageWrite[][] => [ + [{ type: "document.retire", id }], + [{ type: "document.change", id, content: { kind: "base", version: 1, value: { n: 2 } } }], + ], + "is retired", + ], + [ + "a creation at the address an earlier batch took", + (id: DocumentId): StorageWrite[][] => [ + [ + { + type: "document.create", + record: { id: idFromNumber(id + 1), kind: "n", key: "taken", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 2 } }, + }, + ], + [ + { + type: "document.create", + record: { id: idFromNumber(id + 2), kind: "n", key: "taken", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 3 } }, + }, + ], + ], + "already has a current incarnation", + ], + ] as const)("refuses %s against the state the batches before it left", async (_name, calls, message) => { + const { store, storage } = await counted(); + const document = idFromNumber(200); + await commitWrites( + storage, + [ + { + type: "document.create", + record: { id: document, kind: "n", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + context, + ); + const last = store.lastSeq; + const receipts: number[] = []; + const result = await storage.commit(calls(document), (index) => receipts.push(index), context); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain(message); + expect([receipts, store.lastSeq]).toEqual([[0], last + 1]); + expect((await outboxSeqs(store)).at(-1)).toBe(last + 1); + await store.close(context); + }); + + it("refuses a batch whose rows staging cannot encode, keeping the batches before it", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const receipts: number[] = []; + // No integer: the Session never sends one (Storage trusts it); the store's columns refuse it at staging. + const result = await storage.commit( + [[entryWrite(entry)], [entryWrite(idFromNumber(1.5))]], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain('column "id": 1.5 is not a int value'); + expect(receipts).toEqual([0]); + expect(await storage.entry(entry, context)).toBeDefined(); + await store.close(context); + }); + + it("checks a batch against the batches before it in the call: a copy reads their state", async () => { + const { store, storage } = await counted(); + const source = await storage.mintId(); + const child = await storage.mintId(); + const copy = await storage.mintId(); + const record = (id: DocumentId, conversationId: ConversationId) => + ({ + id, + kind: "c", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }) as const; + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: record(source, ROOT_CONVERSATION_ID), + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + [{ type: "document.change", id: source, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }], + [ + { type: "conversation", value: { id: child } }, + { type: "document.copy", record: record(copy, child), source: { id: source, at: "current" } }, + ], + ], + () => {}, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copy, "current", context))?.value).toEqual({ n: 2 }); + await store.close(context); + }); +}); + +describe("failures: a rollback rejects, an unknown outcome stops the store", () => { + it("rejects with nothing stored when a batch's check fails in the backend, and the store goes on", async () => { + const { store, storage, faults } = await counted(); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const last = store.lastSeq; + // The second batch's id check: its first read of the record ids. + let claimReads = 0; + faults.fail = (sql) => sql.includes("record_ids") && ++claimReads === 2; + const receipts: number[] = []; + await expect( + storage.commit([[entryWrite(first)], [entryWrite(second)]], (index) => receipts.push(index), context), + ).rejects.toThrow("injected"); + expect([receipts, store.lastSeq]).toEqual([[], last]); + expect(await storage.entry(first, context)).toBeUndefined(); + expect(await commitWrites(storage, [entryWrite(first)], context)).toBe(last + 1); + await store.close(context); + }); + + it("a failed COMMIT is an unknown outcome: the call rejects storage_failed, nothing is received or stored", async () => { + const path = await storePath(); + const { store, storage, faults } = await counted(path); + const ids = [await storage.mintId(), await storage.mintId()]; + const last = store.lastSeq; + faults.commit = true; + const receipts: number[] = []; + await expect( + storage.commit( + ids.map((id) => [entryWrite(id)]), + (index) => receipts.push(index), + context, + ), + ).rejects.toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect(receipts).toEqual([]); + await expect(storage.entry(ids[0]!, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await store.close(context); + const reopened = await reopen(path); + expect(await reopened.storage.entry(ids[0]!, context)).toBeUndefined(); + expect(await commitWrites(reopened.storage, [entryWrite(ids[1]!)], context)).toBe(last + 1); + await reopened.store.close(context); + }); + + /** The record the audit's source document and its copy share. */ + const record = (id: number, conversationId: number): DocumentCreate => ({ + id: idFromNumber(id), + kind: "audit", + scope: { kind: "conversation", conversationId: idFromNumber(conversationId) }, + history: "rewindable", + fork: "asOf", + }); + + /** The source document, stored before the call. */ + const SEED: StorageWrite[] = [ + { + type: "document.create", + record: record(100, ROOT_CONVERSATION_ID), + content: { kind: "base", version: 1, value: { n: 0 } }, + }, + ]; + + /** A delta of the source document setting `n`. */ + const delta = (n: number): StorageWrite => ({ + type: "document.change", + id: idFromNumber(100), + content: { kind: "delta", version: 1, ops: [["s", ["n"], n]] }, + }); + + /** + * Three batches: an entry and a delta; a conversation, a copy of the source as the first batch left it and an + * entry; then an entry and a delta, or, `refused`, an entry whose id the first batch took. + */ + const batches = (third: "valid" | "refused"): StorageWrite[][] => [ + [entryWrite(idFromNumber(101), "b1"), delta(1)], + [ + { type: "conversation", value: { id: idFromNumber(105) } }, + { + type: "document.copy", + record: record(102, 105), + source: { id: idFromNumber(100), at: "current" }, + }, + entryWrite(idFromNumber(103), "b2"), + ], + third === "valid" + ? [entryWrite(idFromNumber(104), "b3"), delta(2)] + : [entryWrite(idFromNumber(101), "again")], + ]; + + /** What a storage holds of the audit's records. */ + async function stateOf(storage: Storage) { + const entries: (string | null)[] = []; + for (const id of [101, 103, 104]) + entries.push((await storage.entry(idFromNumber(id), context))?.entry.kind ?? null); + return { + entries, + source: (await storage.document(idFromNumber(100), "current", context))?.value ?? null, + copy: (await storage.document(idFromNumber(102), "current", context))?.value ?? null, + child: (await storage.conversation(idFromNumber(105), context)) ?? null, + }; + } + + /** What a call returned and what it left: its result, its receipts' seqs after the store's last one, the state. */ + async function outcomeOf(storage: Storage, third: "valid" | "refused", last: number) { + const seqs: number[] = []; + const result: CommitResult = await storage.commit( + batches(third), + (_index, seq) => seqs.push(seq - last), + context, + ); + return { stored: result.stored, refused: String(result.refused), seqs, state: await stateOf(storage) }; + } + + /** Every error in `error`'s cause chain, as text. */ + function chain(error: unknown): string[] { + const messages: string[] = []; + for (let at = error; at instanceof Error; at = at.cause) messages.push(at.message); + return messages; + } + + it.each(["valid", "refused"] as const)( + "rolls a 3-batch call back at a failure of any of its statements, as at a failed COMMIT, and reopens before it (third %s)", + async (third) => { + // What MemoryStorage makes of the call: the outcome every retry must reach. + const memory = new MemoryStorage(); + await commitWrites(memory, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + await commitWrites(memory, SEED, context); + const expected = await outcomeOf(memory, third, 2); + expect(expected).toMatchObject( + third === "valid" ? { stored: 3, seqs: [1, 2, 3] } : { stored: 2, seqs: [1, 2] }, + ); + // The statements of the call's transaction (a read transaction after it would count its own). + const probe = await counted(); + await commitWrites(probe.storage, SEED, context); + const result = await probe.storage.commit(batches(third), () => {}, context); + const total = probe.counts.statements; + // Its cost: no read beyond a batch's checks, the metadata once (statement counts per batch in the cost guard). + expect(total).toBe(third === "valid" ? 29 : 23); + expect(result.stored).toBe(expected.stored); + expect(await stateOf(probe.storage)).toEqual(expected.state); + await probe.store.close(context); + // Each statement in turn, then the COMMIT. + for (let at = 1; at <= total + 1; at++) { + const path = await storePath(); + const opened = await counted(path); + await commitWrites(opened.storage, SEED, context); + const last = opened.store.lastSeq; + const outbox = await outboxSeqs(opened.store); + const before = await stateOf(opened.storage); + if (at <= total) opened.faults.fail = (_sql, index) => index === at; + else opened.faults.commit = true; + const receipts: number[] = []; + const error = await opened.storage + .commit(batches(third), (each) => receipts.push(each), context) + .then( + () => undefined, + (rejection: unknown) => rejection, + ); + expect(receipts).toEqual([]); + expect(opened.store.lastSeq).toBe(last); + if (at <= total) { + // A rollback: the call rejects with its failure, the store goes on. + expect(chain(error)).toContain(`injected: statement ${at} failed`); + expect(await stateOf(opened.storage)).toEqual(before); + } else { + expect(error).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + } + await opened.store.close(context); + // On disk: as before the call, the seq, the outbox, the records and the allocator. + const again = await reopen(path); + expect(again.store.lastSeq).toBe(last); + expect(await outboxSeqs(again.store)).toEqual(outbox); + expect(await stateOf(again.storage)).toEqual(before); + expect(await again.storage.mintId()).toBeGreaterThan(100); + // A retry reaches MemoryStorage's outcome. + expect(await outcomeOf(again.storage, third, last)).toEqual(expected); + expect(await outboxSeqs(again.store)).toEqual([...outbox, ...expected.seqs.map((seq) => last + seq)]); + await again.store.close(context); + } + }, + ); + + it("a call on a closed storage, or on a stopped store, rejects and stores nothing", async () => { + const { store, storage, faults } = await counted(); + const entry = await storage.mintId(); + faults.commit = true; + await expect(commitWrites(storage, [entryWrite(entry)], context)).rejects.toMatchObject({ + reason: "storage_failed", + }); + await expect(storage.commit([[entryWrite(entry)]], () => {}, context)).rejects.toMatchObject({ + reason: "storage_failed", + }); + await store.close(context); + const other = await counted(); + await other.storage.close(context); + await expect(other.storage.commit([[entryWrite(entry)]], () => {}, context)).rejects.toThrow("closed"); + await other.store.close(context); + }); + + it("a call queued when the store closes rejects released, and stores nothing; the running one is stored", async () => { + const path = await storePath(); + const { store, storage, faults } = await counted(path); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + faults.hold = (_sql, index) => { + if (index !== 1) return undefined; + entered.resolve(); + return release.promise; + }; + const running = storage.commit([[entryWrite(first)]], () => {}, context); + await entered.promise; + const queued = storage.commit([[entryWrite(second)]], () => {}, context); + const closing = store.close(context); + release.resolve(); + expect(await running).toEqual({ stored: 1 }); + await expect(queued).rejects.toMatchObject({ code: "released" }); + await closing; + const again = await reopen(path); + expect(await again.storage.entry(first, context)).toBeDefined(); + expect(await again.storage.entry(second, context)).toBeUndefined(); + await again.store.close(context); + }); +}); + +describe("durability", () => { + it("keeps every batch of a stored call across a crash: reopened, the seqs, rows, documents and ids are there", async () => { + const path = await storePath(); + const { store, storage } = await counted(path); + const entry = await storage.mintId(); + const document = await storage.mintId(); + const record = { + id: document, + kind: "n", + scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID }, + history: "rewindable", + fork: "asOf", + } as const; + const seqs: Seq[] = []; + await storage.commit( + [ + [entryWrite(entry)], + [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 1 } } }], + [{ type: "document.change", id: document, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }], + ], + (_index, seq) => seqs.push(seq), + context, + ); + const minted = await storage.mintId(); + // A crash: the store is never closed; another process opens the file. + const { store: other, storage: reopened } = await reopen(path); + expect((await reopened.entry(entry, context))?.commitSeq).toBe(seqs[0]); + expect((await reopened.document(document, seqs[1]!, context))?.value).toEqual({ n: 1 }); + expect((await reopened.document(document, "current", context))?.value).toEqual({ n: 2 }); + expect(await reopened.mintId()).toBeGreaterThan(document); + expect(await commitWrites(reopened, [entryWrite(minted)], context)).toBe(seqs[2]! + 1); + await other.close(context); + await store.close(context); + }); +}); + +describe("publication", () => { + it("publishes each commit of a call in a host batch of its own, after the call's one transaction", async () => { + const { store, storage, counts } = await counted(); + const host = createHost({ store, onError: () => {} }); + const session = new SessionImpl(storage); + const events: string[] = []; + session.followCommits((commit) => { + if (commit.kind !== "stored") return; + const batch = store.batch!; + events.push(`publish ${commit.seq} running=${batch.running}`); + batch.flushing(() => events.push(`flush ${commit.seq}`)); + }); + // Hold the first call, so the commits after it are sent together in the next one. + const calls: number[] = []; + const gate = Promise.withResolvers(); + const commit = storage.commit.bind(storage); + storage.commit = async (batches, receive, ctx) => { + calls.push(batches.length); + if (calls.length === 1) await gate.promise; + return commit(batches, receive, ctx); + }; + const first = session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "a" }), context); + await new Promise((resolve) => setTimeout(resolve, 0)); + const rest = [1, 2, 3].map((n) => + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: `b${n}` }), context), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + const before = counts.transactions; + gate.resolve(); + await Promise.all([first, ...rest]); + expect(calls).toEqual([1, 3]); + expect(counts.transactions - before).toBe(2); + const seq = store.lastSeq; + expect(events).toEqual( + [seq - 3, seq - 2, seq - 1, seq].flatMap((each) => [`publish ${each} running=true`, `flush ${each}`]), + ); + await session.close(context); + await host.close(context); + await store.close(context); + }); +}); + +describe("other users of the store", () => { + it("another owner's call waits for a series in its transaction; the series' receipts come first", async () => { + const { store, storage, faults } = await counted(); + const other = await StoreStorage.open(store, "other", context); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + faults.hold = (_sql, index) => { + if (index !== 2) return undefined; + entered.resolve(); + return release.promise; + }; + const events: string[] = []; + const series = storage.commit( + [[entryWrite(first)], [entryWrite(second)]], + (index, seq) => events.push(`harness ${index} ${seq}`), + context, + ); + await entered.promise; + const waiting = other.commit( + [[{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }]], + (index, seq) => events.push(`other ${index} ${seq}`), + context, + ); + await macrotask(); + expect(events).toEqual([]); + const last = store.lastSeq; + release.resolve(); + await Promise.all([series, waiting]); + expect(events).toEqual([`harness 0 ${last + 1}`, `harness 1 ${last + 2}`, `other 0 ${last + 3}`]); + expect((await outboxSeqs(store)).slice(-3)).toEqual([last + 1, last + 2, last + 3]); + await other.close(context); + await store.close(context); + }); + + it("a read during a series sees all of it or none of it", async () => { + const { store, storage, faults } = await counted(); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + // Held after the first batch's writes, in the second batch's check. + let claimReads = 0; + faults.hold = (sql) => { + if (!sql.includes("record_ids") || ++claimReads !== 2) return undefined; + entered.resolve(); + return release.promise; + }; + const series = storage.commit([[entryWrite(first)], [entryWrite(second)]], () => {}, context); + await entered.promise; + const read = Promise.all([storage.entry(first, context), storage.entry(second, context)]); + await macrotask(); + release.resolve(); + await series; + const [one, two] = await read; + expect([one === undefined, two === undefined]).toEqual([false, false]); + await store.close(context); + }); + + it("an export holds the queue: a series waits for it and is not in its archive", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const last = store.lastSeq; + const parts = store.export(context); + const manifest = await parts.next(); + expect(manifest.value).toMatchObject({ kind: "manifest", lastSeq: last }); + let settled = false; + const series = storage + .commit([[entryWrite(entry)]], () => {}, context) + .finally(() => { + settled = true; + }); + await macrotask(); + expect(settled).toBe(false); + await parts.return?.(); + expect(await series).toEqual({ stored: 1 }); + expect(store.lastSeq).toBe(last + 1); + await store.close(context); + }); +}); + +describe("GC", () => { + it("keeps nothing of a settled call: its batches are collected", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const { store, storage } = await counted(); + const sent = async (): Promise> => { + const batches = [[entryWrite(await storage.mintId())]]; + await storage.commit(batches, () => {}, context); + return new WeakRef(batches); + }; + const ref = await sent(); + for (let round = 0; round < 10 && ref.deref() !== undefined; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(ref.deref()).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/durable/test/store-storage-node.ts b/packages/durable/test/store-storage-node.ts new file mode 100644 index 000000000..3dff775d4 --- /dev/null +++ b/packages/durable/test/store-storage-node.ts @@ -0,0 +1,23 @@ +// What `vitest.store.config.ts` puts in the place of `../src/storage/sqlite/node.ts`: the same exports, with +// `openNodeSqliteStorage(path)` opening Durable's storage on Chord's store over that file (`StoreStorage`), so the +// harness test files that take a SQLite storage run unchanged on the store. + +import type { NodeSqliteDatabaseOptions } from "@earendil-works/chord/sqlite/node"; +import type { SqliteStorageOptions } from "../src/storage/sqlite/index.ts"; +import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +export { + NodeSqliteDatabase, + type NodeSqliteDatabaseOptions, + openNodeSqliteDatabase, +} from "@earendil-works/chord/sqlite/node"; + +/** Node SQLite connection settings plus durable storage options, as the original takes them. */ +export type NodeSqliteStorageOptions = NodeSqliteDatabaseOptions & SqliteStorageOptions; + +/** A `StoreStorage` on a store over the file at `path`; a table prefix becomes nothing (the owner name separates). */ +export function openNodeSqliteStorage(path: string, options: NodeSqliteStorageOptions = {}): Promise { + const { prefix: _prefix, ...database } = options; + return openStoreStorage(path, database); +} diff --git a/packages/durable/test/store-storage-support.ts b/packages/durable/test/store-storage-support.ts new file mode 100644 index 000000000..1143d6eaf --- /dev/null +++ b/packages/durable/test/store-storage-support.ts @@ -0,0 +1,46 @@ +// StoreStorage over a SQLite file for tests: a store opened on the file, Durable's storage in it under a test owner +// name, and a close that closes the store too (the app's part: in tests the storage is the store's only owner). + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { openStore, sqliteBackend } from "@earendil-works/chord/next"; +import { type NodeSqliteDatabaseOptions, openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import type { Storage } from "../src/types.ts"; + +/** The owner name the test storages use. */ +export const TEST_OWNER = "durable.test"; + +/** A `StoreStorage` on a store over the SQLite file at `path` (`:memory:` too) whose `close` closes the store too. */ +export async function openStoreStorage(path: string, options: NodeSqliteDatabaseOptions = {}): Promise { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path, options)), BACKGROUND_CONTEXT); + let storage: StoreStorage; + try { + storage = await StoreStorage.open(store, TEST_OWNER, BACKGROUND_CONTEXT); + } catch (error) { + await store.close(BACKGROUND_CONTEXT); + throw error; + } + let closing: Promise | undefined; + const close = (context: Parameters[0]): Promise => { + closing ??= storage.close(context).then(() => store.close(context)); + return closing; + }; + // StoreStorage keeps its state in private fields: its methods run on the storage itself, bound once. + const bound = new Map([["close", close]]); + return new Proxy(storage, { + get(target, property) { + let value = bound.get(property); + if (value !== undefined) return value; + value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + value = value.bind(target); + bound.set(property, value); + return value; + }, + // A test replacing a method (`storage.commit = …`) gets its own on the next read. + set(target, property, value) { + bound.delete(property); + return Reflect.set(target, property, value, target); + }, + }); +} diff --git a/packages/durable/test/store-storage.test.ts b/packages/durable/test/store-storage.test.ts new file mode 100644 index 000000000..cd1ca25ce --- /dev/null +++ b/packages/durable/test/store-storage.test.ts @@ -0,0 +1,611 @@ +// StoreStorage (slice 6b §2): Durable's storage conformance suite on SQLite `:memory:` and across reopen, then what is +// its own: claims and close, owners sharing a store, the store's semantics for a refused commit, the allocator. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { Context } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT, withTelemetry } from "@earendil-works/chord/context"; +import { + applyBatch, + type ChordStore, + createHost, + defineFacet, + defineState, + importStore, + int, + local, + method, + obj, + openStore, + type Store, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { describe, expect, it } from "vitest"; +import { StorageRequestError } from "../src/errors.ts"; +import { idFromNumber } from "../src/ids.ts"; +import { type CompactionResult, createRegistry, Harness, LiveDoc, type Tx } from "../src/index.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { + CommitResult, + ConversationId, + DocumentId, + EntryId, + Id, + Seq, + Storage, + StorageWrite, + TaskId, +} from "../src/types.ts"; +import { ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +const context = BACKGROUND_CONTEXT; +/** A storage that closes and reopens its store file after every commit (Durable's reopen harness). */ +class ReopeningStorage implements Storage { + #current: Storage; + readonly #path: string; + #closed = false; + + constructor(current: Storage, path: string) { + this.#current = current; + this.#path = path; + } + + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.#closed) return this.#current.commit(batches, receive, commitContext); + try { + return await this.#current.commit(batches, receive, commitContext); + } finally { + await this.#current.close(context); + this.#current = await openStoreStorage(this.#path); + } + } + + mintId>(): Promise { + return this.#current.mintId(); + } + conversation: Storage["conversation"] = (id, c) => this.#current.conversation(id, c); + scanConversations: Storage["scanConversations"] = (q, l, cursor, c) => + this.#current.scanConversations(q, l, cursor, c); + entry(id: EntryId, c: Context): ReturnType; + entry(conversationId: ConversationId, id: EntryId, c: Context): ReturnType; + entry(first: EntryId | ConversationId, second: EntryId | Context, third?: Context) { + if (third === undefined) return this.#current.entry(idFromNumber(first), second as Context); + return this.#current.entry(idFromNumber(first), second as EntryId, third); + } + findLatestHeadMarker: Storage["findLatestHeadMarker"] = (id, at, c) => this.#current.findLatestHeadMarker(id, at, c); + scanEntries: Storage["scanEntries"] = (q, l, cursor, c) => this.#current.scanEntries(q, l, cursor, c); + task: Storage["task"] = (id, c) => this.#current.task(id, c); + scanTasks: Storage["scanTasks"] = (q, l, cursor, c) => this.#current.scanTasks(q, l, cursor, c); + submission: Storage["submission"] = (id, c) => this.#current.submission(id, c); + scanSubmissions: Storage["scanSubmissions"] = (q, l, cursor, c) => this.#current.scanSubmissions(q, l, cursor, c); + submissionByRequest: Storage["submissionByRequest"] = (id, request, c) => + this.#current.submissionByRequest(id, request, c); + findDocument: Storage["findDocument"] = (address, at, c) => this.#current.findDocument(address, at, c); + document: Storage["document"] = (id, at, c) => this.#current.document(id, at, c); + scanDocuments: Storage["scanDocuments"] = (q, l, cursor, c) => this.#current.scanDocuments(q, l, cursor, c); + + async close(closeContext: Context): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#current.close(closeContext); + } +} + +registerStorageConformance({ describe, expect, it }, "StoreStorage", async (use) => { + const storage = await openStoreStorage(":memory:"); + try { + await use(storage); + } finally { + await storage.close(context); + } +}); + +registerStorageConformance({ describe, expect, it }, "StoreStorage across reopen", async (use) => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-conformance-")); + const path = join(directory, "store.sqlite"); + const storage = new ReopeningStorage(await openStoreStorage(path), path); + try { + await use(storage); + } finally { + await storage.close(context); + await rm(directory, { recursive: true, force: true }); + } +}); + +/** A store on `:memory:`. */ +async function memoryStore() { + return openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +} + +describe("claims and close", () => { + it("refuse a second StoreStorage on one owner until the first closed; two owners share a store", async () => { + const store = await memoryStore(); + const first = await StoreStorage.open(store, "harness", context); + await expect(StoreStorage.open(store, "harness", context)).rejects.toMatchObject({ + code: "invalid", + reason: "args", + }); + const other = await StoreStorage.open(store, "other", context); + await commitWrites(first, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + expect(await other.conversation(ROOT_CONVERSATION_ID, context)).toBeUndefined(); + await first.close(context); + await expect(first.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("StoreStorage is closed"); + const reopened = await StoreStorage.open(store, "harness", context); + expect(await reopened.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); + + it("waits at close for its calls underway, then releases its claims", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const commit = commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const closing = storage.close(context); + await commit; + await closing; + await StoreStorage.open(store, "harness", context); + await store.close(context); + }); +}); + +describe("the store's semantics (D4)", () => { + it("a refused Durable commit rejects with Durable's message and keeps the store usable", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const root = { type: "conversation", value: { id: ROOT_CONVERSATION_ID } } as const; + await commitWrites(storage, [root], context); + await expect(commitWrites(storage, [root], context)).rejects.toThrow("ID 1 already belongs to conversation"); + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + await commitWrites(storage, [{ type: "entry", value: entry }], context); + expect((await storage.entry(entry.id, context))?.entry).toEqual(entry); + await store.close(context); + }); + + it("raises the allocator above every id it holds when it opens (a store built by import)", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const entry = { id: idFromNumber(500), conversationId: ROOT_CONVERSATION_ID, kind: "m" }; + await commitWrites( + storage, + [ + { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, + { type: "entry", value: entry }, + ], + context, + ); + expect(await storage.mintId()).toBe(501); + await store.close(context); + }); +}); + +const MemoDoc = defineState("memo", { + schema: obj({ m: int() }), + version: 1, + initial: () => ({ m: 1 }), + persist: true, + checkpointWhen: () => true, +}); + +const Memo = local("app.memo", { version: 1, params: {}, members: { memo: obj({ m: int() }), set: method() } }); + +/** A store on `:memory:` with an outbox from its first commit, and Durable's storage in it. */ +async function recorded() { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, { + outbox: true, + })) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + return { store, storage }; +} + +/** Rewindable history, a retired latest document, a create-plus-retire and a change-plus-retire. */ +async function history(storage: Storage): Promise<{ readonly rewindable: DocumentId; readonly seqs: Seq[] }> { + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const rewindable = await storage.mintId(); + const latest = await storage.mintId(); + const empty = await storage.mintId(); + const seqs: Seq[] = []; + const base = (n: number) => ({ kind: "base", version: 1, value: { n } }) as const; + seqs.push( + await commitWrites( + storage, + [ + { + type: "document.create", + record: { id: rewindable, kind: "r", scope, history: "rewindable", fork: "asOf" }, + content: base(0), + }, + { + type: "document.create", + record: { id: latest, kind: "l", scope, history: "latest", fork: "current" }, + content: base(0), + }, + ], + context, + ), + ); + seqs.push( + await commitWrites( + storage, + [{ type: "document.change", id: rewindable, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + context, + ), + ); + seqs.push( + await commitWrites( + storage, + [ + { type: "document.change", id: latest, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }, + { type: "document.retire", id: latest }, + { type: "document.create", record: { id: empty, kind: "e", scope: { kind: "session" } }, content: base(3) }, + { type: "document.retire", id: empty }, + ], + context, + ), + ); + return { rewindable, seqs }; +} + +describe("archives and batches of Durable data", () => { + it("export and import keep rewindable history, retired records and Chord's own data: reads at a point agree", async () => { + const { store, storage } = await recorded(); + const { rewindable, seqs } = await history(storage); + // Chord's own data in the same store: an object's persisted document. + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Memo, (_, self) => { + const memo = self.state(MemoDoc); + return { + memo, + set: async (ctx) => { + await memo.change((draft) => { + draft.m = 2; + }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + await (await host.get(Memo, {}, context)).set(context); + await host.close(context); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "document.create").map(({ record }) => record.owner)).toContain( + "app.memo", + ); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + const imported = await StoreStorage.open(copy, "harness", context); + for (const at of [seqs[0]!, seqs[1]!, "current" as const]) { + expect(await imported.document(rewindable, at, context)).toEqual( + await storage.document(rewindable, at, context), + ); + } + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const at = seqs[1]!; + expect(await imported.scanDocuments({ scope, at }, 10, undefined, context)).toEqual( + await storage.scanDocuments({ scope, at }, 10, undefined, context), + ); + const again = []; + for await (const part of copy.export(context)) again.push(part); + expect(again).toEqual(parts); + expect(await imported.mintId()).toBe(await storage.mintId()); + await copy.close(context); + await store.close(context); + }); + + it("an outbox of Durable commits replayed into an empty store gives the same archive (create-plus-retire included)", async () => { + const { store, storage } = await recorded(); + await history(storage); + const commits = await (store as ChordStore).outbox!.read(0, 1_000, context); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, commits, context); + const exported = async (from: Store) => { + const parts = []; + for await (const part of from.export(context)) parts.push(part); + return parts; + }; + expect(await exported(replica)).toEqual(await exported(store)); + await replica.close(context); + await store.close(context); + }); +}); + +describe("head markers (the entry_heads table)", () => { + it("a head entry is found after reopen, in an imported archive and in a replayed outbox", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-heads-")); + const path = join(directory, "store.sqlite"); + const open = async () => + (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: true })) as ChordStore; + let store = await open(); + let storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const entry = async (head?: EntryId): Promise => { + const id = await storage.mintId(); + const value = { id, conversationId: ROOT_CONVERSATION_ID, kind: "k", data: {}, ...(head ? { head } : {}) }; + await commitWrites(storage, [{ type: "entry", value }], context); + return id; + }; + const first = await entry(); + const marker = await entry(first); + await entry(); + /** The newest head marker, and the one at or before the first entry (none). */ + const heads = async (of: Storage) => [ + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id, + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, first, context))?.id, + ]; + expect(await heads(storage)).toEqual([marker, undefined]); + await storage.close(context); + await store.close(context); + store = await open(); + storage = await StoreStorage.open(store, "harness", context); + expect(await heads(storage)).toEqual([marker, undefined]); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "put" && part.table === "harness:entry_heads")).toEqual([ + expect.objectContaining({ row: { id: marker, conversation_id: ROOT_CONVERSATION_ID } }), + ]); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + expect(await heads(await StoreStorage.open(copy, "harness", context))).toEqual([marker, undefined]); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, await store.outbox!.read(0, 1_000, context), context); + expect(await heads(await StoreStorage.open(replica, "harness", context))).toEqual([marker, undefined]); + for (const each of [copy, replica, store]) await each.close(context); + await rm(directory, { recursive: true, force: true }); + }); +}); + +describe("reads (slice 6b §1.4, §2.4)", () => { + it("a request error inside a multi-query read rejects that read only; the store goes on", async () => { + const { store, storage } = await recorded(); + await expect( + storage.scanEntries({ conversationId: idFromNumber(999) }, 10, undefined, context), + ).rejects.toBeInstanceOf(StorageRequestError); + await expect(storage.document(idFromNumber(999), "current", context)).resolves.toBeUndefined(); + expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); +}); + +describe("commits (slice 6b §1.3, §2.5)", () => { + it("publishes each commit of a Session inside the attached host's batch, and plainly without a host", async () => { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + const plain: boolean[] = []; + storage.publishing(() => plain.push(store.batch === undefined), context); + expect(plain).toEqual([true]); + const host = createHost({ store, onError: () => {} }); + const session = new SessionImpl(storage); + const running: boolean[] = []; + session.followCommits(() => { + running.push(store.batch!.running); + }); + await session.commitWith((tx) => tx.createRootConversation(), context); + await Promise.all([ + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "a" }), context), + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "b" }), context), + ]); + expect(running).toEqual([true, true, true]); + await session.close(context); + await host.close(context); + await store.close(context); + }); + + it("counts four statements for an entry append and for a delta, with a host attached", async () => { + const telemetry = new InMemoryTelemetryContext(); + const span = await telemetry.startSpan({ name: "test" }, (each) => each); + const metered = withTelemetry(span, context); + const counts = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.store.statements") + .reduce((total, { value }) => total + value, 0); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + const host = createHost({ store, onError: () => {} }); + const storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + const before = counts(); + await storage.commit([[{ type: "entry", value: entry }]], () => {}, metered); + const appended = counts() - before; + const id = await storage.mintId(); + const record = { id, kind: "s", scope: { kind: "session" } } as const; + await commitWrites( + storage, + [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 0 } } }], + context, + ); + const changed = counts(); + await commitWrites( + storage, + [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + metered, + ); + expect([appended, counts() - changed]).toEqual([4, 4]); + await host.close(context); + await store.close(context); + }); +}); + +describe("Durable's views as Chord sources on the store (slice 6b §3)", () => { + const Chat = local("app.chat", { + version: 1, + params: {}, + members: { entries: int(), note: method(obj({ kind: str() })) }, + }); + + it("a derive over the conversation view and the task graph: one frame per Durable commit, both views in one batch", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + let computes = 0; + const facet = defineFacet((h) => + h.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + const entries = self.derive([view, graph], ([v]) => { + computes++; + return v.entries.length; + }); + return { + entries, + note: async ({ kind }, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const chat = await host.get(Chat, {}, context); + const calls: unknown[] = []; + const watch = await chat.entries.watch(context); + watch.start(async (call) => void calls.push(call.kind === "value" ? call.value : call.kind)); + const before = computes; + await chat.note({ kind: "test.note" }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(chat.entries.value).toBeGreaterThan(0); + expect(calls).toEqual([chat.entries.value]); + expect(computes - before).toBe(1); + await host.close(context); + await store.close(context); + }); + + it("a deferred Durable commit, then an immediate one, reach the followed document in order", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const Live = local("app.live", { + version: 1, + params: {}, + members: { n: int(), bump: method() }, + }); + /** A commit setting the root's `LiveDoc` failure note to `value`. */ + const set = (value: number) => async (tx: Tx) => { + (await tx.doc(LiveDoc, ROOT_CONVERSATION_ID)).compactionFailure = { + taskId: idFromNumber>(1), + trigger: "manual", + error: String(value), + }; + }; + const facet = defineFacet((h) => + h.serve(Live, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Live.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await (await opened.root(open)).commit(set(0), open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const live = self.follow((observer, ctx) => + harness.value.observeDoc(LiveDoc, ROOT_CONVERSATION_ID, observer, ctx), + ); + return { + n: self.derive([live], ([value]) => Number(value.compactionFailure?.error ?? -1)), + bump: async (ctx) => { + const root = (await harness.value.conversation(ROOT_CONVERSATION_ID, ctx))!; + await root.commit(set(1), ctx, { persistence: { kind: "deferred", withinMs: 10_000 } }); + await root.commit(set(2), ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const live = await host.get(Live, {}, context); + const seen: unknown[] = []; + const watch = await live.n.watch(context); + watch.start(async (call) => void seen.push(call.kind === "value" ? call.value : call.kind)); + await live.bump(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(live.n.value).toBe(2); + expect(seen.at(-1)).toBe(2); + await host.close(context); + await store.close(context); + }); +}); + +describe("review regressions", () => { + it('a head-marker entry whose kind is "refused" is returned, not taken for a request error', async () => { + const storage = await openStoreStorage(":memory:"); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const first = await storage.mintId(); + const marker = await storage.mintId(); + await commitWrites( + storage, + [ + { type: "entry", value: { id: first, conversationId: ROOT_CONVERSATION_ID, kind: "message" } }, + { + type: "entry", + value: { id: marker, conversationId: ROOT_CONVERSATION_ID, kind: "refused", head: first }, + }, + ], + context, + ); + expect((await storage.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id).toBe(marker); + const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); + expect(page.items.map(({ kind }) => kind)).toEqual(["refused", "message"]); + await storage.close(context); + }); +}); + +describe("GC", () => { + it("a closed StoreStorage is collected while its store stays open", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const store = await memoryStore(); + const opened = async (): Promise> => { + const storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + await storage.close(context); + return new WeakRef(storage); + }; + const ref = await opened(); + for (let round = 0; round < 10 && ref.deref() !== undefined; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(ref.deref()).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/durable/test/system-order-cache-e2e.test.ts b/packages/durable/test/system-order-cache-e2e.test.ts index 0d67629d5..f6581ac4a 100644 --- a/packages/durable/test/system-order-cache-e2e.test.ts +++ b/packages/durable/test/system-order-cache-e2e.test.ts @@ -4,7 +4,6 @@ import { type Model, type Models, normalizeContext, - Type, uuidv7, } from "@earendil-works/pi-ai"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; @@ -18,6 +17,7 @@ import { hook, MemoryStorage, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { streamSimple as streamSimpleAnthropic } from "../../ai/src/api/anthropic-messages.ts"; import { resolveApiKey } from "../../ai/test/oauth.ts"; @@ -41,7 +41,7 @@ function instructions(nonce: string): string { return lines.join("\n"); } -const Parameters = Type.Object({ text: Type.String() }); +const Parameters = obj({ text: str() }); const first = defineTool({ name: "probe_first", description: "Unused probe tool.", diff --git a/packages/durable/test/task-support.ts b/packages/durable/test/task-support.ts index a94373050..fc0258955 100644 --- a/packages/durable/test/task-support.ts +++ b/packages/durable/test/task-support.ts @@ -71,6 +71,7 @@ export async function openTasks( readonly settings?: HarnessSettings; readonly onWake?: HarnessOptions["onWake"]; readonly setTimer?: HarnessOptions["setTimer"]; + readonly telemetry?: HarnessOptions["telemetry"]; } = {}, ): Promise<{ readonly harness: Harness; @@ -87,7 +88,7 @@ export async function openTasks( { models: createModels(), registry, - onReport: (error, info) => { + onReport: ({ error, ...info }) => { reports.push(error); infos.push(info); }, @@ -95,6 +96,7 @@ export async function openTasks( ...(options.settings === undefined ? {} : { settings: options.settings }), ...(options.onWake === undefined ? {} : { onWake: options.onWake }), ...(options.setTimer === undefined ? {} : { setTimer: options.setTimer }), + ...(options.telemetry === undefined ? {} : { telemetry: options.telemetry }), }, context, ); diff --git a/packages/durable/test/tool-args-typebox.test.ts b/packages/durable/test/tool-args-typebox.test.ts new file mode 100644 index 000000000..7a606cf96 --- /dev/null +++ b/packages/durable/test/tool-args-typebox.test.ts @@ -0,0 +1,61 @@ +// Tool arguments typed from a TypeBox-built parameter schema. Durable does not depend on TypeBox: the schema below +// has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { defineTool, type ToolRegistration } from "@earendil-works/pi-durable"; +import { describe, expectTypeOf, it } from "vitest"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TInteger { + "~kind": "Integer"; + type: "integer"; +} + +interface TArray { + "~kind": "Array"; + type: "array"; + items: Items; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ path: TString; offset: TOptional; tags: TArray }> = { + "~kind": "Object", + type: "object", + properties: { + path: { "~kind": "String", type: "string" }, + offset: { "~kind": "Integer", type: "integer", "~optional": true }, + tags: { "~kind": "Array", type: "array", items: { "~kind": "String", type: "string" } }, + }, + required: ["path", "tags"], +}; + +type Args = { path: string; offset?: number; tags: string[] }; + +describe("tool arguments of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool = defineTool({ + name: "read", + description: "Reads a file", + parameters, + async execute(args) { + expectTypeOf(args).toEqualTypeOf(); + return {}; + }, + }); + expectTypeOf[0]>().toEqualTypeOf(); + expectTypeOf< + ReturnType["prepareArguments"]>> + >().toEqualTypeOf(); + }); +}); diff --git a/packages/durable/test/tool-output-bench.ts b/packages/durable/test/tool-output-bench.ts index 040c3c7b4..ac7a54d29 100644 --- a/packages/durable/test/tool-output-bench.ts +++ b/packages/durable/test/tool-output-bench.ts @@ -7,6 +7,7 @@ * then close the Harness mid-round (persistent backends) to measure replay of the round's delta chain on reopen, and * finish the run to measure the stored size after the round's base and reclamation. Throughput scenarios push 1 GiB. */ + import { execFile } from "node:child_process"; import { mkdtemp, open, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -16,7 +17,7 @@ import { promisify } from "node:util"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createRegistry, defineTool, Harness, LiveDoc, MemoryStorage, type Storage } from "../src/index.ts"; import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; @@ -145,7 +146,7 @@ async function runScenario(scenario: Scenario): Promise { defineTool({ name: "emit", description: "", - parameters: Type.Object({}), + parameters: obj({}), outputLimits: { retain: scenario.retain }, execute: async (_args, api, callContext) => { let counter = 0; @@ -195,13 +196,17 @@ async function runScenario(scenario: Scenario): Promise { let liveOpBytes = 0; let outputSets = 0; const observe = (target: Harness): void => { - target.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "document" || change.record.kind !== "pi.live") continue; - liveCommits++; - liveOpBytes += JSON.stringify(change.ops).length; - for (const op of change.ops) if (op[0] === "s" && op[1].at(-1) === "output") outputSets++; - } + target.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type !== "document" || change.kind !== "change" || change.record.kind !== "pi.live") + continue; + liveCommits++; + liveOpBytes += JSON.stringify(change.ops).length; + for (const op of change.ops) if (op[0] === "s" && op[1].at(-1) === "output") outputSets++; + } + }, + end: () => {}, }); }; observe(harness); @@ -257,7 +262,7 @@ async function openStorage(backend: Backend, path: string): Promise { return openNodeJsonlStorage(path, context); } -/** Measure every Storage commit. */ +/** Measure every Storage commit call. */ function timed(storage: Storage, latencies: number[]): Storage { return new Proxy(storage, { get(target, property) { diff --git a/packages/durable/test/tool-search.test.ts b/packages/durable/test/tool-search.test.ts index 895240bbb..988c85d5a 100644 --- a/packages/durable/test/tool-search.test.ts +++ b/packages/durable/test/tool-search.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -9,6 +9,7 @@ import { MemoryStorage, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { num, obj, opt } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createToolSearchExtension } from "../src/tool-search/index.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; @@ -19,7 +20,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -120,7 +121,7 @@ describe("deferred tools and tool_search", () => { const grow = defineTool({ name: "grow", description: "grow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [], control: { addTools: ["github_get_issue", "alpha", "ghost"] } }), }); addTool(chat.registry, grow); diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts index e8486be10..b1e30fe4c 100644 --- a/packages/durable/test/tools-read-differential.test.ts +++ b/packages/durable/test/tools-read-differential.test.ts @@ -17,8 +17,10 @@ import { DEFAULT_MAX_BYTES, formatSize, truncateHead } from "../src/truncate.ts" function referenceRead(bytes: Uint8Array, path: string, offset: number | undefined, limit: number | undefined) { const mimeType = detectSupportedImageMimeType(bytes); if (mimeType) { + const block = { type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }; return { - output: [{ type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }], + output: [block], + structuredOutput: block, diagnostics: [{ severity: "info", code: "image", message: `Read image file [${mimeType}].` }], }; } @@ -73,6 +75,7 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin } return { output: outputText === "" ? [] : [{ type: "text", text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/durable/vitest.store.config.ts b/packages/durable/vitest.store.config.ts new file mode 100644 index 000000000..b98674172 --- /dev/null +++ b/packages/durable/vitest.store.config.ts @@ -0,0 +1,47 @@ +// The harness test files that take a SQLite storage, run on Durable's storage on Chord's store (`StoreStorage`, slice +// 6b): `../src/storage/sqlite/node.ts`'s opener is replaced by `test/store-storage-node.ts`; the files are unchanged. +// Four tests edit or read `SqliteStorage`'s own tables with raw SQL (`tasks`, `durable_metadata`) and are left out. + +import { fileURLToPath } from "node:url"; +import { defineConfig, mergeConfig } from "vitest/config"; +import base from "./vitest.config.ts"; + +const storeOpener = fileURLToPath(new URL("./test/store-storage-node.ts", import.meta.url)); + +export default mergeConfig( + base, + defineConfig({ + test: { + testNamePattern: + /^(?!.*(resumes a retry backoff after reopen|sleeps again for a retry stored as a running phase|polls a checkpoint stored without a thinking level|refuses to create the root of an empty storage))/, + include: [ + "test/harness-after-run.test.ts", + "test/harness-blocked.test.ts", + "test/harness-compaction.test.ts", + "test/harness-continue.test.ts", + "test/harness-conversations.test.ts", + "test/harness-generation-recovery.test.ts", + "test/harness-inbox.test.ts", + "test/harness-input-entries.test.ts", + "test/harness-lifecycle.test.ts", + "test/harness-nested-tools-restart.test.ts", + "test/harness-nested-tools.test.ts", + "test/harness-ownership.test.ts", + "test/harness-read-only.test.ts", + "test/harness-restart-failure.test.ts", + "test/harness-scheduler-indexes.test.ts", + "test/harness-scheduler-retention.test.ts", + "test/harness-sleep.test.ts", + "test/harness-structured.test.ts", + "test/harness-submissions.test.ts", + "test/harness-task-graph.test.ts", + "test/harness-tasks-recovery.test.ts", + "test/harness-tool-hooks.test.ts", + "test/harness-tools-recovery.test.ts", + "test/harness-turn-limit.test.ts", + "test/harness-view.test.ts", + ], + }, + resolve: { alias: [{ find: /^\.\.\/src\/storage\/sqlite\/node\.ts$/, replacement: storeOpener }] }, + }), +); diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 4d2a1c0e1..6c101bdd4 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -2,6 +2,17 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Breaking Changes + +- `RemoteWatcher` follows Durable's `FileWatcher`: `close(context)` is `stop(context)`, and its changes are tagged (`{ kind: "paths", paths }`, `{ kind: "overflow" }`, `{ kind: "failed", error }`). The daemon's `watch` protocol is unchanged. Migration: `await watcher.close(ctx)` becomes `await watcher.stop(ctx)`; test `change.kind` instead of `"overflow" in change` or `"error" in change`. + +### Fixed + +- `exec` and `pty` fail with `spawn_error` instead of hanging when the daemon cannot start a helper thread (for example at a cgroup task limit); the started command is killed and reaped ([#11433](https://github.com/earendil-works/pi/issues/11433)). +- `Connection.close()` on Unix no longer leaves running commands behind: the daemon used to die from the client's `SIGTERM` before killing its process groups. It now kills them on `SIGTERM`, `SIGINT` or `SIGHUP` before exiting. + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/env/daemon/src/exec.rs b/packages/env/daemon/src/exec.rs index 1d8b03845..d92c51279 100644 --- a/packages/env/daemon/src/exec.rs +++ b/packages/env/daemon/src/exec.rs @@ -217,8 +217,8 @@ fn spawn_reader( sender: Sender, stop: Arc, output: Option>, -) -> JoinHandle<()> { - thread::spawn(move || { +) -> std::io::Result> { + thread::Builder::new().spawn(move || { sys::read_pipe(pipe, &stop, |bytes| { let delivered = sender.send(Message::Data(stream, bytes.to_vec())).is_ok(); if let Some(output) = &output { @@ -298,29 +298,46 @@ pub fn run( } else { None }; - let readers = [ - spawn_reader( - 0, - child.stdout.take().unwrap(), - sender.clone(), - stop.clone(), - gate.clone(), - ), - spawn_reader( - 1, - child.stderr.take().unwrap(), - sender.clone(), - stop.clone(), - gate, - ), - ]; + // A thread that cannot start (a task limit) fails the command; the started child is killed and reaped. + let abandon = |mut child: Child, readers: &[JoinHandle<()>], error: std::io::Error| { + stop.store(true, Ordering::SeqCst); + for reader in readers { + sys::interrupt_reader(reader); + } + kill_group(pid); + // On Windows the group kill spawns taskkill, which can fail under the same limit; kill the child directly. + let _ = child.kill(); + let _ = child.wait(); + groups.lock().unwrap().remove(&pid); + Failure::new("spawn_error", error.to_string()) + }; + let mut readers = Vec::with_capacity(2); + let stdout = child.stdout.take().unwrap(); + match spawn_reader(0, stdout, sender.clone(), stop.clone(), gate.clone()) { + Ok(reader) => readers.push(reader), + Err(error) => return Err(abandon(child, &readers, error)), + } + let stderr = child.stderr.take().unwrap(); + match spawn_reader(1, stderr, sender.clone(), stop.clone(), gate) { + Ok(reader) => readers.push(reader), + Err(error) => return Err(abandon(child, &readers, error)), + } + // The waiter receives the child once it runs, so a waiter that cannot start leaves the child here to reap. + let (give, take) = mpsc::channel::(); let waiter = sender.clone(); - thread::spawn(move || { + let started = thread::Builder::new().spawn(move || { + let Ok(mut child) = take.recv() else { + return; + }; let _ = waiter.send(match child.wait() { Ok(status) => Message::Exit(status), Err(error) => Message::WaitFailed(error.to_string()), }); }); + if let Err(error) = started { + return Err(abandon(child, &readers, error)); + } + let _ = give.send(child); drop(sender); let mut decoders = [StreamDecoder::new(), StreamDecoder::new()]; diff --git a/packages/env/daemon/src/main.rs b/packages/env/daemon/src/main.rs index 4457fef4d..ea720eabb 100644 --- a/packages/env/daemon/src/main.rs +++ b/packages/env/daemon/src/main.rs @@ -470,6 +470,15 @@ fn serve_frames(server: &Arc, input: &mut impl Read) -> io::Result<()> { } fn serve(token: &str) -> io::Result<()> { + let groups: exec::Groups = Arc::new(Mutex::new(Default::default())); + { + // A client that stops the daemon (Node's `child.kill()`), or a hangup, leaves nothing running either. + let groups = groups.clone(); + sys::on_exit_signal(move |signal| { + kill_all(&groups); + std::process::exit(128 + signal); + }); + } { let mut stdout = io::stdout().lock(); writeln!(stdout, "PI-ENV {token}")?; @@ -488,7 +497,7 @@ fn serve(token: &str) -> io::Result<()> { handles: Mutex::new(HashMap::new()), next_handle: AtomicU64::new(1), controls: Mutex::new(HashMap::new()), - groups: Arc::new(Mutex::new(Default::default())), + groups, terminals: Arc::new(Mutex::new(HashMap::new())), }); let last_seen = Arc::new(AtomicU64::new(now_ms())); diff --git a/packages/env/daemon/src/pty.rs b/packages/env/daemon/src/pty.rs index 00cc566aa..e8be9e222 100644 --- a/packages/env/daemon/src/pty.rs +++ b/packages/env/daemon/src/pty.rs @@ -182,7 +182,7 @@ pub fn run( format!("Working directory does not exist: {}", request.cwd), )); } - let (master, mut child) = + let (master, child) = unix::spawn(&request).map_err(|error| Failure::new("spawn_error", error.to_string()))?; let master = Arc::new(master); let pid = child.id(); @@ -203,12 +203,21 @@ pub fn run( )); let stop = Arc::new(AtomicBool::new(false)); - let reader = { + // A thread that cannot start (a task limit) fails the terminal; the started program is killed and reaped. + let abandon = |mut child: std::process::Child, error: std::io::Error| { + stop.store(true, Ordering::SeqCst); + kill_group(pid); + let _ = child.wait(); + terminals.lock().unwrap().remove(&id); + groups.lock().unwrap().remove(&pid); + Failure::new("spawn_error", error.to_string()) + }; + let started = { let master = master.clone(); let sender = sender.clone(); let stop = stop.clone(); let output = output.clone(); - thread::spawn(move || { + thread::Builder::new().spawn(move || { unix::read(&master, &stop, |bytes| { let delivered = sender.send(Message::Data(0, bytes.to_vec())).is_ok(); // Like a full pipe: a program that writes faster than the link waits. @@ -218,24 +227,40 @@ pub fn run( let _ = sender.send(Message::Eof(0)); }) }; - { + let reader = match started { + Ok(reader) => reader, + Err(error) => return Err(abandon(child, error)), + }; + let started = { let master = master.clone(); let stop = stop.clone(); - thread::spawn(move || { + thread::Builder::new().spawn(move || { for bytes in inputs { if !unix::write_all(&master, &bytes, &stop) { return; } } - }); + }) + }; + if let Err(error) = started { + return Err(abandon(child, error)); } + // The waiter receives the child once it runs, so a waiter that cannot start leaves the child here to reap. + let (give, take) = mpsc::channel::(); let waiter = sender.clone(); - thread::spawn(move || { + let started = thread::Builder::new().spawn(move || { + let Ok(mut child) = take.recv() else { + return; + }; let _ = waiter.send(match child.wait() { Ok(status) => Message::Exit(status), Err(error) => Message::WaitFailed(error.to_string()), }); }); + if let Err(error) = started { + return Err(abandon(child, error)); + } + let _ = give.send(child); drop(sender); let mut status = None; diff --git a/packages/env/daemon/src/sys/unix.rs b/packages/env/daemon/src/sys/unix.rs index 764cad1d1..4b0637f0e 100644 --- a/packages/env/daemon/src/sys/unix.rs +++ b/packages/env/daemon/src/sys/unix.rs @@ -283,6 +283,29 @@ pub fn configure( pub fn after_spawn(_child: &Child) {} +/// Run `on_signal` with the signal on its own thread when `SIGTERM`, `SIGINT` or `SIGHUP` arrives, instead of dying +/// at once. Call before starting any other thread: threads inherit the blocked signals, so only this one receives +/// them. Commands reset their signal mask before they run. +pub fn on_exit_signal(on_signal: impl FnOnce(i32) + Send + 'static) { + // SAFETY: a zeroed sigset_t initialized by sigemptyset, and plain signal mask calls. + let set = unsafe { + let mut set: libc::sigset_t = std::mem::zeroed(); + libc::sigemptyset(&mut set); + for signal in [libc::SIGTERM, libc::SIGINT, libc::SIGHUP] { + libc::sigaddset(&mut set, signal); + } + libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut()); + set + }; + std::thread::spawn(move || { + let mut signal = 0; + // SAFETY: waits for one of the signals blocked above. + if unsafe { libc::sigwait(&set, &mut signal) } == 0 { + on_signal(signal); + } + }); +} + /// Kill the command's process group, or the process if the group is gone. pub fn kill_tree(pid: u32) { let pid = pid as i32; diff --git a/packages/env/daemon/src/sys/windows.rs b/packages/env/daemon/src/sys/windows.rs index f43a4bacf..401719369 100644 --- a/packages/env/daemon/src/sys/windows.rs +++ b/packages/env/daemon/src/sys/windows.rs @@ -550,6 +550,10 @@ pub fn after_spawn(child: &Child) { } } +/// Nothing to do: the daemon cannot catch `TerminateProcess`. When it ends, the job object kills each command's own +/// process; processes those start leave the job (silent breakaway, as in libuv) and keep running. +pub fn on_exit_signal(_on_signal: impl FnOnce(i32) + Send + 'static) {} + /// Node's `killProcessTree` on Windows: `taskkill /F /T`, not waited for. pub fn kill_tree(pid: u32) { let system_root = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".into()); diff --git a/packages/env/daemon/src/watch.rs b/packages/env/daemon/src/watch.rs index 18d202b82..b53c83d4b 100644 --- a/packages/env/daemon/src/watch.rs +++ b/packages/env/daemon/src/watch.rs @@ -4,7 +4,7 @@ use crate::errors::Failure; use crate::exec::Control; -use crate::frame::{EVENT, Frame}; +use crate::frame::{EVENT, Frame, MAX_PAYLOAD}; use crate::output::Output; use crate::sys; use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; @@ -662,7 +662,7 @@ impl FileWatcher<'_> { if !changed.is_empty() && !self.cancelled() { let mut paths: Vec = changed.into_iter().collect(); sort_like_javascript(&mut paths); - self.send(json!({ "kind": "change", "paths": paths })); + self.send(change_event(paths)); } true } @@ -675,6 +675,16 @@ impl FileWatcher<'_> { } } +/// A `change` event listing `paths`, or an `overflow` one when the list would not fit in a frame: a frame that is too +/// large would end the connection's output. +fn change_event(paths: Vec) -> Value { + let event = json!({ "kind": "change", "paths": paths }); + if serde_json::to_vec(&event).map_or(true, |json| json.len() > MAX_PAYLOAD) { + return json!({ "kind": "change", "overflow": true }); + } + event +} + fn parse_targets(json: &Value) -> Result, Failure> { let invalid = || Failure::new("EINVAL", "watch needs targets"); json["targets"] @@ -776,3 +786,22 @@ pub fn run(id: u32, json: &Value, output: &Output, control: &Control) -> Result< } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn change_event_lists_paths_that_fit() { + let event = change_event(vec!["/a".to_string(), "/b".to_string()]); + assert_eq!(event, json!({ "kind": "change", "paths": ["/a", "/b"] })); + } + + #[test] + fn change_event_overflows_when_paths_do_not_fit_in_a_frame() { + let path = "x".repeat(4096); + let count = MAX_PAYLOAD / path.len() + 1; + let event = change_event(vec![path; count]); + assert_eq!(event, json!({ "kind": "change", "overflow": true })); + } +} diff --git a/packages/env/docs/protocol.md b/packages/env/docs/protocol.md index ae43fb767..c5e482856 100644 --- a/packages/env/docs/protocol.md +++ b/packages/env/docs/protocol.md @@ -62,9 +62,11 @@ command as a full pipe would. ## Liveness Each side sends `ping` every 5 seconds. The daemon kills every process group it started and exits after 30 seconds -without receiving any bytes, or when stdin ends. A dropped connection therefore stops remote commands. The client -fails requests of a lost connection with `{ code: "unknown", lost: true }` and starts a new daemon on the next request; -handles belong to the daemon that opened them, so requests on them fail instead of reaching the new one. +without receiving any bytes, when stdin ends, or on `SIGTERM`, `SIGINT` or `SIGHUP`. A dropped connection or a +stopped daemon therefore stops remote commands. On Windows, a terminated daemon cannot clean up: a job object kills +each command's own process, but not processes that one started. The client fails requests of a lost connection with +`{ code: "unknown", lost: true }` and starts a new daemon on the next request; handles belong to the daemon that opened +them, so requests on them fail instead of reaching the new one. ## Operations diff --git a/packages/env/package.json b/packages/env/package.json index ab09e6546..f78d49c20 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "description": "Remote execution environments for Pi Durable: a small daemon deployed over SSH and its ExecutionEnv client", "type": "module", "main": "./dist/index.js", @@ -22,7 +22,7 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:daemon": "cargo build --manifest-path daemon/Cargo.toml --locked", "test": "vitest --run", @@ -45,11 +45,10 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/env/src/connection.ts b/packages/env/src/connection.ts index 484e7662c..527d90b19 100644 --- a/packages/env/src/connection.ts +++ b/packages/env/src/connection.ts @@ -1,5 +1,6 @@ import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; +import { stripVTControlCharacters } from "node:util"; /** Frame types (docs/protocol.md). */ const REQUEST = 1; @@ -15,6 +16,9 @@ const PING_INTERVAL_MS = 5000; const SILENCE_LIMIT_MS = 30_000; /** How long starting the daemon and its `hello` may take. */ const START_TIMEOUT_MS = 60_000; +/** How much stderr a startup error shows: short, because a tool error can reach the model. */ +const STDERR_TAIL_CHARS = 2000; +const STDERR_TAIL_LINES = 20; export type Json = Record; @@ -131,6 +135,24 @@ function closed(): RemoteError { return new RemoteError({ code: "unknown", message: "Connection closed" }); } +/** + * `message` plus the last lines of `stderr`. The remote machine wrote that text, so terminal escapes and control + * characters are removed. + */ +function withStderr(message: string, stderr: string): string { + let text = ""; + for (const char of stripVTControlCharacters(stderr.replace(/\r\n?/g, "\n"))) { + const code = char.charCodeAt(0); + if (char === "\n" || char === "\t" || (code >= 0x20 && (code < 0x7f || code > 0x9f))) text += char; + } + const lines = text + .split("\n") + .map((line) => line.trimEnd()) + .filter((line) => line !== ""); + if (lines.length === 0) return message; + return `${message}\nremote stderr:\n${lines.slice(-STDERR_TAIL_LINES).join("\n")}`; +} + /** One running daemon. */ class Session { readonly id: number; @@ -144,6 +166,7 @@ class Session { lastSeen = Date.now(); timer: ReturnType | undefined; live = true; + stderrTail = ""; constructor(id: number, child: ChildProcessWithoutNullStreams, token: string) { this.id = id; @@ -281,6 +304,7 @@ export class Connection { const child = spawn(program, [...args, "serve", "--token", token], { stdio: ["pipe", "pipe", "pipe"] }); const session = new Session(++this.#sessions, child, token); this.#session = session; + let ready = false; let startTimer: ReturnType | undefined; const failed = new Promise((_resolve, reject) => { startTimer = setTimeout( @@ -290,13 +314,20 @@ export class Connection { child.once("error", (error) => reject(new RemoteError({ code: "spawn_error", message: error.message, lost: true })), ); - child.once("exit", (code) => reject(lost(`pi-env exited with code ${code} before it was ready`))); + child.once("close", (code) => reject(lost(`pi-env exited with code ${code} before it was ready`))); }); failed.catch(() => {}); child.stdout.on("data", (chunk: Buffer) => this.#onData(session, chunk)); - child.stderr.on("data", (chunk: Buffer) => this.#options.onLog?.(chunk.toString("utf8"))); + child.stderr.setEncoding("utf8").on("data", (text: string) => { + if (!ready) session.stderrTail = (session.stderrTail + text).slice(-STDERR_TAIL_CHARS); + this.#options.onLog?.(text); + }); child.on("error", () => this.#teardown(session, lost("pi-env connection lost"))); - child.on("exit", () => this.#teardown(session, lost("pi-env connection lost"))); + // A ready session fails as soon as the daemon exits, even while a process it started keeps a pipe open. + child.on("exit", () => { + if (ready) this.#teardown(session, lost("pi-env connection lost")); + }); + child.on("close", () => this.#teardown(session, lost("pi-env connection lost"))); child.stdin.on("error", () => {}); session.timer = setInterval(() => { this.#write(session, frame(PING, 0, {})); @@ -308,10 +339,19 @@ export class Connection { try { const { json } = await Promise.race([this.#send(session, "hello", { protocol: 1 }, {}), failed]); if (json.protocol !== 1) throw lost(`Unsupported protocol ${json.protocol}`); + ready = true; + session.stderrTail = ""; return { info: json as unknown as RemoteInfo, session }; } catch (error) { - this.#teardown(session, error instanceof Error ? error : lost(String(error))); - throw error; + // Add the captured stderr to whatever ended the start; only a pre-ready exit waits for it to drain. + const failure = + error instanceof RemoteError + ? new RemoteError({ ...error.fields, message: withStderr(error.message, session.stderrTail) }) + : error instanceof Error + ? error + : lost(String(error)); + this.#teardown(session, failure); + throw failure; } finally { clearTimeout(startTimer); } diff --git a/packages/env/src/remote-env.ts b/packages/env/src/remote-env.ts index c77ce201e..c2656f747 100644 --- a/packages/env/src/remote-env.ts +++ b/packages/env/src/remote-env.ts @@ -9,6 +9,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -25,7 +26,6 @@ import { StreamDecoder, type TextLine, type TextLineReader, - type WatchChange, type WatchTarget, } from "@earendil-works/pi-durable/env"; import { type Connection, type Json, nodePlatform, RemoteError, type RemoteInfo } from "./connection.ts"; @@ -826,7 +826,7 @@ export class RemoteExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); @@ -840,7 +840,7 @@ export class RemoteExecutionEnv implements ExecutionEnv { const watcher = await RemoteWatcher.open(this.connection, resolved, onChange, this.#watchOptions); const afterOpen = abortResult(context.abortSignal); if (afterOpen) { - await watcher.close(context); + await watcher.stop(context); return afterOpen; } return ok(watcher); diff --git a/packages/env/src/watch.ts b/packages/env/src/watch.ts index 402869891..b80d41e2c 100644 --- a/packages/env/src/watch.ts +++ b/packages/env/src/watch.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "@earendil-works/pi-durable/env"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "@earendil-works/pi-durable/env"; import { type Connection, isConnectionLost } from "./connection.ts"; import { toFileError } from "./errors.ts"; @@ -13,6 +13,9 @@ export interface RemoteWatchOptions { maxDirectories?: number; } +/** The one `overflow` change; it carries nothing else. */ +const OVERFLOW: FileChange = { kind: "overflow" }; + const RECONNECT_FIRST_MS = 1000; const RECONNECT_MAX_MS = 30_000; @@ -23,10 +26,10 @@ const RECONNECT_MAX_MS = 30_000; export class RemoteWatcher implements FileWatcher { readonly #connection: Connection; readonly #targets: readonly WatchTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #options: RemoteWatchOptions; #mode: "native" | "polling" = "native"; - #closed = false; + #stopped = false; #controller: AbortController | undefined; #request: Promise | undefined; #wake: (() => void) | undefined; @@ -34,7 +37,7 @@ export class RemoteWatcher implements FileWatcher { private constructor( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions, ) { this.#connection = connection; @@ -47,7 +50,7 @@ export class RemoteWatcher implements FileWatcher { static async open( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions = {}, ): Promise { const watcher = new RemoteWatcher(connection, targets, onChange, options); @@ -59,16 +62,16 @@ export class RemoteWatcher implements FileWatcher { return this.#mode; } - async close(_context: Context): Promise { - if (this.#closed) return; - this.#closed = true; + async stop(_context: Context): Promise { + if (this.#stopped) return; + this.#stopped = true; this.#wake?.(); this.#controller?.abort(); await this.#request; } - #deliver(change: WatchChange): void { - if (this.#closed) return; + #deliver(change: FileChange): void { + if (this.#stopped) return; try { this.#onChange(change); } catch { @@ -101,12 +104,13 @@ export class RemoteWatcher implements FileWatcher { resolve(); } else if (event.kind === "change") { if (event.mode === "polling") this.#mode = "polling"; - if (event.overflow === true) this.#deliver({ overflow: true }); - else this.#deliver({ paths: event.paths as string[] }); + // The daemon's wire format: `{ overflow: true }` or `{ paths }`. + if (event.overflow === true) this.#deliver(OVERFLOW); + else this.#deliver({ kind: "paths", paths: event.paths as string[] }); } else if (event.kind === "error") { const code = event.code === "permission_denied" ? "permission_denied" : "invalid"; - this.#deliver({ error: new FileError(code, String(event.message)) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: new FileError(code, String(event.message)) }); + this.#stopped = true; } }, }, @@ -118,20 +122,20 @@ export class RemoteWatcher implements FileWatcher { reject(error); return; } - if (this.#closed) return; + if (this.#stopped) return; if (isConnectionLost(error)) { void this.#reconnect(); return; } - this.#deliver({ error: toFileError(error) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: toFileError(error) }); + this.#stopped = true; }, ); }); } async #reconnect(): Promise { - for (let delay = RECONNECT_FIRST_MS; !this.#closed; delay = Math.min(delay * 2, RECONNECT_MAX_MS)) { + for (let delay = RECONNECT_FIRST_MS; !this.#stopped; delay = Math.min(delay * 2, RECONNECT_MAX_MS)) { await new Promise((resolve) => { const timer = setTimeout(resolve, delay); timer.unref(); @@ -141,16 +145,16 @@ export class RemoteWatcher implements FileWatcher { }; }); this.#wake = undefined; - if (this.#closed) return; + if (this.#stopped) return; try { await this.#start(); // Changes while disconnected were not seen. - this.#deliver({ overflow: true }); + this.#deliver(OVERFLOW); return; } catch (error) { if (isConnectionLost(error)) continue; - this.#deliver({ error: toFileError(error) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: toFileError(error) }); + this.#stopped = true; return; } } diff --git a/packages/env/test/connection.test.ts b/packages/env/test/connection.test.ts new file mode 100644 index 000000000..8b6eb35d6 --- /dev/null +++ b/packages/env/test/connection.test.ts @@ -0,0 +1,223 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { Connection, isConnectionLost, RemoteError } from "../src/connection.ts"; + +const connections: Connection[] = []; +afterAll(() => { + for (const connection of connections) connection.close(); +}); + +/** A connection whose "daemon" is a Node script; the script receives `serve --token ` as its arguments. */ +function connect(script: string | (() => string), onLog?: (text: string) => void): Connection { + const connection = new Connection({ + command: async () => [process.execPath, "-e", typeof script === "string" ? script : script()], + ...(onLog === undefined ? {} : { onLog }), + }); + connections.push(connection); + return connection; +} + +/** A script that writes `stderr` and exits with `code` once the write is flushed. */ +function failing(stderr: string, code = 1): string { + return `process.stderr.write(${JSON.stringify(stderr)}, () => process.exit(${code}));`; +} + +async function startError(connection: Connection): Promise { + try { + await connection.info(); + } catch (error) { + if (error instanceof Error) return error; + throw error; + } + throw new Error("The start did not fail"); +} + +/** + * Speaks the daemon's sync line and framing. `hello` gets `helloReply` (a result, or an error frame). The next request + * makes it exit while a process it started keeps stdout and stderr open for a few seconds. + */ +function fakeDaemon(helloReply: "result" | "error"): string { + return ` +const { spawn } = require("node:child_process"); +const token = process.argv[process.argv.indexOf("--token") + 1]; +process.stdout.write("PI-ENV " + token + "\\n"); +function frame(type, id, json) { + const body = Buffer.from(JSON.stringify(json)); + const header = Buffer.alloc(13); + header.writeUInt32BE(9 + body.length, 0); + header.writeUInt8(type, 4); + header.writeUInt32BE(id, 5); + header.writeUInt32BE(body.length, 9); + return Buffer.concat([header, body]); +} +let buffered = Buffer.alloc(0); +let requests = 0; +process.stdin.on("data", (chunk) => { + buffered = Buffer.concat([buffered, chunk]); + while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE(0)) { + const length = buffered.readUInt32BE(0); + const type = buffered.readUInt8(4); + const id = buffered.readUInt32BE(5); + buffered = buffered.subarray(4 + length); + if (type !== 1) continue; + if (++requests === 1) { + const info = { protocol: 1, version: "test", os: "linux", arch: "x86_64", home: "/", tmpdir: "/tmp", + separator: "/", cwd: "/", driveCwds: {}, pid: process.pid }; + const refusal = { code: "EINVAL", message: "hello refused", path: "/daemon", extra: 1 }; + process.stdout.write(${JSON.stringify(helloReply)} === "result" ? frame(2, id, info) : frame(3, id, refusal)); + continue; + } + spawn(process.execPath, ["-e", "setTimeout(() => {}, 3000)"], { stdio: ["ignore", "inherit", "inherit"] }).unref(); + process.exit(0); + } +}); +`; +} + +describe("Connection start failures", () => { + it("includes the end of stderr when the daemon exits before it is ready", async () => { + const error = await startError(connect(failing("ssh: Permission denied (publickey).\n", 255))); + expect(isConnectionLost(error)).toBe(true); + expect(error.message).toBe( + "pi-env exited with code 255 before it was ready\nremote stderr:\nssh: Permission denied (publickey).", + ); + }); + + it("keeps the message unchanged when the daemon wrote nothing printable to stderr", async () => { + expect((await startError(connect("process.exit(3);"))).message).toBe( + "pi-env exited with code 3 before it was ready", + ); + expect((await startError(connect(failing(" \n\u0007\u001b[0m\n", 3)))).message).toBe( + "pi-env exited with code 3 before it was ready", + ); + }); + + it("drops terminal escapes and C0 and C1 control characters, keeping other text", async () => { + const stderr = + "\u001b[31mred\u001b[0m\r\nbell\u0007 del\u007f\tc1:\u0085\u009d end\nÜberprüfung fehlgeschlagen ✗\n\n"; + const error = await startError(connect(failing(stderr))); + expect(error.message).toBe( + "pi-env exited with code 1 before it was ready\nremote stderr:\nred\nbell del\tc1: end\nÜberprüfung fehlgeschlagen ✗", + ); + }); + + it("keeps only the last 20 lines of stderr", async () => { + const lines = Array.from({ length: 30 }, (_, index) => `line ${index}`); + const error = await startError(connect(failing(`${lines.join("\n")}\n`))); + expect(error.message).toBe( + `pi-env exited with code 1 before it was ready\nremote stderr:\n${lines.slice(10).join("\n")}`, + ); + }); + + it("keeps only the last 2000 characters of stderr", async () => { + const error = await startError(connect(failing(`first\n${"x".repeat(5000)}\n`))); + const tail = error.message.split("\nremote stderr:\n")[1] ?? ""; + expect(tail).toBe("x".repeat(1999)); + }); + + it("passes stderr to onLog unchanged", async () => { + const logged: string[] = []; + await startError(connect(failing("\u001b[31mred\u001b[0m\n"), (text) => logged.push(text))); + expect(logged.join("")).toBe("\u001b[31mred\u001b[0m\n"); + }); + + // #10716: acknowledge the first stderr read before writing the remaining UTF-8 bytes. + it.each(["\u00e9", "\u9519", "\u{1f600}"])("decodes %s split across stderr chunks", async (character) => { + const directory = mkdtempSync(join(tmpdir(), "pi-env-stderr-")); + const acknowledged = join(directory, "logged"); + const stderr = `prefix ${character} suffix\n`; + const bytes = Buffer.from(stderr); + const split = Buffer.byteLength("prefix ") + 1; + const logged: string[] = []; + const connection = connect( + ` +const { existsSync } = require("node:fs"); +const bytes = Buffer.from(${JSON.stringify([...bytes])}); +const timeout = setTimeout(() => process.exit(99), 3000); +process.stderr.write(bytes.subarray(0, ${split})); +const poll = setInterval(() => { + if (!existsSync(${JSON.stringify(acknowledged)})) return; + clearInterval(poll); + clearTimeout(timeout); + process.stderr.write(bytes.subarray(${split}), () => process.exit(7)); +}, 5); +`, + (text) => { + logged.push(text); + if (!existsSync(acknowledged)) writeFileSync(acknowledged, ""); + }, + ); + try { + const error = await startError(connection); + expect(logged.length).toBeGreaterThanOrEqual(2); + expect(logged.join("")).toBe(stderr); + expect(error.message).toBe( + `pi-env exited with code 7 before it was ready\nremote stderr:\n${stderr.trimEnd()}`, + ); + } finally { + connection.close(); + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("flushes an incomplete UTF-8 character when stderr ends", async () => { + const logged: string[] = []; + const error = await startError( + connect("process.stderr.write(Buffer.from([0x78, 0xf0, 0x9f]), () => process.exit(7));", (text) => + logged.push(text), + ), + ); + expect(logged.join("")).toBe("x\uFFFD"); + expect(error.message).toBe("pi-env exited with code 7 before it was ready\nremote stderr:\nx\uFFFD"); + }); + + it("reports only the stderr of the attempt that failed", async () => { + let attempt = 0; + const connection = connect(() => failing(`failure ${++attempt}\n`)); + expect((await startError(connection)).message).toContain("failure 1"); + const second = await startError(connection); + expect(second.message).toContain("failure 2"); + expect(second.message).not.toContain("failure 1"); + }); + + it("keeps the fields of a hello the daemon refused", async () => { + const connection = connect(fakeDaemon("error")); + for (let attempt = 0; attempt < 2; attempt++) { + const error = await startError(connection); + if (!(error instanceof RemoteError)) throw error; + expect(error.code).toBe("EINVAL"); + expect(error.path).toBe("/daemon"); + expect(error.fields.extra).toBe(1); + expect(error.message.startsWith("hello refused")).toBe(true); + expect(isConnectionLost(error)).toBe(false); + } + }); + + it("reports a daemon that cannot be started as a spawn error", async () => { + const connection = new Connection({ command: ["/nonexistent/pi-env"] }); + connections.push(connection); + const error = await startError(connection); + if (!(error instanceof RemoteError)) throw error; + expect(error.code).toBe("spawn_error"); + expect(isConnectionLost(error)).toBe(true); + }); +}); + +describe("Connection of a ready daemon", () => { + it("fails pending requests as soon as the daemon exits, even while its pipes stay open", async () => { + const connection = connect(fakeDaemon("result")); + await connection.info(); + const started = Date.now(); + const outcome = await connection.request("lstat", { path: "/" }).then( + () => undefined, + (error: unknown) => error, + ); + if (!(outcome instanceof Error)) throw new Error("The request did not fail"); + expect(isConnectionLost(outcome)).toBe(true); + expect(outcome.message).toBe("pi-env connection lost"); + // The process the daemon started holds its pipes for 3 s; the request must not wait for them. + expect(Date.now() - started).toBeLessThan(2000); + }); +}); diff --git a/packages/env/test/remote.test.ts b/packages/env/test/remote.test.ts index 4b1241148..4f7370679 100644 --- a/packages/env/test/remote.test.ts +++ b/packages/env/test/remote.test.ts @@ -117,6 +117,31 @@ describe("RemoteExecutionEnv", () => { expect(Date.now() - started).toBeLessThan(3000); }); + // The daemon used to die on Connection.close()'s SIGTERM before killing its commands. + it.skipIf(process.platform === "win32")("kills running commands when the connection closes", async () => { + const { env, connection } = environment(); + const pid = Promise.withResolvers(); + const running = env.exec( + ["sh", "-c", "echo $$; exec sleep 30"], + { onOutput: (text: string) => pid.resolve(Number.parseInt(text, 10)) }, + context, + ); + const command = await pid.promise; + connection.close(); + await running; + const alive = () => { + try { + process.kill(command, 0); + return true; + } catch { + return false; + } + }; + const deadline = Date.now() + 5000; + while (alive() && Date.now() < deadline) await new Promise((done) => setTimeout(done, 50)); + expect(alive()).toBe(false); + }); + it("fails requests on handles of a lost connection instead of reaching the new daemon", async () => { const { env, connection } = environment(); getOrThrow(await env.writeFile("a.txt", "AAA", context)); diff --git a/packages/evals/README.md b/packages/evals/README.md index 74c4904b0..406dec988 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -7,7 +7,7 @@ Behavioral evals for Pi's coding agent, built with `vitest-evals`. Eval definitions are flat under `evals/`: - `*.docs.eval.ts` is a documentation-lift eval. `eval:docs` runs each case in isolated `without_docs` and `with_docs` containers and reports lift. -- Other `*.eval.ts` files are host evals. `eval:host` runs them with Vitest on this machine. They are ordinary vitest-evals suites, not paired comparisons. +- Other `*.eval.ts` files are host evals. `eval:host` runs the smoke suite; `eval:audit` runs the documentation audit. They are ordinary vitest-evals suites, not paired comparisons. Runner code lives in `src/`: @@ -21,25 +21,25 @@ Eval suites and their fixtures live under `evals/`. Image build files live in `d ## Run evals -Host evals (smoke, documentation audit) and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. +The host smoke eval, on-demand documentation audit, and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval -w packages/evals ``` -That runs host evals, then the documentation comparison. Extra CLI flags after `--` go to `eval:docs` only. +That runs the host smoke eval, then the documentation comparison. The documentation audit remains explicit because it scans every documentation page. Extra CLI flags after `--` go to `eval:docs` only. -Host only: +Host smoke only: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval:host -w packages/evals ``` -One host suite: +Documentation audit: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol \ - npm run eval:host -w packages/evals -- evals/documentation-audit.eval.ts + npm run eval:audit -w packages/evals ``` ## Run documentation comparisons diff --git a/packages/evals/evals/configured-runtime.ts b/packages/evals/evals/configured-runtime.ts index 8aaca0eee..c343634d5 100644 --- a/packages/evals/evals/configured-runtime.ts +++ b/packages/evals/evals/configured-runtime.ts @@ -1,4 +1,3 @@ -import { join } from "node:path"; import { type Api, type Context, contentText, type Model, type ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { ModelRuntime } from "@earendil-works/pi-coding-agent"; @@ -53,15 +52,6 @@ function modelFields(model: Model): ModelFields { }; } -export function loadConfiguredModelRuntime(agentDir: string): Promise { - return ModelRuntime.create({ - modelsPath: join(agentDir, "models.json"), - authPath: join(agentDir, "auth.json"), - modelsStorePath: join(agentDir, "models-store.json"), - allowModelNetwork: false, - }); -} - export async function inspectProvider( runtime: ModelRuntime, scenario: ProviderScenario, diff --git a/packages/evals/evals/documentation-audit.eval.ts b/packages/evals/evals/documentation-audit.eval.ts index dc2d6a582..baf8c6f0f 100644 --- a/packages/evals/evals/documentation-audit.eval.ts +++ b/packages/evals/evals/documentation-audit.eval.ts @@ -1,5 +1,5 @@ import { globSync } from "node:fs"; -import { resolve } from "node:path"; +import { isAbsolute, relative, resolve } from "node:path"; import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { expect } from "vitest"; @@ -31,6 +31,7 @@ const submitAudit = defineTool({ const repositoryRoot = resolve(import.meta.dirname, "../../.."); const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs"); +const sourceRoot = resolve(repositoryRoot, "packages/coding-agent/src"); const pages = globSync("**/*.md", { cwd: docsRoot }) .map((path) => ({ path: path.replaceAll("\\", "/") })) .sort((left, right) => left.path.localeCompare(right.path)); @@ -56,7 +57,20 @@ For a mismatch, quote the claim, cite the implementation path and symbol, and st Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`); - const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME); + const calls = toolCalls(result); + const readPaths = calls.flatMap((call) => { + const path = call.name === "read" && call.status === "ok" ? call.arguments?.path : undefined; + return typeof path === "string" ? [resolve(path)] : []; + }); + expect(readPaths).toContain(documentationPath); + expect( + readPaths.some((readPath) => { + const sourceRelative = relative(sourceRoot, readPath); + return sourceRelative !== "" && !sourceRelative.startsWith("..") && !isAbsolute(sourceRelative); + }), + ).toBe(true); + + const auditCalls = calls.filter((call) => call.name === TOOL_NAME); expect(auditCalls).toHaveLength(1); const auditCall = auditCalls[0]; expect(auditCall?.status).toBe("ok"); diff --git a/packages/evals/evals/extensions.docs.eval.ts b/packages/evals/evals/extensions.docs.eval.ts index ef21c8f60..5fd51666d 100644 --- a/packages/evals/evals/extensions.docs.eval.ts +++ b/packages/evals/evals/extensions.docs.eval.ts @@ -16,7 +16,6 @@ const harness = createPiDocumentationEvalHarness({ return { response, extensionErrors: extensions.errors, - toolLoaded: extensions.extensions.some(({ tools }) => tools.has(TOOL_NAME)), toolResult: result?.role === "toolResult" ? contentText(result.content) : null, }; }, @@ -31,7 +30,6 @@ describeEval( expected: { response: TOOL_RESULT, extensionErrors: [], - toolLoaded: true, toolResult: TOOL_RESULT, }, match: "strict", diff --git a/packages/evals/evals/prompt-templates.docs.eval.ts b/packages/evals/evals/prompt-templates.docs.eval.ts new file mode 100644 index 000000000..08edd2238 --- /dev/null +++ b/packages/evals/evals/prompt-templates.docs.eval.ts @@ -0,0 +1,79 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { contentText } from "@earendil-works/pi-ai"; +import { describeEval, StructuredOutputJudge } from "vitest-evals"; +import { createPiDocumentationEvalHarness } from "../src/harness.ts"; + +const TEMPLATE_NAME = "current-time"; +const COMMAND = `/${TEMPLATE_NAME}`; +const TEMPLATE_PROMPT = "Tell me the current time and timezone."; + +const SCENARIOS = [ + { + title: "Create a project prompt template", + request: `This project is already trusted. Add a ${COMMAND} prompt template for this project containing exactly: ${TEMPLATE_PROMPT}`, + scope: "project", + }, + { + title: "Create a user prompt template", + request: `Make a ${COMMAND} prompt template available in all my Pi sessions containing exactly: ${TEMPLATE_PROMPT}`, + scope: "user", + }, +] as const; + +function registerScenario(scenario: (typeof SCENARIOS)[number]): void { + const harness = createPiDocumentationEvalHarness({ + output: ({ session, agentDir }) => { + const workspace = session.sessionManager.getCwd(); + const projectPath = join(workspace, ".pi", "prompts", `${TEMPLATE_NAME}.md`); + const userPath = join(agentDir, "prompts", `${TEMPLATE_NAME}.md`); + const expectedPath = scenario.scope === "project" ? projectPath : userPath; + const otherPath = scenario.scope === "project" ? userPath : projectPath; + const { prompts, diagnostics } = session.resourceLoader.getPrompts(); + const extensions = session.resourceLoader.getExtensions(); + const matchingTemplates = prompts.filter(({ name }) => name === TEMPLATE_NAME); + const template = matchingTemplates.length === 1 ? matchingTemplates[0] : undefined; + const expandedPrompt = [...session.messages] + .reverse() + .find((message) => message.role === "user"); + const expandedText = expandedPrompt?.role === "user" ? contentText(expandedPrompt.content).trim() : ""; + + return { + template: template + ? { scope: template.sourceInfo.scope, atExpectedPath: template.filePath === expectedPath } + : null, + promptDiagnostics: diagnostics.map(({ message }) => message), + extensions: extensions.extensions.filter(({ hidden }) => !hidden).map(({ path }) => path), + extensionErrors: extensions.errors.map(({ error }) => error), + skills: session.resourceLoader.getSkills().skills.map(({ name }) => name), + otherScopeTemplateExists: existsSync(otherPath), + expandedToTimeRequest: expandedText === TEMPLATE_PROMPT, + }; + }, + }); + const judge = StructuredOutputJudge({ + expected: { + template: { scope: scenario.scope, atExpectedPath: true }, + promptDiagnostics: [], + extensions: [], + extensionErrors: [], + skills: [], + otherScopeTemplateExists: false, + expandedToTimeRequest: true, + }, + match: "strict", + allowExtras: false, + }); + + describeEval(scenario.title, { harness, judges: [judge], judgeThreshold: null }, (it) => { + it(`creates, reloads, and invokes ${COMMAND}`, async ({ run }) => { + await run([ + { type: "prompt", content: scenario.request }, + { type: "reload" }, + { type: "prompt", content: COMMAND }, + ]); + }); + }); +} + +for (const scenario of SCENARIOS) registerScenario(scenario); diff --git a/packages/evals/package.json b/packages/evals/package.json index b36546214..f740c3368 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -1,21 +1,21 @@ { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "private": true, "type": "module", "scripts": { - "clean": "shx rm -rf .eval", + "clean": "node ../../scripts/package-files.mjs clean .eval", "eval": "npm run eval:host && npm run eval:docs --", "eval:host": "vitest run --config vitest.evals.config.ts --project host", + "eval:audit": "vitest run --config vitest.evals.config.ts --project audit", "eval:docs": "node --experimental-strip-types src/cli.ts", "test": "vitest run --config vitest.test.config.ts" }, "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts index 7c5e1d093..cd7192d39 100644 --- a/packages/evals/src/report.ts +++ b/packages/evals/src/report.ts @@ -98,7 +98,7 @@ function validateScore(value: unknown): number | undefined { return value; } -export function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { +function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { if (status === "failed") return "errored"; if (status === "skipped" || status === "todo" || status === "disabled") return "skipped"; if (status === "pending") return "pending"; @@ -145,15 +145,16 @@ export async function readTaskObservation( if (!loaded) return { ...identity, outcome: "errored" }; const [{ workspace }, rawReport] = loaded; const reportedFullName = `${task.evalSet} ${task.caseId}`; - const assertions = rawReport.testResults.flatMap(({ assertionResults }) => assertionResults); + const assertions = rawReport.testResults + .flatMap(({ assertionResults }) => assertionResults) + .filter(({ fullName }) => fullName === reportedFullName); if (assertions.length !== 1) return { ...identity, outcome: "errored" }; const assertion = assertions[0]; - if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; const statusOutcome = classifyCaseStatus(assertion.status); if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome }; - if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" }; - const caseResult = workspace.cases[0]; - if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; + const caseResults = workspace.cases.filter(({ fullName }) => fullName === reportedFullName); + if (caseResults.length !== 1) return { ...identity, outcome: "errored" }; + const caseResult = caseResults[0]; if (caseResult.status !== assertion.status) return { ...identity, outcome: "errored" }; const run = caseResult.harness?.run; if (!run) return { ...identity, outcome: "errored" }; diff --git a/packages/evals/test/comparison.test.ts b/packages/evals/test/comparison.test.ts index ba2ad550d..4a00bac8e 100644 --- a/packages/evals/test/comparison.test.ts +++ b/packages/evals/test/comparison.test.ts @@ -78,6 +78,8 @@ describe("summarizeEvalObservations", () => { lift: 0.5, totalTokens: { eligiblePairs: 2, controlMean: 150, treatmentMean: 150, meanDelta: 0 }, toolCalls: { eligiblePairs: 2, controlMean: 2.5, treatmentMean: 2, meanDelta: -0.5 }, + totalMs: { eligiblePairs: 2, controlMean: 1000, treatmentMean: 900, meanDelta: -100 }, + estimatedCostUsd: { eligiblePairs: 2, controlMean: 0.01, treatmentMean: 0.01, meanDelta: 0 }, }), ]); }); diff --git a/packages/evals/test/configured-runtime.test.ts b/packages/evals/test/configured-runtime.test.ts index 7b74c94d3..267151486 100644 --- a/packages/evals/test/configured-runtime.test.ts +++ b/packages/evals/test/configured-runtime.test.ts @@ -1,6 +1,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ModelRuntime } from "@earendil-works/pi-coding-agent"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { createAcmeServer, @@ -9,7 +10,7 @@ import { OPENAI_PROBE_RESPONSE, OPENAI_PROVIDER_ID, } from "../evals/acme-server.ts"; -import { inspectAddedModel, inspectProvider, loadConfiguredModelRuntime } from "../evals/configured-runtime.ts"; +import { inspectAddedModel, inspectProvider } from "../evals/configured-runtime.ts"; const PROVIDER_ID = OPENAI_PROVIDER_ID; const MODEL = { @@ -43,6 +44,15 @@ async function runtimeWith(modelsJson: unknown) { return loadConfiguredModelRuntime(await agentDirWith(modelsJson)); } +function loadConfiguredModelRuntime(agentDir: string): Promise { + return ModelRuntime.create({ + modelsPath: join(agentDir, "models.json"), + authPath: join(agentDir, "auth.json"), + modelsStorePath: join(agentDir, "models-store.json"), + allowModelNetwork: false, + }); +} + function acmeModelsJson() { return { providers: { diff --git a/packages/evals/test/report.test.ts b/packages/evals/test/report.test.ts index f872ea4f3..594f2b96a 100644 --- a/packages/evals/test/report.test.ts +++ b/packages/evals/test/report.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { EvalTask } from "../src/plan.ts"; -import { classifyCaseStatus, PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; +import { PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; const temporaryDirectories: string[] = []; afterEach(async () => { @@ -22,40 +22,41 @@ const task: EvalTask = { const SESSION = '{"type":"session"}\n'; type ReportAssertion = { - status?: "passed" | "skipped" | "pending" | "failed"; + caseId?: string; + status?: "passed" | "skipped" | "pending" | "todo" | "disabled" | "failed"; meta?: Record; }; -async function writeTaskReport(assertion: ReportAssertion = {}) { +async function writeTaskReport(assertions: ReportAssertion[]) { const directory = await mkdtemp(join(tmpdir(), "pi-eval-report-test-")); temporaryDirectories.push(directory); const reportPath = join(directory, "vitest.json"); - const status = assertion.status ?? "passed"; + const results = assertions.map(({ caseId = task.caseId, status = "passed", meta = {} }) => ({ + ancestorTitles: [task.evalSet], + fullName: `${task.evalSet} ${caseId}`, + status, + title: caseId, + failureMessages: [], + meta, + })); + const count = (status: ReportAssertion["status"]): number => + results.filter((result) => result.status === status).length; await writeFile( reportPath, JSON.stringify({ - numFailedTests: 0, - numPassedTests: status === "passed" ? 1 : 0, - numPendingTests: status === "pending" || status === "skipped" ? 1 : 0, - numTodoTests: 0, - numTotalTests: 1, + numFailedTests: count("failed"), + numPassedTests: count("passed"), + numPendingTests: count("pending") + count("skipped") + count("disabled"), + numTodoTests: count("todo"), + numTotalTests: results.length, startTime: 0, - success: true, + success: count("failed") === 0, testResults: [ { message: "", name: "/repo/packages/evals/evals/example.docs.eval.ts", - status: "passed", - assertionResults: [ - { - ancestorTitles: [task.evalSet], - fullName: `${task.evalSet} ${task.caseId}`, - status, - title: task.caseId, - failureMessages: [], - meta: assertion.meta ?? {}, - }, - ], + status: count("failed") === 0 ? "passed" : "failed", + assertionResults: results, }, ], }), @@ -97,26 +98,19 @@ function scoredMeta(overrides?: { }; } -async function readObservation(assertion?: ReportAssertion) { - const { directory, reportPath } = await writeTaskReport(assertion); +async function readObservation(assertion: ReportAssertion = {}) { + const { directory, reportPath } = await writeTaskReport([assertion]); return { directory, observation: await readTaskObservation(task, reportPath, directory) }; } -describe("classifyCaseStatus", () => { - it.each(["skipped", "todo", "disabled"] as const)("maps %s to skipped", (status) => { - expect(classifyCaseStatus(status)).toBe("skipped"); - }); - - it("maps failed infrastructure to errored", () => { - expect(classifyCaseStatus("failed")).toBe("errored"); - }); -}); - describe("readTaskObservation", () => { - it("preserves a skipped outcome when no harness run exists", async () => { - const { observation } = await readObservation({ status: "skipped" }); - expect(observation).toMatchObject({ outcome: "skipped" }); - }); + it.each(["skipped", "todo", "disabled"] as const)( + "preserves a skipped outcome for %s cases without harness runs", + async (status) => { + const { observation } = await readObservation({ status }); + expect(observation).toMatchObject({ outcome: "skipped" }); + }, + ); it("preserves a pending outcome when no harness run exists", async () => { const { observation } = await readObservation({ status: "pending" }); @@ -126,7 +120,7 @@ describe("readTaskObservation", () => { it("preserves metrics from a failed eval with a partial harness run", async () => { const { directory, observation } = await readObservation({ status: "failed", - meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }), + meta: scoredMeta(), }); expect(observation).toEqual({ evalSet: task.evalSet, @@ -181,6 +175,15 @@ describe("readTaskObservation", () => { ).resolves.toBe(SESSION); }); + it("selects the requested case when a sibling case is skipped", async () => { + const { directory, reportPath } = await writeTaskReport([ + { caseId: "sibling case", status: "skipped" }, + { caseId: task.caseId, meta: scoredMeta() }, + ]); + const observation = await readTaskObservation(task, reportPath, directory); + expect(observation).toMatchObject({ outcome: "scored", score: 0.5, totalTokens: 15 }); + }); + it("treats a zero score as scored data", async () => { const { observation } = await readObservation({ meta: scoredMeta({ avgScore: 0 }) }); expect(observation).toMatchObject({ outcome: "scored", score: 0 }); diff --git a/packages/evals/vitest.evals.config.ts b/packages/evals/vitest.evals.config.ts index ced768eaf..20ecd73e7 100644 --- a/packages/evals/vitest.evals.config.ts +++ b/packages/evals/vitest.evals.config.ts @@ -24,7 +24,17 @@ const evalConfig = defineConfig({ test: { name: "host", include: ["evals/**/*.eval.ts"], - exclude: ["evals/**/*.docs.eval.ts"], + exclude: ["evals/**/*.docs.eval.ts", "evals/documentation-audit.eval.ts"], + sequence: { concurrent: false }, + testTimeout: 300_000, + hookTimeout: 300_000, + }, + }, + { + extends: true, + test: { + name: "audit", + include: ["evals/documentation-audit.eval.ts"], sequence: { concurrent: false }, testTimeout: 300_000, hookTimeout: 300_000, diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 74908a649..a0e3f2cc8 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Changed + +- README: the `AgentTool` wrapping example passes the MCP input schema as plain JSON Schema instead of through TypeBox's `Type.Unsafe`. + ### Fixed - OAuth metadata parsing and `startAuthorization()` now accept only `http:` and `https:` URLs. Before, a server could supply a `file:` or custom protocol authorization endpoint (for example `search-ms:`), which the platform URL handler would launch on sign-in. diff --git a/packages/mcp/README.md b/packages/mcp/README.md index b09e043fb..ebb1b2973 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -36,7 +36,6 @@ Wrapping an MCP tool as a `pi-agent-core` `AgentTool`: ```typescript import type { AgentTool } from "@earendil-works/pi-agent-core"; import { toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ // Providers allow at most 64 characters of [A-Za-z0-9_-]. @@ -44,7 +43,7 @@ const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ label: tool.title ?? tool.name, description: tool.description ?? tool.name, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }), + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }, execute: async (_toolCallId, params, signal) => { const result = await client.callTool(tool.name, params as Record, { signal }); // MCP reports tool failures in the result instead of as a protocol error. diff --git a/packages/mcp/package.json b/packages/mcp/package.json index b016e8067..8a9bac663 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "description": "Standalone Model Context Protocol client for pi and other applications", "type": "module", "main": "./dist/index.js", @@ -30,7 +30,7 @@ "LICENSES" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -55,7 +55,6 @@ }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md new file mode 100644 index 000000000..1fb3218b0 --- /dev/null +++ b/packages/schema/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## [Unreleased] + +## [1.5.0] - 2026-11-02 + +### Added + +- Initial release: `compile(schema, options)` validates JSON Schema drafts 2020-12 and 7 without `eval` or `new Function`, compiling a schema once into plain closures. `Validator.check` is the boolean check (a type guard for schemas built with the builders), `Validator.errors` reports each failure with a JSON Pointer path. Format checks are in the optional `@earendil-works/pi-schema/formats` entry point. +- Schema builders (`str`, `num`, `int`, `bool`, `nul`, `unknown`, `lit`, `oneOf`, `obj`, `opt`, `arr`, `tuple`, `map`, `union`, `nullable`, `bytes`) that return plain JSON Schema typed with the values it accepts (`Static`). +- `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. +- `Static` also types TypeBox-built schemas, from TypeBox 1.x's type shapes and without importing TypeBox: String, Number, Integer, Boolean, Null, Literal, Array, Object (with `Type.Optional` and `Type.Readonly` properties), Union, Record, Unsafe, Unknown and Any give TypeBox's static type; other TypeBox kinds give `unknown`. +- `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. +- `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. +- `npm run bench` compares conformance, speed (with and without eval), setup cost, memory and bundle size with ata-validator (generated code and its interpreter), Ajv, Zod (native schemas, and `z.fromJSONSchema` for the test suite), TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/LICENSES/typebox-LICENSE b/packages/schema/LICENSES/typebox-LICENSE new file mode 100644 index 000000000..0330da432 --- /dev/null +++ b/packages/schema/LICENSES/typebox-LICENSE @@ -0,0 +1,23 @@ +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/packages/schema/README.md b/packages/schema/README.md new file mode 100644 index 000000000..f1ff34f4d --- /dev/null +++ b/packages/schema/README.md @@ -0,0 +1,296 @@ +# @earendil-works/pi-schema + +JSON Schema builders and validation without `eval`. A schema compiles once into plain closures, so it checks values +at close to the speed of validators that generate code, also under a strict Content Security Policy, and needs no +dependencies. + +- Drafts 2020-12 and 7. Passes 2,222 of the 2,223 core tests of the official + [JSON-Schema-Test-Suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite). +- Builders return plain JSON Schema typed with the values it accepts; a compiled check narrows to that type. +- Errors as JSON Pointer paths. A missing or unknown field is reported at the field. +- Never modifies a schema; frozen schemas work. +- 8.5 KB minified and gzipped. Format checks are an optional import. + +## Installation + +```bash +npm install @earendil-works/pi-schema +``` + +## Usage + +```typescript +import { compile, int, obj, oneOf, opt, type Static, str } from "@earendil-works/pi-schema"; + +const Read = obj({ + path: str({ description: "Path to the file", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["text", "binary"]), +}); +type Read = Static; // { path: string; mode: "text" | "binary"; offset?: number } + +const read = compile(Read); +if (read.check(value)) value.path; // value: Read + +read.errors({ offset: 0, extra: true }); +// [ +// { path: "/path", keyword: "required", message: "missing" }, +// { path: "/mode", keyword: "required", message: "missing" }, +// { path: "/offset", keyword: "minimum", message: "expected >= 1" }, +// { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, +// ] +``` + +`Read` is plain JSON Schema (`{ type: "object", properties: {...}, required: ["path", "mode"], additionalProperties: +false }`), ready for a tool definition or `JSON.stringify`. `compile` takes any JSON Schema, built with the builders +or not. + +`check(value)` is the fast path: a boolean, no allocations for most schemas, and a plain function you can pass +around. `errors(value)` explains a failure and returns `[]` for a valid value. It is built on the first failure and +kept, so a schema whose values always pass never pays for it. + +### Builders + +| builder | schema | type | +|---|---|---| +| `str({ minLength, maxLength, pattern, format })` | `{ type: "string", ... }` | `string` | +| `num({ minimum, maximum, exclusiveMinimum, exclusiveMaximum, multipleOf })`, `int(...)` | `{ type: "number" }`, `{ type: "integer" }` | `number` | +| `bool()`, `nul()`, `unknown()` | `{ type: "boolean" }`, `{ type: "null" }`, `{}` | `boolean`, `null`, `unknown` | +| `lit(value)` | `{ const: value }` | the literal | +| `oneOf(["a", "b"])` | `{ enum: [...] }` | `"a" \| "b"` | +| `obj({ a: str(), b: opt(num()) }, { open })` | closed object; `open: true` allows other properties | `{ a: string; b?: number }` | +| `arr(items, { minItems, maxItems, uniqueItems })` | `{ type: "array", items }` | `T[]` | +| `tuple([str(), num()])` | `prefixItems`, no more items | `[string, number]` | +| `map(value)` | `{ type: "object", additionalProperties: value }` | `Record` | +| `union([...])` | `{ anyOf: [...] }` | the union | +| `nullable(schema)` | `{ anyOf: [schema, { type: "null" }] }` | `T \| null` | +| `bytes({ minLength, maxLength })` | `{ type: "bytes", ... }` | `Uint8Array` | + +Every builder also takes `title`, `description`, `examples`, `deprecated` and `default`. A `default` is an annotation; +validation never inserts it. + +A union of objects tagged by a `lit()` property checks and reports by the tag: only the branch the tag selects runs. + +```typescript +const Block = union([obj({ kind: lit("text"), text: str() }), obj({ kind: lit("image"), width: int({ minimum: 0 }) })]); +compile(Block).errors({ kind: "video" }); +// [{ path: "/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }] +``` + +`Static` also types schemas built with TypeBox 1.x, read from TypeBox's type shapes (pi-schema does not import +TypeBox): `String`, `Number`, `Integer`, `Boolean`, `Null`, `Literal`, `Array`, `Object` (with `Type.Optional` and +`Type.Readonly` properties), `Union`, `Record`, `Unsafe`, `Unknown` and `Any` give TypeBox's own static type, so +`Static` is `{ path: string }`. Other TypeBox kinds (`Enum`, `Tuple`, +`Intersect`, `Cyclic`, ...) give `unknown` where they stand. Other JSON Schema objects give `unknown`. + +### Options + +```typescript +import { formats } from "@earendil-works/pi-schema/formats"; + +const validator = compile(schema, { + draft: "7", // "2020-12" (default) or "7" + formats, // check `format`; without it `format` is an annotation, as 2020-12 specifies + schemas: { "https://example.com/point.json": point }, // documents for $refs to other documents, by URI + unresolvedRefs: "reject", // a $ref that names nothing rejects values at its place; default "throw" + coerce: true, // adds coerce() and parse(), see Coercion +}); +``` + +`formats` has every format 2020-12 defines (`date-time`, `email`, `uri`, `uuid`, `ipv6`, ...), plus `url` and +`json-pointer-uri-fragment`, which TypeBox and ajv-formats also check. Pass your own map to check fewer formats, or +custom ones. + +A `$ref` that names nothing (a missing definition, a document not in `schemas`) throws when the schema compiles. With +`unresolvedRefs: "reject"` it compiles as `false` instead: only values at that place fail, with the error +`unresolved reference `, and the rest of the schema works. Use it for schemas from other systems, such as MCP +servers, whose references may dangle. + +### TypeBox schemas + +TypeBox 1.x schemas are JSON Schema, so they compile as they are: + +```typescript +import Type from "typebox"; + +const user = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); +``` + +TypeBox's JavaScript-only types (`BigInt`, `Undefined`, `Function`, ...), refinements and codecs are not JSON Schema +and are not supported. + +### Coercion + +`compile(schema, { coerce: true })` adds `coerce()` and `parse()`, for loosely typed input such as a model's tool call +arguments. Without the option the validator has neither, and its `check` pays nothing for them: the coercion rules are +built on the first `coerce()` or `parse()`. + +```typescript +const read = compile(Read, { coerce: true }); + +read.coerce({ path: "a.txt", offset: "10", mode: "text" }); // { path: "a.txt", offset: 10, mode: "text" } + +const result = read.parse({ path: "a.txt", offset: "ten", mode: "text" }); +if (result.ok) result.value.offset; // number +else result.errors; // [{ path: "/offset", keyword: "type", message: "expected integer, got string" }] +``` + +`coerce(value)` returns the coerced value. It copies only the paths it changes, returns `value` itself when nothing +changes, and never modifies `value`. A value no rule applies to stays as it is. `parse(value)` coerces, then checks: +`{ ok: true, value }` with the coerced value, or `{ ok: false, errors }` with the errors of the coerced value. + +Three passes run in order: + +1. **Optional nulls.** A `null` in an optional property whose schema rejects `null` is dropped, as if omitted. +2. **Conversions**, ported from TypeBox 1.3.27's `Value.Convert` and keyed on JSON Schema keywords (the first that + applies to a schema): `$ref` follows the reference; `const` and `enum` take the member the value converts to; + `anyOf`/`oneOf` keep a value one branch accepts, else take the first branch's conversion the whole schema + accepts; `allOf` converts with each member in turn; then by `type`: + - `string`: numbers and booleans to their text, `null` to `"null"`. + - `number`: numeric strings (`""` is 0), `"true"`/`"false"`, booleans to 1/0, `null` to 0. + - `integer`: as `number`, then truncated (`"3.5"` and `3.5` become 3). + - `boolean`: `"true"`/`"false"` (any case), `"1"`/`"0"`, 1/0, `null` to `false`. + - `null`: `"null"`, `"undefined"`, `""`, `"0"`, 0 and `false`. + - `array`: a single value becomes a one-item array, then each item converts; a tuple (`prefixItems` or an `items` + array) converts its items and never wraps. + - `object`: each property converts by `properties`, `patternProperties` or `additionalProperties`. +3. **Primitive coercion by type**, as Ajv's `coerceTypes`, for what pass 2 leaves (type arrays such as + `["integer", "null"]`): numbers from numeric strings and booleans, booleans from `"true"`/`"false"` and 1/0, + strings from numbers and booleans, `null` from `""`, 0 and `false`, and `null` to `""`, 0 or `false`. A value that + already has one of a type array's types stays. + +Where passes 2 and 3 disagree, pass 2 wins: `null` for a required string becomes `"null"`, not `""`. A union's +branches convert in order and only until one is accepted; a union node that also declares properties converts them into +each branch's candidate before checking it. A recursive schema (a tree through `$ref`) that meets a value it cannot +convert leaves the value as it is, so `check` reports it, and results through a `$ref` are reused within one call, so a +recursive union does not convert (or explain) the same child once per branch. + +Pass 1 drops optional nulls inside the values of `properties`, `patternProperties` and `additionalProperties` alike; a +pattern or additional key itself is never dropped. Items past a draft-7 `items` array (`additionalItems`) are not +coerced by any pass. + +### Values + +- A property whose value is `undefined` counts as absent, as in TypeBox. +- `{ type: "bytes", minLength, maxLength }` matches a `Uint8Array` of that length, an extension for values that never + pass through JSON. + +## Supported keywords + +- Every validation keyword of drafts 2020-12 and 7, including draft 7's `dependencies` and `items` arrays with + `additionalItems`. +- `$id`, `$anchor`, `$ref` (local, by anchor, relative and remote), `$dynamicRef` and `$dynamicAnchor`. +- `unevaluatedProperties` and `unevaluatedItems`. +- `pattern` and `patternProperties` are ECMAScript regular expressions with the `u` flag. +- Not supported: meta-schemas that switch vocabularies on or off (`$vocabulary`), `contentMediaType` and + `contentEncoding` assertions (optional in the spec), and draft 2019-09's `$recursiveRef`. + +## Benchmarks + +Node 24.4.1 on one core of an i9-13900. The other libraries: + +- ata-validator 1.61.3: its generated code, and its eval-free interpreter chosen with `engine: "interpreter"`; +- Ajv 8.20.0, with ajv-formats 3.0.1 for the format tests; +- Zod 4.6.7: native Zod schemas for speed, setup and memory, and `z.fromJSONSchema` for the test suite; +- TypeBox 1.3.27; +- @cfworker/json-schema 4.1.1; +- @oh-my-pi/omptype 18.13.3, through `fromJsonSchema`. + +| test suite (tests passed) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| draft 2020-12 core (1,298) | 1,297 | 1,298 | 1,298 | 1,236 | 693 | 1,276 | 1,249 | 863 | +| draft 7 core (925) | 925 | 925 | 925 | 917 | 632 | 924 | 918 | 659 | +| draft 2020-12 format (895), with format checks | 895 | 814 | 814 | 750 | 412 | 895 | 740 | 412 | + +| check, valid value | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 410 ns | 180 ns | 836 ns | 627 ns | 1.3 µs | 475 ns | 43 µs | 730 ns | +| union of 5 messages | 86 ns | 37 ns | 212 ns | 70 ns | 297 ns | 95 ns | 11 µs | 96 ns | +| tool schema (read) | 31 ns | 10 ns | 41 ns | 13 ns | 41 ns | 9 ns | 2.0 µs | 9 ns | +| MCP tool schema with `$ref` | 161 ns | 64 ns | 314 ns | 136 ns | 927 ns | 87 ns | 15 µs | 1.9 µs | +| recursive `$ref` schema | 755 ns | 307 ns | 1.9 µs | 378 ns | 2.7 µs | 1.2 µs | 41 µs | 7.5 µs | + +| same, without eval (strict CSP) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 411 ns | 1.6 µs | 844 ns | fails | 3.4 µs | 44 µs | 44 µs | 2.7 µs | +| union of 5 messages | 87 ns | 396 ns | 209 ns | fails | 652 ns | 10 µs | 11 µs | 501 ns | +| MCP tool schema with `$ref` | 164 ns | 484 ns | 326 ns | fails | 2.0 µs | 39 µs | 13 µs | 2.2 µs | +| recursive `$ref` schema | 746 ns | 2.8 µs | 1.9 µs | fails | 7.3 µs | 130 µs | 42 µs | 9.0 µs | + +| cost | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| compile a new schema | 20 µs | 4.9 µs | 1.3 µs | 1.4 ms | 151 µs | 210 µs | 22 µs | 28 µs | +| its first check | 0.8 µs | 335 µs | 87 µs | 195 µs | 466 µs | 41 µs | 7.1 µs | 109 µs | +| memory, 3,200 distinct schemas | 7.1 MB | 36.0 MB | 51.3 MB | 59.7 MB | 170.8 MB | 54.7 MB | 7.6 MB | 19.9 MB | +| bundle, minified + gzip | 8.5 KB | 134.0 KB | 134.0 KB | 39.7 KB | 24.4 KB | 35.2 KB | 5.8 KB | 38.8 KB | + +Where code generation is unavailable: + +- **ata** answers through its interpreted engine without being told. That automatic fallback measured slower than the + engine chosen with `engine: "interpreter"`. +- **Ajv** cannot compile at all: `compile()` throws `EvalError`. +- **Zod** notices and parses objects without its generated code. +- **TypeBox** falls back to its interpreter. +- **omptype** throws `EvalError` once per schema, then validates through its interpreter. + +ata, Zod and omptype compile on a schema's first check, so their compile cost shows there. ata's bundle is for the +browser without code splitting; with splitting, its first load is about 53 KB plus a 76 KB generator chunk loaded +later. + +Zod is not a JSON Schema validator: + +- **Speed, setup and memory:** Zod gets the same schemas written as native Zod schemas (`z.strictObject`, + `z.discriminatedUnion`, `z.lazy`), with `.describe()` for the corpus descriptions, and `safeParse` as the check. + Its memory includes building the schemas, which the other libraries receive as JSON. Without the descriptions it + is 131 MB. +- **Test suite:** through `z.fromJSONSchema`, which is semi-experimental. It refuses `unevaluatedProperties`, + `unevaluatedItems`, `dependentSchemas`, `dependentRequired`, `if`/`then`/`else`, `not`, and every `$ref` that is + not local. A refused schema fails all its tests: 405 of the 1,298 draft 2020-12 core tests and 107 of the 925 draft + 7 core tests. +- **Bundle:** measured with Zod's documented `import * as z from "zod"`. A named `import { z }` keeps every locale + (about 90 KB gzipped), and `z.fromJSONSchema` brings it to 40.0 KB; `zod/mini` is about 6 KB. + +Flat objects of simple types check in 13–80 ns, against 7–15 ns for TypeBox. Without code generation, reading a +property whose name comes from a list costs a few nanoseconds. + +### Reproduce + +`npm run bench` in `packages/schema` takes about 12 minutes. It runs every library through its fastest boolean path +(ata's `isValidObject()`) and keeps the object each returns. ata runs with `ATA_NO_NATIVE=1`: its optional native +addon only serves its buffer APIs. The scripts are in `test/bench/`. + +- **Conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests + reference (Zod through `z.fromJSONSchema`, which takes no remote documents). ata reads the draft only from `$schema`, so draft 7 schemas without one get it declared. ata's defaults + (`useDefaults`, `assertFormat`) are turned off to match the other libraries; formats are checked only in the + format tests, by each library's own checks. +- **Speed:** realistic schemas, in ns per check after warm-up. Each result is the median of 15 rounds, with the + libraries' order rotated in each round. +- **Speed without eval:** the same, run with `--disallow-code-generation-from-strings`. +- **Setup:** compiling 1,000 schemas a library has not seen, and their first checks. +- **Memory:** 3,200 distinct tool-like schemas, compiled and each checked 3 times, each library in its own process. +- **Size:** esbuild browser bundles of what a validator needs, minified and gzipped. + +### Code generated at build time + +Ajv (`ajv/dist/standalone`) and ata (`ata build`, `ata compile`, `toStandaloneModule()`) can also generate their +validation code when the app is built. The app then imports plain modules and evaluates nothing at run time, so those +checks run at code-generation speed under a strict CSP. This is not in the benchmark, because it is a different +deal: + +- **A build step per schema.** Every schema must be known when the app is built. Schemas that arrive at run time + (MCP servers, extensions, user input) cannot be validated this way. +- **Code per schema.** In a separate measurement on the same 3,200 schemas: + - Ajv: 1.8 ms to build and 8.9 KB of JavaScript per schema; + - ata: 1.1 ms and 13 KB per schema, or 0.3–0.8 KB gzipped when only its `isValid` export is bundled; + - importing all 3,200 generated validators took 40 MB of heap (Ajv, one module) and 68 MB (ata, one module per + schema). + +pi-schema compiles at run time in 25 µs per schema, generates no code, and covers schemas known at build time and +schemas that arrive later. + +## Development + +`npm test` runs the official test suite (drafts 2020-12 and 7, including the optional and format tests, vendored in +`test/fixtures/`) and the package's own tests. + +The format checks are adapted from TypeBox 1.3.27 (MIT); its license is in `LICENSES/`. diff --git a/packages/schema/package.json b/packages/schema/package.json new file mode 100644 index 000000000..4e176a57c --- /dev/null +++ b/packages/schema/package.json @@ -0,0 +1,60 @@ +{ + "name": "@earendil-works/pi-schema", + "version": "1.5.0", + "description": "JSON Schema validation without eval: drafts 2020-12 and 7, compiled into plain closures", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./formats": { + "source": "./src/formats/index.ts", + "types": "./dist/formats/index.d.ts", + "import": "./dist/formats/index.js" + } + }, + "sideEffects": false, + "files": [ + "dist", + "README.md", + "CHANGELOG.md", + "LICENSES" + ], + "scripts": { + "clean": "node ../../scripts/package-files.mjs clean dist", + "build": "tsc -p tsconfig.build.json", + "test": "vitest --run", + "bench": "node --expose-gc test/bench/all.ts", + "prepublishOnly": "npm run clean && npm run build" + }, + "keywords": [ + "json-schema", + "validation", + "validator", + "csp" + ], + "author": "Earendil Works", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/earendil-works/pi.git", + "directory": "packages/schema" + }, + "engines": { + "node": ">=22.19.0" + }, + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", + "typebox": "1.3.27", + "vitest": "4.1.12", + "zod": "4.6.7" + } +} diff --git a/packages/schema/src/annotations.ts b/packages/schema/src/annotations.ts new file mode 100644 index 000000000..5bc9cc4f5 --- /dev/null +++ b/packages/schema/src/annotations.ts @@ -0,0 +1,208 @@ +// Annotation tracking for `unevaluatedProperties` and `unevaluatedItems`: annotating checks record which +// properties and items a schema evaluated. Only schemas that use those keywords compile into these closures. + +import { isObject } from "./checks.ts"; +import type { Check } from "./types.ts"; + +/** What a schema evaluated: property names (or all of them), the evaluated item prefix, items matched by `contains`. */ +export interface Annotations { + props: Set | null; + allProps: boolean; + items: number; + allItems: boolean; + indexes: Set | null; +} + +/** A check that records what it evaluated into `a` when it passes. */ +export type AnnotatingCheck = (value: unknown, a: Annotations) => boolean; + +/** Empty annotations. */ +export function newAnnotations(): Annotations { + return { props: null, allProps: false, items: 0, allItems: false, indexes: null }; +} + +function merge(a: Annotations, b: Annotations): void { + if (b.props !== null) for (const k of b.props) addProp(a, k); + a.allProps ||= b.allProps; + if (b.items > a.items) a.items = b.items; + a.allItems ||= b.allItems; + if (b.indexes !== null) for (const i of b.indexes) addIndex(a, i); +} + +function addProp(a: Annotations, key: string): void { + if (a.props === null) a.props = new Set(); + a.props.add(key); +} + +function addIndex(a: Annotations, index: number): void { + if (a.indexes === null) a.indexes = new Set(); + a.indexes.add(index); +} + +export const ANNOTATING_TRUE: AnnotatingCheck = () => true; +export const ANNOTATING_FALSE: AnnotatingCheck = () => false; + +/** The indirection a recursive reference goes through. */ +export function indirectAnnotating(slot: { check: AnnotatingCheck | undefined }): AnnotatingCheck { + return (v, a) => (slot.check as AnnotatingCheck)(v, a); +} + +/** A plain check in annotating position: it evaluates nothing that unevaluated* sees. */ +export function ignoreAnnotations(check: Check): AnnotatingCheck { + return (v) => check(v); +} + +/** An annotating check run as a plain check, with fresh annotations. */ +export function withFreshAnnotations(check: AnnotatingCheck): Check { + return (v) => check(v, newAnnotations()); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs. */ +export function scopedAnnotating(scope: string[], uri: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + scope.push(uri); + const ok = check(v, a); + scope.pop(); + return ok; + }; +} + +/** unevaluated* sees only the annotations of its own schema, never those of its siblings in the parent. */ +export function isolateAnnotations(check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (!check(v, b)) return false; + merge(a, b); + return true; + }; +} + +/** The non-annotating keywords, then each annotating part in order. */ +export function annotateAll(plain: Check, parts: readonly AnnotatingCheck[]): AnnotatingCheck { + return (v, a) => { + if (!plain(v)) return false; + for (let i = 0; i < parts.length; i++) if (!parts[i](v, a)) return false; + return true; + }; +} + +/** `properties`, `patternProperties` and `additionalProperties`. */ +export function annotateObject( + props: readonly (readonly [string, Check])[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, +): AnnotatingCheck { + const declared = new Set(props.map(([k]) => k)); + return (v, a) => { + if (!isObject(v)) return true; + const seen: string[] = []; + for (const [k, check] of props) { + if (!Object.hasOwn(v, k)) continue; + if (!check(v[k])) return false; + seen.push(k); + } + for (const k of Object.keys(v)) { + let matched = declared.has(k); + for (const [re, check] of patterns) { + if (re.test(k)) { + if (!check(v[k])) return false; + matched = true; + seen.push(k); + } + } + if (!matched && additional !== undefined) { + if (!additional(v[k])) return false; + seen.push(k); + } + } + for (const k of seen) addProp(a, k); + return true; + }; +} + +/** `prefixItems`/`items`/`additionalItems` and `contains` with its bounds. */ +export function annotateArray( + prefix: readonly Check[], + rest: Check | undefined, + contains: Check | undefined, + minContains: number, + maxContains: number, +): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v)) return true; + const m = Math.min(v.length, prefix.length); + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest !== undefined) { + for (let i = prefix.length; i < v.length; i++) if (!rest(v[i])) return false; + a.allItems = true; + } + if (m > a.items) a.items = m; + if (contains !== undefined) { + let n = 0; + for (let i = 0; i < v.length; i++) { + if (contains(v[i])) { + n++; + addIndex(a, i); + } + } + if (n < minContains || n > maxContains) return false; + } + return true; + }; +} + +/** `anyOf` (or `oneOf` when `exactlyOne`): every passing branch contributes its annotations. */ +export function annotateAny(branches: readonly AnnotatingCheck[], exactlyOne: boolean): AnnotatingCheck { + return (v, a) => { + let n = 0; + for (const branch of branches) { + const b = newAnnotations(); + if (branch(v, b)) { + n++; + merge(a, b); + } + } + return exactlyOne ? n === 1 : n > 0; + }; +} + +/** `if`, `then` and `else`: a passing `if` contributes its annotations even without `then`. */ +export function annotateIf( + cond: AnnotatingCheck, + then: AnnotatingCheck | undefined, + otherwise: AnnotatingCheck | undefined, +): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (cond(v, b)) { + merge(a, b); + return then === undefined || then(v, a); + } + return otherwise === undefined || otherwise(v, a); + }; +} + +/** One `dependentSchemas` entry. */ +export function annotateDependent(key: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => !isObject(v) || !Object.hasOwn(v, key) || check(v, a); +} + +/** `unevaluatedProperties`: every property no other keyword evaluated passes `check`. */ +export function unevaluatedProperties(check: Check): AnnotatingCheck { + return (v, a) => { + if (!isObject(v) || a.allProps) return true; + for (const k of Object.keys(v)) if (!(a.props?.has(k) ?? false) && !check(v[k])) return false; + a.allProps = true; + return true; + }; +} + +/** `unevaluatedItems`: every item no other keyword evaluated passes `check`. */ +export function unevaluatedItems(check: Check): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v) || a.allItems) return true; + for (let i = a.items; i < v.length; i++) if (!(a.indexes?.has(i) ?? false) && !check(v[i])) return false; + a.allItems = true; + return true; + }; +} diff --git a/packages/schema/src/builders.ts b/packages/schema/src/builders.ts new file mode 100644 index 000000000..2dc61a520 --- /dev/null +++ b/packages/schema/src/builders.ts @@ -0,0 +1,200 @@ +// Schema builders: short functions that return plain JSON Schema objects and carry the static type of the values +// they accept, so `Static` is the TypeScript type a valid value has. The objects are ordinary data: +// they serialize with JSON.stringify and compile with compile() like any other schema. + +import type { TypeBoxStatic } from "./typebox-static.ts"; + +declare const STATIC: unique symbol; +declare const OPTIONAL_TYPE: unique symbol; + +/** A JSON Schema object whose valid values have type `T` (a type-level brand; nothing extra exists at run time). */ +export type Schema = { readonly [keyword: string]: unknown } & { readonly [STATIC]?: T }; + +// TypeBox is tested first and in its own branch: a builder brand's fallback through TypeBoxStatic would leave +// `Static` of a generic `S` (constrained to builder schemas) unresolvable to its constraint's types. +/** + * The type of the values `S` accepts: the type a builder recorded; for a TypeBox schema (`'~kind'` or `'~unsafe'`), + * TypeBox's static type for the common kinds (see `TypeBoxStatic`); else `unknown`. + */ +export type Static = S extends { readonly "~kind": unknown } | { readonly "~unsafe": unknown } + ? TypeBoxStatic + : S extends { readonly [STATIC]?: infer T } + ? unknown extends T + ? unknown + : T + : unknown; + +/** Annotations every builder accepts: they describe the schema and never change what it accepts. */ +export interface Annotations { + readonly title?: string; + readonly description?: string; + readonly examples?: readonly T[]; + readonly deprecated?: boolean; + /** A default value, an annotation for tools and forms; validation never inserts it. */ + readonly default?: T; +} + +/** Length bounds of a string, an array or bytes. */ +export interface LengthBounds { + readonly minLength?: number; + readonly maxLength?: number; +} + +/** A string, with optional length bounds (in code points), pattern and format. */ +export function str( + options: Annotations & LengthBounds & { readonly pattern?: string; readonly format?: string } = {}, +): Schema { + return { type: "string", ...options }; +} + +/** Bounds of a number. */ +export interface NumberBounds { + readonly minimum?: number; + readonly maximum?: number; + readonly exclusiveMinimum?: number; + readonly exclusiveMaximum?: number; + readonly multipleOf?: number; +} + +/** A number within optional bounds. */ +export function num(options: Annotations & NumberBounds = {}): Schema { + return { type: "number", ...options }; +} + +/** An integer within optional bounds. */ +export function int(options: Annotations & NumberBounds = {}): Schema { + return { type: "integer", ...options }; +} + +/** A boolean. */ +export function bool(options: Annotations = {}): Schema { + return { type: "boolean", ...options }; +} + +/** `null`. */ +export function nul(options: Annotations = {}): Schema { + return { type: "null", ...options }; +} + +/** Any value. */ +export function unknown(options: Annotations = {}): Schema { + return { ...options }; +} + +/** Exactly `value` (a string, number, boolean or null). */ +export function lit( + value: V, + options: Annotations = {}, +): Schema { + return { const: value, ...options }; +} + +/** One of the given values. */ +export function oneOf( + values: V, + options: Annotations = {}, +): Schema { + return { enum: values, ...options }; +} + +/** + * Any of the given schemas. A union of objects tagged by a `lit()` property (`{ kind: lit("text"), ... }`) is checked + * by the tag: only the branch it selects runs, and its errors are the ones reported. + */ +export function union( + schemas: S, + options: Annotations> = {}, +): Schema> { + return { anyOf: schemas, ...options }; +} + +/** `schema` or `null`. */ +export function nullable(schema: Schema, options: Annotations = {}): Schema { + return { anyOf: [schema, { type: "null" }], ...options }; +} + +/** An array of `items`, with optional length bounds and unique items. */ +export function arr( + items: Schema, + options: Annotations & { + readonly minItems?: number; + readonly maxItems?: number; + readonly uniqueItems?: boolean; + } = {}, +): Schema { + return { type: "array", items, ...options }; +} + +/** A fixed-length array whose items match `items` in order. */ +export function tuple( + items: S, + options: Annotations<{ -readonly [K in keyof S]: Static }> = {}, +): Schema<{ -readonly [K in keyof S]: Static }> { + return { type: "array", prefixItems: items, items: false, minItems: items.length, ...options }; +} + +/** An object with string keys and values of `value` (a record). */ +export function map(value: Schema, options: Annotations> = {}): Schema> { + return { type: "object", additionalProperties: value, ...options }; +} + +/** A Uint8Array with optional length bounds (an extension of JSON Schema for values that never pass through JSON). */ +export function bytes(options: Annotations & LengthBounds = {}): Schema { + return { type: "bytes", ...options }; +} + +/** An optional property of obj(): `{ name: opt(str()) }` may omit `name` or set it to undefined. */ +export class Optional { + declare readonly [OPTIONAL_TYPE]?: S; + readonly schema: S; + + constructor(schema: S) { + this.schema = schema; + } +} + +/** Marks an obj() property as optional. */ +export function opt(schema: S): Optional { + return new Optional(schema); +} + +/** The properties of obj(): a schema or an opt() schema per name. */ +export type Properties = { readonly [name: string]: Schema | Optional }; + +/** The value type of an obj() schema. */ +export type StaticObject

= Simplify< + { -readonly [K in keyof P as P[K] extends Optional ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? K : never]?: P[K] extends Optional + ? Static + : never; + } +>; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +/** + * A closed object: the given properties (required unless opt()) and no others. `open: true` allows other + * properties, as tool schemas from other systems often do. + */ +export function obj( + properties: P, + options: Annotations> & { readonly open?: boolean } = {}, +): Schema> { + const { open, ...annotations } = options; + const props: Record = {}; + const required: string[] = []; + for (const [name, property] of Object.entries(properties)) { + if (property instanceof Optional) props[name] = property.schema; + else { + props[name] = property; + required.push(name); + } + } + return { + type: "object", + properties: props, + ...(required.length > 0 ? { required } : {}), + ...(open === true ? {} : { additionalProperties: false }), + ...annotations, + }; +} diff --git a/packages/schema/src/checks.ts b/packages/schema/src/checks.ts new file mode 100644 index 000000000..222d3dbde --- /dev/null +++ b/packages/schema/src/checks.ts @@ -0,0 +1,451 @@ +// Check factories: each returns a closure that captures only its arguments, never the compiler, so a compiled +// schema keeps no compile-time state alive. Leaf checks are module constants that object and array loops inline. + +import type { Check } from "./types.ts"; + +export const ALWAYS: Check = () => true; +export const NEVER: Check = () => false; +const IS_STRING: Check = (v) => typeof v === "string"; +const IS_NUMBER: Check = (v) => typeof v === "number"; +const IS_INTEGER: Check = (v) => Number.isInteger(v); +const IS_BOOLEAN: Check = (v) => typeof v === "boolean"; +const IS_NULL: Check = (v) => v === null; +export const IS_ARRAY: Check = (v) => Array.isArray(v); +const IS_BYTES: Check = (v) => v instanceof Uint8Array; +export const IS_OBJECT: Check = (v) => isObject(v); + +/** Whether `v` is a JSON object: not null, an array or bytes. */ +export function isObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v) && !(v instanceof Uint8Array); +} + +/** The check of a `type` name; `bytes` (a Uint8Array) extends JSON Schema for values that never pass through JSON. */ +export const TYPE_CHECKS: ReadonlyMap = new Map([ + ["string", IS_STRING], + ["number", IS_NUMBER], + ["integer", IS_INTEGER], + ["boolean", IS_BOOLEAN], + ["null", IS_NULL], + ["array", IS_ARRAY], + ["object", IS_OBJECT], + ["bytes", IS_BYTES], +]); + +/** Leaf checks that object and array loops run inline by code instead of calling them. */ +export const LEAF_CODES: ReadonlyMap = new Map([ + [IS_STRING, 1], + [IS_NUMBER, 2], + [IS_INTEGER, 3], + [IS_BOOLEAN, 4], + [IS_NULL, 5], + [ALWAYS, 6], +]); + +function leafOk(code: number, x: unknown): boolean { + switch (code) { + case 1: + return typeof x === "string"; + case 2: + return typeof x === "number"; + case 3: + return Number.isInteger(x); + case 4: + return typeof x === "boolean"; + case 5: + return x === null; + default: + return true; + } +} + +/** Whether a combined object or array check also checks the type: not at all, strictly, or passing other types. */ +export const GUARD_NONE = 0; +export const GUARD_STRICT = 1; +export const GUARD_LOOSE = 2; +export type Guard = typeof GUARD_NONE | typeof GUARD_STRICT | typeof GUARD_LOOSE; + +/** Flags of a declared property in an object check's entries: its leaf code and two bits. */ +export const CODE = 7; +export const REQUIRED = 8; +export const INHERITED = 16; + +/** All of `parts`; two and three parts get closures without a loop. */ +export function and(parts: readonly Check[]): Check { + switch (parts.length) { + case 0: + return ALWAYS; + case 1: + return parts[0]; + case 2: { + const [a, b] = parts; + return (v) => a(v) && b(v); + } + case 3: { + const [a, b, c] = parts; + return (v) => a(v) && b(v) && c(v); + } + default: + return (v) => { + for (let i = 0; i < parts.length; i++) if (!parts[i](v)) return false; + return true; + }; + } +} + +/** A check whose implementation is set later: the indirection a recursive reference goes through. */ +export function indirect(slot: { check: Check | undefined }): Check { + return (v) => (slot.check as Check)(v); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs (only for schemas with `$dynamicRef`). */ +export function scopedCheck(scope: string[], uri: string, check: Check): Check { + return (v) => { + scope.push(uri); + const ok = check(v); + scope.pop(); + return ok; + }; +} + +/** Any of the named types. */ +export function typeCheck(types: readonly unknown[]): Check { + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (checks.length === 1) return checks[0]; + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** A Uint8Array within the length bounds. */ +export function bytesCheck(min: number, max: number): Check { + return (v) => v instanceof Uint8Array && v.length >= min && v.length <= max; +} + +/** The number of Unicode code points of `s`: JSON Schema string lengths count code points, not UTF-16 units. */ +export function codePoints(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++, n++) { + const c = s.charCodeAt(i); + // A high surrogate starts a pair only when a low surrogate follows; an unpaired one is a code point of its own. + if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) { + const next = s.charCodeAt(i + 1); + if (next >= 0xdc00 && next <= 0xdfff) i++; + } + } + return n; +} + +// UTF-16 units bound code points from both sides (units / 2 <= points <= units), so most strings need no count. +function lengthOk(s: string, min: number, max: number): boolean { + const units = s.length; + if (units < min) return false; + if (units < 2 * min && codePoints(s) < min) return false; + return units <= max || codePoints(s) <= max; +} + +/** The string keywords: length bounds, pattern and format. */ +export function stringCheck( + strict: boolean, + min: number, + max: number, + re: RegExp | undefined, + format: ((value: string) => boolean) | undefined, +): Check { + if (re !== undefined && min === 0 && max === Number.POSITIVE_INFINITY && format === undefined) { + return strict ? (v) => typeof v === "string" && re.test(v) : (v) => typeof v !== "string" || re.test(v); + } + if (re === undefined && format === undefined) { + return strict + ? (v) => typeof v === "string" && lengthOk(v, min, max) + : (v) => typeof v !== "string" || lengthOk(v, min, max); + } + return (v) => { + if (typeof v !== "string") return !strict; + return lengthOk(v, min, max) && (re === undefined || re.test(v)) && (format === undefined || format(v)); + }; +} + +/** The number keywords. `mode`: 0 passes other types, 1 requires a number, 2 an integer. */ +export function numberCheck( + mode: 0 | 1 | 2, + min: number, + max: number, + xmin: number, + xmax: number, + multipleOf: number | undefined, +): Check { + return (v) => { + if (typeof v !== "number") return mode === 0; + if (mode === 2 && !Number.isInteger(v)) return false; + return v >= min && v <= max && v > xmin && v < xmax && (multipleOf === undefined || isMultiple(v, multipleOf)); + }; +} + +/** Whether `v` is a multiple of `d`, exactly for decimals: 0.0075 / 0.0001 is 74.99999999999999 in floating point. */ +export function isMultiple(v: number, d: number): boolean { + const q = v / d; + if (Number.isInteger(q)) return true; + if (!Number.isFinite(q)) return v % d === 0; + const [vc, ve] = decimal(v); + const [dc, de] = decimal(d); + return ve >= de ? (vc * 10n ** BigInt(ve - de)) % dc === 0n : vc % (dc * 10n ** BigInt(de - ve)) === 0n; +} + +function decimal(x: number): [bigint, number] { + const [mantissa, exponent = "0"] = Math.abs(x).toString().split("e"); + const point = mantissa.indexOf("."); + const fraction = point < 0 ? 0 : mantissa.length - point - 1; + return [BigInt(mantissa.replace(".", "")), Number(exponent) - fraction]; +} + +/** `inner` for objects; other values pass unless `strict`. */ +export function guardObject(inner: Check, strict: boolean): Check { + return strict ? (v) => isObject(v) && inner(v) : (v) => !isObject(v) || inner(v); +} + +/** `inner` for arrays; other values pass unless `strict`. */ +export function guardArray(inner: Check, strict: boolean): Check { + return strict ? (v) => Array.isArray(v) && inner(v) : (v) => !Array.isArray(v) || inner(v); +} + +/** + * An object with declared properties: `entries` is `[key, check, flags]` per property. The value's own keys are + * walked with for...in (whose `o[k]` V8 reads from the enum cache, unlike a load by a name from a list), each found + * among the declared names; `closed` rejects any other key. A property whose value is `undefined` counts as absent. + */ +export function declaredObject( + entries: readonly unknown[], + requiredCount: number, + closed: boolean, + guard: Guard, +): Check { + const n = entries.length; + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + // Guarded: by the check above, or by the caller for GUARD_NONE. + const o = v as Record; + let required = 0; + for (const k in o) { + let i = 0; + while (i < n && entries[i] !== k) i += 3; + if (i === n) { + if (closed) return false; + continue; + } + const x = o[k]; + if (x === undefined) continue; + const f = entries[i + 2] as number; + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + if (f & REQUIRED) required++; + } + return required === requiredCount; + }; +} + +/** An object with declared properties, required keys not declared, pattern properties and additional properties. */ +export function openObject( + entries: readonly unknown[], + keys: readonly string[], + extraRequired: readonly string[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, + guard: Guard, +): Check { + const known = patterns.length > 0 || additional !== undefined ? new Set(keys) : undefined; + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + const o = v as Record; + for (let i = 0; i < entries.length; i += 3) { + const k = entries[i] as string; + const x = o[k]; + const f = entries[i + 2] as number; + if (x === undefined || (f & INHERITED && !Object.hasOwn(o, k))) { + if (f & REQUIRED) return false; + continue; + } + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + } + for (let i = 0; i < extraRequired.length; i++) if (!Object.hasOwn(o, extraRequired[i])) return false; + if (known === undefined) return true; + for (const k of Object.keys(o)) { + let matched = known.has(k); + for (let i = 0; i < patterns.length; i++) { + if (patterns[i][0].test(k)) { + matched = true; + if (!patterns[i][1](o[k])) return false; + } + } + if (!matched && additional !== undefined && !additional(o[k])) return false; + } + return true; + }; +} + +/** Every property name passes `check` (`v` is an object). */ +export function propertyNamesCheck(check: Check): Check { + return (v) => { + for (const k in v as object) if (!check(k)) return false; + return true; + }; +} + +/** `minProperties` and `maxProperties` (`v` is an object). */ +export function propertyCountCheck(min: number, max: number): Check { + return (v) => { + let n = 0; + for (const _ in v as object) n++; + return n >= min && n <= max; + }; +} + +/** `dependentRequired` (`v` is an object). */ +export function dependentRequiredCheck(entries: readonly (readonly [string, readonly string[]])[]): Check { + return (v) => { + const o = v as object; + for (const [k, list] of entries) + if (Object.hasOwn(o, k)) for (const r of list) if (!Object.hasOwn(o, r)) return false; + return true; + }; +} + +/** `dependentSchemas` (`v` is an object). */ +export function dependentSchemasCheck(entries: readonly (readonly [string, Check])[]): Check { + return (v) => { + for (const [k, check] of entries) if (Object.hasOwn(v as object, k) && !check(v)) return false; + return true; + }; +} + +/** The common array: guard, length bounds, prefix items and the items after them, in one closure. */ +export function listCheck( + guard: Guard, + min: number, + max: number, + prefix: readonly Check[], + rest: Check | undefined, +): Check { + const n = prefix.length; + const code = rest === undefined ? 0 : (LEAF_CODES.get(rest) ?? 0); + return (v) => { + if (!Array.isArray(v)) return guard === GUARD_LOOSE; + const length = v.length; + if (length < min || length > max) return false; + const m = length < n ? length : n; + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest === undefined) return true; + if (code !== 0) { + for (let i = n; i < length; i++) if (!leafOk(code, v[i])) return false; + return true; + } + for (let i = n; i < length; i++) if (!rest(v[i])) return false; + return true; + }; +} + +/** Prefix items and the items after them (`v` is an array). */ +export function itemsCheck(prefix: readonly Check[], rest: Check | undefined): Check { + return listCheck(GUARD_NONE, 0, Number.POSITIVE_INFINITY, prefix, rest); +} + +/** `minItems` and `maxItems` (`v` is an array). */ +export function lengthCheck(min: number, max: number): Check { + return (v) => (v as unknown[]).length >= min && (v as unknown[]).length <= max; +} + +/** `contains` with its bounds (`v` is an array). */ +export function containsCheck(check: Check, min: number, max: number): Check { + return (v) => { + const list = v as unknown[]; + let n = 0; + for (let i = 0; i < list.length; i++) if (check(list[i]) && ++n > max) return false; + return n >= min; + }; +} + +/** `uniqueItems` (`v` is an array). */ +export function uniqueCheck(v: unknown): boolean { + const list = v as unknown[]; + for (let i = 1; i < list.length; i++) for (let j = 0; j < i; j++) if (deepEqual(list[i], list[j])) return false; + return true; +} + +/** `enum`: up to 8 primitives by a scan, more by a Set, objects and arrays by deep equality. */ +export function enumCheck(values: readonly unknown[]): Check { + const primitives = values.filter((x) => typeof x !== "object" || x === null); + if (primitives.length === values.length && primitives.length <= 8) { + return (v) => { + for (let i = 0; i < primitives.length; i++) if (primitives[i] === v) return true; + return false; + }; + } + const set = new Set(primitives); + const objects = values.filter((x) => typeof x === "object" && x !== null); + return (v) => set.has(v) || (typeof v === "object" && objects.some((o) => deepEqual(v, o))); +} + +/** `const`. */ +export function constCheck(value: unknown): Check { + if (typeof value !== "object" || value === null) return (v) => v === value; + return (v) => deepEqual(v, value); +} + +/** `anyOf`. */ +export function anyCheck(checks: readonly Check[]): Check { + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** `oneOf`. */ +export function oneCheck(checks: readonly Check[]): Check { + return (v) => { + let n = 0; + for (let i = 0; i < checks.length; i++) if (checks[i](v) && ++n > 1) return false; + return n === 1; + }; +} + +/** `not`. */ +export function notCheck(check: Check): Check { + return (v) => !check(v); +} + +/** `if`, `then` and `else`. */ +export function ifCheck(cond: Check, then: Check, otherwise: Check): Check { + return (v) => (cond(v) ? then(v) : otherwise(v)); +} + +/** A union of objects tagged by a const on `key`: the tag selects the one branch that can match. */ +export function taggedCheck(key: string, tags: readonly unknown[], checks: readonly Check[]): Check { + return (v) => { + if (!isObject(v)) return false; + const tag = v[key]; + for (let i = 0; i < tags.length; i++) if (tags[i] === tag) return checks[i](v); + return false; + }; +} + +/** JSON equality: same primitives, arrays with equal items in order, objects with the same keys and equal values. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false; + const array = Array.isArray(a); + if (array !== Array.isArray(b)) return false; + if (array) { + const x = a as unknown[]; + const y = b as unknown[]; + if (x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (!deepEqual(x[i], y[i])) return false; + return true; + } + const x = a as Record; + const y = b as Record; + const keys = Object.keys(x); + if (keys.length !== Object.keys(y).length) return false; + for (const k of keys) if (!Object.hasOwn(y, k) || !deepEqual(x[k], y[k])) return false; + return true; +} diff --git a/packages/schema/src/coerce.ts b/packages/schema/src/coerce.ts new file mode 100644 index 000000000..5dee2487f --- /dev/null +++ b/packages/schema/src/coerce.ts @@ -0,0 +1,556 @@ +// Coercion: turns a loosely typed value (a model's tool call arguments, form input, query strings) into the value a +// schema most likely meant, before it is checked. Only validators compiled with `{ coerce: true }` have it, and they +// build it on their first coerce() or parse(); the check path never runs it. +// +// Three passes, in this order: +// 1. Optional nulls: a `null` in an optional property whose schema rejects null is dropped, as if omitted. +// 2. Conversion rules ported from TypeBox 1.3.27's Value.Convert (MIT), keyed on JSON Schema structure (`$ref`, `const`, +// `enum`, `anyOf`/`oneOf`, `allOf`, `type`, `items`/`prefixItems`, `properties`/`patternProperties`) instead of +// TypeBox's kind marks: `null` becomes "null" for a string, `"3.5"` becomes 3 for an integer, a single value becomes +// a one-item array, and a union takes the first branch whose conversion passes. +// 3. Primitive coercion by `type` (also type arrays), as in Ajv's `coerceTypes`: what pass 2 left, for example a type +// array `["integer", "null"]`, or `""` for `null`. +// A value that cannot be coerced is left as it is; the check then reports it. +// +// Every pass is copy-on-write: an object or array is copied only when one of its members changes, so an unchanged +// value comes back as the same reference and the input is never modified. All passes walk the same keywords for +// members: `properties`, then `patternProperties`, then `additionalProperties` for keys neither names; tuple items by +// `prefixItems` (or a draft-7 `items` array), the rest by `items`. Draft 7's `additionalItems` is not coerced. + +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject } from "./types.ts"; + +/** Coerces values for one compiled schema. */ +export class Coercer { + private readonly c: Compiler; + private readonly schema: unknown; + /** + * The (schema node, value) pairs pass 2 is converting through a re-entry point, as a stack. Conversion can enter + * a node again with the same value only through a `$ref` or by wrapping a single value into an array (also for + * schemas that cycle through shared JS objects): a recursive schema that meets a value no branch accepts would + * convert it without end. A pair already on the stack (primitives by value, objects by reference) leaves the value + * unchanged. Only those two points push, so schemas without references or wraps pay nothing. + */ + private readonly entered: unknown[] = []; + /** How often a re-entry was cut short in this coerce() call; results computed across one are not memoized. */ + private cuts = 0; + /** + * Results of `$ref` targets by (node, value) within one coerce() call: a recursive union whose branches all + * convert the same children (and all fail) would convert them once per branch, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + } + + /** `value` coerced: a copy along the changed paths only, or `value` itself when nothing changes. */ + coerce(value: unknown): unknown { + const base = this.c.r.rootBase; + const withoutNulls = this.dropOptionalNulls(value, this.schema, base); + return this.coerceTypes(this.convertOnly(withoutNulls), this.schema, base); + } + + /** Pass 2 alone, as TypeBox's `Value.Convert` (the TypeBox conformance tests run it). */ + convertOnly(value: unknown): unknown { + try { + return this.convert(this.schema, this.c.r.rootBase, value); + } finally { + this.memo = undefined; + this.cuts = 0; + } + } + + /** `node` converted for `value` at a re-entry point: unchanged when that pair is already being converted. */ + private reenter(node: object, base: string, value: unknown, memoize: boolean): unknown { + const entered = this.entered; + for (let i = 0; i < entered.length; i += 2) { + if (entered[i] === node && Object.is(entered[i + 1], value)) { + this.cuts++; + return value; + } + } + const memo = memoize ? this.memo?.get(node) : undefined; + if (memo?.has(value)) return memo.get(value); + const cuts = this.cuts; + entered.push(node, value); + let result: unknown; + try { + result = this.convert(node, base, value); + } finally { + entered.length -= 2; + } + if (memoize && this.cuts === cuts) { + this.memo ??= new Map(); + let values = this.memo.get(node); + if (values === undefined) { + values = new Map(); + this.memo.set(node, values); + } + values.set(value, result); + } + return result; + } + + /** Whether `value` passes the schema `node`; undefined when `node` cannot be compiled. */ + private passes(node: unknown, base: string, value: unknown): boolean | undefined { + try { + return this.c.check(node, this.baseOf(node, base))(value); + } catch { + return undefined; + } + } + + private baseOf(node: unknown, base: string): string { + return isSchemaObject(node) ? (this.c.r.baseOf.get(node) ?? base) : base; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 1: optional nulls + // --------------------------------------------------------------------------------------------------------------- + + private dropOptionalNulls(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + if (Array.isArray(value)) { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return itemSchema === undefined ? item : this.dropOptionalNulls(item, itemSchema, b); + }); + } + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + if (patterns.length === 0 && additional === undefined && !isSchemaObject(schema.properties)) return value; + const required = new Set(Array.isArray(schema.required) ? schema.required : []); + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (!Object.hasOwn(value, key)) continue; + const isRef = isSchemaObject(property) && typeof property.$ref === "string"; + if (value[key] === null && !required.has(key) && !isRef && this.passes(property, b, null) === false) { + out ??= { ...value }; + delete out[key]; + continue; + } + set(key, this.dropOptionalNulls(value[key], property, b)); + } + // Values of pattern and additional properties are walked for their own optional nulls; the keys themselves + // are not optional properties, so they stay. + if (patterns.length > 0 || additional !== undefined) { + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.dropOptionalNulls((out ?? value)[key], item, b)); + } + if (!matched && additional !== undefined) set(key, this.dropOptionalNulls(value[key], additional, b)); + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 2: TypeBox's conversion rules + // --------------------------------------------------------------------------------------------------------------- + + private convert(node: unknown, base: string, value: unknown): unknown { + if (!isSchemaObject(node)) return value; + const b = this.baseOf(node, base); + if (typeof node.$ref === "string") { + let target: { node: unknown; base: string }; + try { + target = this.c.r.resolve(node.$ref, b); + } catch { + return value; + } + return isSchemaObject(target.node) ? this.reenter(target.node, target.base, value, true) : value; + } + return this.convertNode(node, b, value); + } + + /** One schema node's conversion: its first keyword among `const`, `enum`, the unions, `allOf` and `type`. */ + private convertNode(node: SchemaObject, b: string, value: unknown): unknown { + if (Object.hasOwn(node, "const")) return convertLiteral(node.const, value); + if (Array.isArray(node.enum)) return this.convertEnum(node, node.enum, b, value); + // A union or intersection node may also declare members (plain JSON Schema; TypeBox's never do): they convert + // after the branches. + if (Array.isArray(node.anyOf)) return this.convertUnion(node, node.anyOf, b, value); + if (Array.isArray(node.oneOf)) return this.convertUnion(node, node.oneOf, b, value); + if (Array.isArray(node.allOf)) { + let next = value; + for (const member of node.allOf) next = this.convert(member, b, next); + return this.convertMembers(node, b, next); + } + switch (node.type) { + case "string": + return orValue(tryString(value), value); + case "number": + return orValue(tryNumber(value), value); + case "integer": { + const result = tryNumber(value); + return result === undefined ? value : Math.trunc(result.value); + } + case "boolean": + return orValue(tryBoolean(value), value); + case "null": + return orValue(tryNull(value), value); + case "array": + return this.convertArray(node, b, value); + case "object": + return this.convertObject(node, b, value); + case undefined: + if (node.properties !== undefined || node.patternProperties !== undefined) { + return this.convertObject(node, b, value); + } + if (node.items !== undefined || node.prefixItems !== undefined) return this.convertArray(node, b, value); + return value; + default: + return value; + } + } + + /** An enum is a union of its values: a member stays, else the first value the input converts to. */ + private convertEnum(node: SchemaObject, values: readonly unknown[], base: string, value: unknown): unknown { + if (values.some((member) => member === value)) return value; + for (const member of values) { + const candidate = convertLiteral(member, value); + if (candidate !== value && this.passes(node, base, candidate) === true) return candidate; + } + return value; + } + + /** + * A value one branch accepts stays; else the first branch's conversion that the whole union accepts. Branches are + * converted in order and only until one is accepted, so a later recursive branch is not entered needlessly. A node + * that also declares members (plain JSON Schema; TypeBox's never do) converts them into each candidate before it is + * checked, so a branch is not discarded for a sibling property that is converted after it. + */ + private convertUnion(node: SchemaObject, branches: readonly unknown[], base: string, value: unknown): unknown { + if (branches.some((branch) => this.passes(branch, base, value) === true)) + return this.convertMembers(node, base, value); + for (const branch of branches) { + const candidate = this.convertMembers(node, base, this.convert(branch, base, value)); + if (this.passes(node, base, candidate) === true) return candidate; + } + return this.convertMembers(node, base, value); + } + + /** The object members of a union or intersection node, when it declares any. */ + private convertMembers(node: SchemaObject, base: string, value: unknown): unknown { + const declares = + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined; + return declares ? this.convertObject(node, base, value) : value; + } + + private convertArray(node: SchemaObject, base: string, value: unknown): unknown { + const prefix = Array.isArray(node.items) + ? node.items + : Array.isArray(node.prefixItems) + ? node.prefixItems + : undefined; + if (prefix !== undefined) { + // A tuple converts its items and never wraps a single value. + if (!Array.isArray(value)) return value; + const rest = Array.isArray(node.items) ? undefined : node.items; + return mapItems(value, (item, i) => { + const schema = i < prefix.length ? prefix[i] : rest; + return schema === undefined ? item : this.convert(schema, base, item); + }); + } + if (Array.isArray(value)) { + return node.items === undefined ? value : mapItems(value, (item) => this.convert(node.items, base, item)); + } + // The wrapped single value is converted by the item schema with the same value: a re-entry point. + const item = node.items; + return [isSchemaObject(item) ? this.reenter(item, base, value, false) : value]; + } + + private convertObject(node: SchemaObject, base: string, value: unknown): unknown { + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const patterns = isSchemaObject(node.patternProperties) + ? Object.entries(node.patternProperties).map(([pattern, schema]) => [sharedRegex(pattern), schema] as const) + : []; + const additional = isSchemaObject(node.additionalProperties) ? node.additionalProperties : undefined; + let out: Record | undefined; + for (const key of Object.keys(value)) { + // An undefined property counts as absent. + if (value[key] === undefined) continue; + let next: unknown = value[key]; + let matched = Object.hasOwn(properties, key); + if (matched) next = this.convert(properties[key], base, next); + for (const [pattern, schema] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + next = this.convert(schema, base, next); + } + if (!matched && additional !== undefined) next = this.convert(additional, base, next); + if (next !== value[key]) { + out ??= { ...value }; + out[key] = next; + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 3: primitive coercion by type + // --------------------------------------------------------------------------------------------------------------- + + private coerceTypes(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + let next = value; + if (Array.isArray(schema.allOf)) for (const member of schema.allOf) next = this.coerceTypes(next, member, b); + if (Array.isArray(schema.anyOf)) next = this.coerceUnion(next, schema.anyOf, b); + if (Array.isArray(schema.oneOf)) next = this.coerceUnion(next, schema.oneOf, b); + const types = typesOf(schema); + const matchesUnionMember = types.length > 1 && types.some((type) => matchesType(next, type)); + if (types.length > 0 && !matchesUnionMember) { + for (const type of types) { + const candidate = coercePrimitive(next, type); + if (candidate !== next) { + next = candidate; + break; + } + } + } + if (types.includes("object") && isPlainObject(next)) next = this.coerceProperties(next, schema, b); + if (types.includes("array") && Array.isArray(next)) next = this.coerceItems(next, schema, b); + return next; + } + + private coerceUnion(value: unknown, branches: readonly unknown[], base: string): unknown { + for (const branch of branches) if (this.passes(branch, base, value) === true) return value; + for (const branch of branches) { + const coerced = this.coerceTypes(value, branch, base); + if (this.passes(branch, base, coerced) === true) return coerced; + } + return value; + } + + private coerceProperties( + value: Record, + schema: SchemaObject, + base: string, + ): Record { + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (key in value) set(key, this.coerceTypes(value[key], property, base)); + } + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.coerceTypes((out ?? value)[key], item, base)); + } + if (!matched && additional !== undefined) set(key, this.coerceTypes(value[key], additional, base)); + } + return out ?? value; + } + + private coerceItems(value: unknown[], schema: SchemaObject, base: string): unknown[] { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return isSchemaObject(itemSchema) ? this.coerceTypes(item, itemSchema, base) : item; + }); + } +} + +/** + * The schema of an array's item at `index`: a tuple's (`prefixItems`, or a draft-7 `items` array) for its prefix, + * `items` after `prefixItems`, or `items` for every item. Undefined when nothing applies. + */ +function itemSchemaAt(schema: SchemaObject, index: number): unknown { + if (Array.isArray(schema.items)) return schema.items[index]; + if (Array.isArray(schema.prefixItems) && index < schema.prefixItems.length) return schema.prefixItems[index]; + return schema.items; +} + +/** `items` mapped by `map`; the same array when every item maps to itself. */ +function mapItems(items: unknown[], map: (item: unknown, index: number) => unknown): unknown[] { + let out: unknown[] | undefined; + for (let i = 0; i < items.length; i++) { + const next = map(items[i], i); + if (next === items[i]) continue; + out ??= items.slice(); + out[i] = next; + } + return out ?? items; +} + +// ------------------------------------------------------------------------------------------------------------------- +// TypeBox's conversions of one value (value/convert/try in TypeBox 1.3.27): a result, or undefined when there is none +// ------------------------------------------------------------------------------------------------------------------- + +type Converted = { readonly value: T } | undefined; + +function orValue(result: Converted, value: unknown): unknown { + return result === undefined ? value : result.value; +} + +/** A literal (`const`, an enum value) taken from a value of another type that converts to exactly it. */ +function convertLiteral(literal: unknown, value: unknown): unknown { + if (literal === value) return value; + const result = + typeof literal === "string" + ? tryString(value) + : typeof literal === "number" + ? tryNumber(value) + : typeof literal === "boolean" + ? tryBoolean(value) + : undefined; + return result !== undefined && result.value === literal ? result.value : value; +} + +function tryString(value: unknown): Converted { + if (typeof value === "bigint" || typeof value === "boolean" || isFiniteNumber(value)) + return { value: String(value) }; + if (value === null) return { value: "null" }; + if (typeof value === "string") return { value }; + if (value === undefined) return { value: "" }; + return undefined; +} + +const MAX_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +const MIN_BIGINT = BigInt(Number.MIN_SAFE_INTEGER); + +function tryNumber(value: unknown): Converted { + if (typeof value === "bigint") + return value <= MAX_BIGINT && value >= MIN_BIGINT ? { value: Number(value) } : undefined; + if (typeof value === "boolean") return { value: value ? 1 : 0 }; + if (isFiniteNumber(value)) return { value }; + if (value === null || value === undefined) return { value: 0 }; + if (typeof value !== "string") return undefined; + const coerced = +value; + if (Number.isFinite(coerced)) return { value: coerced }; + const lowercase = value.toLowerCase(); + if (lowercase === "false") return { value: 0 }; + if (lowercase === "true") return { value: 1 }; + const big = bigIntOfString(value); + return big !== undefined && big <= MAX_BIGINT && big >= MIN_BIGINT ? { value: Number(big) } : undefined; +} + +/** TypeBox's TryBigInt of a string: `12n`, `12.5` (truncated) or `12`; true and false are not reached from tryNumber. */ +function bigIntOfString(value: string): bigint | undefined { + if (/^-?(0|[1-9]\d*)n$/.test(value)) return BigInt(value.slice(0, -1)); + if (/^-?(0|[1-9]\d*)\.\d+$/.test(value)) return BigInt(value.split(".")[0]); + if (/^-?(0|[1-9]\d*)$/.test(value)) return BigInt(value); + return undefined; +} + +function tryBoolean(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: false } : value === 1n ? { value: true } : undefined; + if (typeof value === "boolean") return { value }; + if (isFiniteNumber(value)) return value === 0 ? { value: false } : value === 1 ? { value: true } : undefined; + if (value === null || value === undefined) return { value: false }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + if (lowercase === "false" || value === "0") return { value: false }; + if (lowercase === "true" || value === "1") return { value: true }; + return undefined; +} + +function tryNull(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: null } : undefined; + if (typeof value === "boolean") return value === false ? { value: null } : undefined; + if (isFiniteNumber(value)) return value === 0 ? { value: null } : undefined; + if (value === null || value === undefined) return { value: null }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + return lowercase === "undefined" || lowercase === "null" || value === "" || value === "0" + ? { value: null } + : undefined; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +// ------------------------------------------------------------------------------------------------------------------- +// Primitive coercion by type (pass 3) +// ------------------------------------------------------------------------------------------------------------------- + +function typesOf(schema: SchemaObject): string[] { + if (typeof schema.type === "string") return [schema.type]; + if (Array.isArray(schema.type)) return schema.type.filter((type): type is string => typeof type === "string"); + return []; +} + +function matchesType(value: unknown, type: string): boolean { + switch (type) { + case "number": + return typeof value === "number"; + case "integer": + return typeof value === "number" && Number.isInteger(value); + case "boolean": + return typeof value === "boolean"; + case "string": + return typeof value === "string"; + case "null": + return value === null; + case "array": + return Array.isArray(value); + case "object": + return isPlainObject(value); + default: + return false; + } +} + +function coercePrimitive(value: unknown, type: string): unknown { + switch (type) { + case "number": + case "integer": { + if (value === null) return 0; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + if (type === "number" ? Number.isFinite(parsed) : Number.isInteger(parsed)) return parsed; + } + if (typeof value === "boolean") return value ? 1 : 0; + return value; + } + case "boolean": + if (value === null) return false; + if (value === "true" || value === 1) return true; + if (value === "false" || value === 0) return false; + return value; + case "string": + if (value === null) return ""; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return value; + case "null": + return value === "" || value === 0 || value === false ? null : value; + default: + return value; + } +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/schema/src/compiler.ts b/packages/schema/src/compiler.ts new file mode 100644 index 000000000..8140e38d5 --- /dev/null +++ b/packages/schema/src/compiler.ts @@ -0,0 +1,593 @@ +// The compiler: a schema to one check closure, built once. Keywords group by the type they constrain, so a value +// is classified once per schema, and the common shapes (closed objects, arrays of one item schema, tagged unions, +// leaf schemas) get one specialized closure each. + +import { + ANNOTATING_FALSE, + ANNOTATING_TRUE, + type AnnotatingCheck, + annotateAll, + annotateAny, + annotateArray, + annotateDependent, + annotateIf, + annotateObject, + ignoreAnnotations, + indirectAnnotating, + isolateAnnotations, + scopedAnnotating, + unevaluatedItems, + unevaluatedProperties, + withFreshAnnotations, +} from "./annotations.ts"; +import { + ALWAYS, + and, + anyCheck, + bytesCheck, + constCheck, + containsCheck, + declaredObject, + dependentRequiredCheck, + dependentSchemasCheck, + enumCheck, + GUARD_LOOSE, + GUARD_NONE, + GUARD_STRICT, + type Guard, + guardArray, + guardObject, + INHERITED, + IS_ARRAY, + IS_OBJECT, + ifCheck, + indirect, + itemsCheck, + LEAF_CODES, + lengthCheck, + listCheck, + NEVER, + notCheck, + numberCheck, + oneCheck, + openObject, + propertyCountCheck, + propertyNamesCheck, + REQUIRED, + scopedCheck, + stringCheck, + TYPE_CHECKS, + taggedCheck, + typeCheck, + uniqueCheck, +} from "./checks.ts"; +import { documentOf, isSchemaObject, Resolver, resolveUri } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { Check, CompileOptions, FormatCheck, JsonSchema, SchemaObject } from "./types.ts"; + +/** Thrown when a compile without dynamic scope tracking meets a bookended `$dynamicRef`: compile again with it. */ +export const RESTART = Symbol("restart"); + +/** One compile of a schema. Its caches and indexes are garbage once the root check is built. */ +export class Compiler { + readonly r: Resolver; + readonly draft: "2020-12" | "7"; + readonly formats: Readonly> | undefined; + /** Whether checks track the dynamic scope (the schema has a bookended `$dynamicRef`). */ + readonly dynamic: boolean; + private readonly schema: JsonSchema; + private readonly scope: string[]; + private readonly cache = new Map(); + private readonly annotatingCache = new Map(); + + constructor(schema: JsonSchema, options: CompileOptions, dynamic: boolean) { + this.schema = schema; + this.r = new Resolver(schema, options); + this.draft = this.r.draft; + this.formats = options.formats; + this.dynamic = dynamic; + this.scope = []; + } + + /** The check of the root schema. */ + root(): Check { + return this.check(this.schema, this.r.rootBase); + } + + /** The check of a schema (cached by schema object; a recursive reference goes through an indirection). */ + check(node: unknown, base: string): Check { + if (node === true) return ALWAYS; + if (node === false) return NEVER; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.cache.get(node); + if (cached !== undefined) return cached; + const bare = this.bareType(node); + if (bare !== undefined) return bare; + const slot: { check: Check | undefined } = { check: undefined }; + this.cache.set(node, indirect(slot)); + const check = this.build(node, this.r.baseFor(node, base)); + slot.check = check; + this.cache.set(node, check); + return check; + } + + private build(node: SchemaObject, base: string): Check { + if (this.draft === "7" && typeof node.$ref === "string") return this.ref(node.$ref, base); + if (this.draft !== "7" && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + return withFreshAnnotations(this.annotating(node, base)); + } + const check = this.plain(node, base); + return this.dynamic && this.r.isResource(node, base) ? scopedCheck(this.scope, base, check) : check; + } + + /** + * The module-level check of a schema that is only a `type` (with annotations), `ALWAYS` for one with no assertion + * at all; undefined for every other schema. These are the most common subschemas and cost nothing to share. + */ + private bareType(node: SchemaObject): Check | undefined { + let type: unknown; + for (const k in node) { + if (k === "type") type = node.type; + else if (!NEUTRAL_KEYWORDS.has(k)) return undefined; + } + if (type === undefined) return ALWAYS; + return typeof type === "string" ? TYPE_CHECKS.get(type) : undefined; + } + + /** Every keyword of `node`, not tracking annotations. */ + private plain(node: SchemaObject, base: string): Check { + const parts: Check[] = []; + const types = node.type === undefined ? undefined : Array.isArray(node.type) ? node.type : [node.type]; + const single = types?.length === 1 ? types[0] : undefined; + let typeInGroup = false; + if (single === "bytes") { + parts.push(bytesCheck(number(node.minLength) ?? 0, number(node.maxLength) ?? Number.POSITIVE_INFINITY)); + typeInGroup = true; + } else { + const string = this.stringGroup(node, single === "string"); + if (string !== undefined) parts.push(string); + const numeric = single === "number" || single === "integer"; + const number = this.numberGroup(node, single === "number" ? 1 : single === "integer" ? 2 : 0); + if (number !== undefined) parts.push(number); + const object = this.objectGroup(node, base, single === "object"); + if (object !== undefined) parts.push(object); + const array = this.arrayGroup(node, base, single === "array"); + if (array !== undefined) parts.push(array); + typeInGroup = + (single === "string" && string !== undefined) || + (numeric && number !== undefined) || + (single === "object" && object !== undefined) || + (single === "array" && array !== undefined); + } + if (types !== undefined && !typeInGroup) parts.unshift(typeCheck(types)); + this.inPlace(node, base, parts); + return and(parts); + } + + // The keywords that apply to the value itself, whatever its type. + private inPlace(node: SchemaObject, base: string, parts: Check[]): void { + if (Array.isArray(node.enum)) parts.push(enumCheck(node.enum)); + if (Object.hasOwn(node, "const")) parts.push(constCheck(node.const)); + if (Array.isArray(node.allOf)) parts.push(and(node.allOf.map((s) => this.check(s, base)))); + if (Array.isArray(node.anyOf)) parts.push(this.union(node.anyOf, base, false)); + if (Array.isArray(node.oneOf)) parts.push(this.union(node.oneOf, base, true)); + if (node.not !== undefined) parts.push(notCheck(this.check(node.not, base))); + if (node.if !== undefined && (node.then !== undefined || node.else !== undefined)) { + const then = node.then === undefined ? ALWAYS : this.check(node.then, base); + const otherwise = node.else === undefined ? ALWAYS : this.check(node.else, base); + parts.push(ifCheck(this.check(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.ref(node.$ref, base)); + if (this.draft !== "7" && typeof node.$dynamicRef === "string") { + parts.push(this.dynamicRef(node.$dynamicRef, base)); + } + } + + private union(branches: readonly unknown[], base: string, exactlyOne: boolean): Check { + const checks = () => branches.map((s) => this.check(s, base)); + return ( + constUnion(branches, exactlyOne) ?? + this.tagged(branches, base) ?? + (exactlyOne ? oneCheck(checks()) : anyCheck(checks())) + ); + } + + private ref(ref: string, base: string): Check { + const target = this.r.resolve(ref, base); + const check = this.check(target.node, target.base); + return this.dynamic && target.base !== base ? scopedCheck(this.scope, target.base, check) : check; + } + + private dynamicRef(ref: string, base: string): Check { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.ref(ref, base); + const { initial, name } = resolved; + const fallback = this.check(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.check(node, b)); + const scope = this.scope; + return (v) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v); + } + return fallback(v); + }; + } + + /** The initial target and anchor name of a `$dynamicRef` that resolves dynamically, or undefined for a plain ref. */ + private bookended( + ref: string, + base: string, + ): { initial: { node: unknown; base: string }; name: string } | undefined { + const initial = this.r.resolve(ref, base); + const uri = resolveUri(ref, base); + const name = uri.slice(documentOf(uri).length + 1); + const anchored = isSchemaObject(initial.node) && initial.node.$dynamicAnchor === name; + if (name === "" || name.startsWith("/") || !anchored) return undefined; + if (!this.dynamic) throw RESTART; + return { initial, name }; + } + + // Resolved at run time against the dynamic scope, so a schema with a bookended $dynamicRef keeps its compiler. + private dynamicLookup(name: string, compile: (node: SchemaObject, base: string) => T): (uri: string) => T | null { + const found = new Map(); + return (uri) => { + const key = `${uri}#${name}`; + let check = found.get(key); + if (check === undefined) { + const target = this.r.dynamicAnchors.get(key); + check = target === undefined ? null : compile(target, this.r.baseOf.get(target) ?? uri); + found.set(key, check); + } + return check; + }; + } + + /** anyOf/oneOf whose branches are all objects with a distinct required const on one key: the tag picks the branch. */ + private tagged(branches: readonly unknown[], base: string): Check | undefined { + if (branches.length < 2) return undefined; + const resolved: SchemaObject[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch)) return undefined; + const ref = typeof branch.$ref === "string" && Object.keys(branch).length === 1 ? branch.$ref : undefined; + const node = ref === undefined ? branch : this.r.resolve(ref, this.r.baseFor(branch, base)).node; + if (!isSchemaObject(node)) return undefined; + resolved.push(node); + } + const first = resolved[0].properties; + if (!isSchemaObject(first)) return undefined; + for (const key of Object.keys(first)) { + const tags = resolved.map((s) => tagOf(s, key)); + if (tags.some((t) => t === undefined) || new Set(tags).size !== tags.length) continue; + return taggedCheck( + key, + tags, + branches.map((s) => this.check(s, base)), + ); + } + return undefined; + } + + private stringGroup(node: SchemaObject, strict: boolean): Check | undefined { + const min = number(node.minLength) ?? 0; + const max = number(node.maxLength) ?? Number.POSITIVE_INFINITY; + const re = typeof node.pattern === "string" ? sharedRegex(node.pattern) : undefined; + const format = + this.formats !== undefined && typeof node.format === "string" && Object.hasOwn(this.formats, node.format) + ? this.formats[node.format] + : undefined; + if (min === 0 && max === Number.POSITIVE_INFINITY && re === undefined && format === undefined) return undefined; + return stringCheck(strict, min, max, re, format); + } + + private numberGroup(node: SchemaObject, mode: 0 | 1 | 2): Check | undefined { + const min = number(node.minimum); + const max = number(node.maximum); + const xmin = number(node.exclusiveMinimum); + const xmax = number(node.exclusiveMaximum); + const multipleOf = number(node.multipleOf); + if ([min, max, xmin, xmax, multipleOf].every((x) => x === undefined)) return undefined; + const inf = Number.POSITIVE_INFINITY; + return numberCheck(mode, min ?? -inf, max ?? inf, xmin ?? -inf, xmax ?? inf, multipleOf); + } + + private objectGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + const onlyCore = OBJECT_EXTRAS.every((k) => node[k] === undefined); + // A schema with only the core object keywords gets one closure that also does the type guard. + const core = this.objectCore(node, base, onlyCore ? (strict ? GUARD_STRICT : GUARD_LOOSE) : GUARD_NONE); + if (core !== undefined && onlyCore) return core; + const parts: Check[] = core === undefined ? [] : [core]; + if (node.propertyNames !== undefined) parts.push(propertyNamesCheck(this.check(node.propertyNames, base))); + const minProperties = number(node.minProperties); + const maxProperties = number(node.maxProperties); + if (minProperties !== undefined || maxProperties !== undefined) { + parts.push(propertyCountCheck(minProperties ?? 0, maxProperties ?? Number.POSITIVE_INFINITY)); + } + const dependentRequired: [string, string[]][] = []; + const dependentSchemas: [string, Check][] = []; + if (this.draft !== "7") { + for (const [k, list] of entries(node.dependentRequired)) + if (Array.isArray(list)) dependentRequired.push([k, list]); + for (const [k, s] of entries(node.dependentSchemas)) dependentSchemas.push([k, this.check(s, base)]); + } + for (const [k, d] of entries(node.dependencies)) { + if (Array.isArray(d)) dependentRequired.push([k, d]); + else dependentSchemas.push([k, this.check(d, base)]); + } + if (dependentRequired.length > 0) parts.push(dependentRequiredCheck(dependentRequired)); + if (dependentSchemas.length > 0) parts.push(dependentSchemasCheck(dependentSchemas)); + if (parts.length === 0) return strict ? IS_OBJECT : undefined; + return guardObject(and(parts), strict); + } + + private objectCore(node: SchemaObject, base: string, guard: Guard): Check | undefined { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required.filter((k) => typeof k === "string") : []; + const patterns = isSchemaObject(node.patternProperties) ? node.patternProperties : {}; + const additional = node.additionalProperties; + const keys = Object.keys(properties); + const patternKeys = Object.keys(patterns); + if ( + keys.length + required.length + patternKeys.length === 0 && + (additional === undefined || additional === true) + ) { + return undefined; + } + // One array per object: key, check, flags (leaf code, required, key also names an Object.prototype member). + const entries: unknown[] = []; + for (const k of keys) { + const check = this.check(properties[k], base); + const flags = + (LEAF_CODES.get(check) ?? 0) | + (required.includes(k) ? REQUIRED : 0) | + (k in Object.prototype ? INHERITED : 0); + entries.push(k, check, flags); + } + const extraRequired = required.filter((k) => !Object.hasOwn(properties, k)); + const pats = patternKeys.map((p) => [sharedRegex(p), this.check(patterns[p], base)] as const); + const extra = additional === undefined || additional === true ? undefined : this.check(additional, base); + if (pats.length === 0 && extraRequired.length === 0 && (extra === undefined || additional === false)) { + const requiredCount = keys.filter((k) => required.includes(k)).length; + return declaredObject(entries, requiredCount, additional === false, guard); + } + return openObject(entries, keys, extraRequired, pats, extra, guard); + } + + private arrayGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + const prefixChecks = prefix.map((s) => this.check(s, base)); + const restCheck = rest === undefined || rest === true ? undefined : this.check(rest, base); + const minItems = number(node.minItems); + const maxItems = number(node.maxItems); + if ( + node.contains === undefined && + node.uniqueItems !== true && + (prefixChecks.length > 0 || restCheck !== undefined) + ) { + // The common array: one closure for the guard, the length bounds and the items. + const guard = strict ? GUARD_STRICT : GUARD_LOOSE; + return listCheck(guard, minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY, prefixChecks, restCheck); + } + const parts: Check[] = []; + if (prefixChecks.length > 0 || restCheck !== undefined) parts.push(itemsCheck(prefixChecks, restCheck)); + if (minItems !== undefined || maxItems !== undefined) { + parts.push(lengthCheck(minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY)); + } + if (node.contains !== undefined) { + const min = this.draft === "7" ? 1 : (number(node.minContains) ?? 1); + const max = + this.draft === "7" ? Number.POSITIVE_INFINITY : (number(node.maxContains) ?? Number.POSITIVE_INFINITY); + parts.push(containsCheck(this.check(node.contains, base), min, max)); + } + if (node.uniqueItems === true) parts.push(uniqueCheck); + if (parts.length === 0) return strict ? IS_ARRAY : undefined; + return guardArray(and(parts), strict); + } + + /** The annotating check of a schema evaluated in place (only below `unevaluatedProperties`/`unevaluatedItems`). */ + annotating(node: unknown, base: string): AnnotatingCheck { + if (node === true) return ANNOTATING_TRUE; + if (node === false) return ANNOTATING_FALSE; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.annotatingCache.get(node); + if (cached !== undefined) return cached; + const slot: { check: AnnotatingCheck | undefined } = { check: undefined }; + this.annotatingCache.set(node, indirectAnnotating(slot)); + const check = this.buildAnnotating(node, this.r.baseFor(node, base)); + slot.check = check; + this.annotatingCache.set(node, check); + return check; + } + + private buildAnnotating(node: SchemaObject, base: string): AnnotatingCheck { + const rest: Record = {}; + let annotates = false; + for (const k of Object.keys(node)) { + if (ANNOTATING.has(k)) annotates = true; + else rest[k] = node[k]; + } + const plain = this.plain(rest, base); + const parts: AnnotatingCheck[] = []; + if (annotates) this.annotatingParts(node, base, parts); + const all = parts.length === 0 ? ignoreAnnotations(plain) : annotateAll(plain, parts); + const own = + node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined + ? isolateAnnotations(all) + : all; + return this.dynamic && this.r.isResource(node, base) ? scopedAnnotating(this.scope, base, own) : own; + } + + private annotatingParts(node: SchemaObject, base: string, parts: AnnotatingCheck[]): void { + if ( + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined + ) { + const props = entries(node.properties).map(([k, s]) => [k, this.check(s, base)] as const); + const pats = entries(node.patternProperties).map(([p, s]) => [sharedRegex(p), this.check(s, base)] as const); + const extra = + node.additionalProperties === undefined ? undefined : this.check(node.additionalProperties, base); + parts.push(annotateObject(props, pats, extra)); + } + if (node.prefixItems !== undefined || node.items !== undefined || node.contains !== undefined) { + const prefix = Array.isArray(node.prefixItems) + ? node.prefixItems + : Array.isArray(node.items) + ? node.items + : []; + const rest = Array.isArray(node.items) ? node.additionalItems : node.items; + parts.push( + annotateArray( + prefix.map((s) => this.check(s, base)), + rest === undefined ? undefined : this.check(rest, base), + node.contains === undefined ? undefined : this.check(node.contains, base), + number(node.minContains) ?? 1, + number(node.maxContains) ?? Number.POSITIVE_INFINITY, + ), + ); + } + if (Array.isArray(node.allOf)) for (const s of node.allOf) parts.push(this.annotating(s, base)); + if (Array.isArray(node.anyOf)) + parts.push( + annotateAny( + node.anyOf.map((s) => this.annotating(s, base)), + false, + ), + ); + if (Array.isArray(node.oneOf)) + parts.push( + annotateAny( + node.oneOf.map((s) => this.annotating(s, base)), + true, + ), + ); + if (node.if !== undefined) { + const then = node.then === undefined ? undefined : this.annotating(node.then, base); + const otherwise = node.else === undefined ? undefined : this.annotating(node.else, base); + parts.push(annotateIf(this.annotating(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.annotatingRef(node.$ref, base)); + if (typeof node.$dynamicRef === "string") parts.push(this.annotatingDynamicRef(node.$dynamicRef, base)); + for (const [k, s] of entries(node.dependentSchemas)) parts.push(annotateDependent(k, this.annotating(s, base))); + if (node.unevaluatedProperties !== undefined) { + parts.push(unevaluatedProperties(this.check(node.unevaluatedProperties, base))); + } + if (node.unevaluatedItems !== undefined) parts.push(unevaluatedItems(this.check(node.unevaluatedItems, base))); + } + + private annotatingRef(ref: string, base: string): AnnotatingCheck { + const target = this.r.resolve(ref, base); + const check = this.annotating(target.node, target.base); + return this.dynamic && target.base !== base ? scopedAnnotating(this.scope, target.base, check) : check; + } + + private annotatingDynamicRef(ref: string, base: string): AnnotatingCheck { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.annotatingRef(ref, base); + const { initial, name } = resolved; + const fallback = this.annotating(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.annotating(node, b)); + const scope = this.scope; + return (v, a) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v, a); + } + return fallback(v, a); + }; + } +} + +/** Keywords that never affect a check. */ +export const NEUTRAL_KEYWORDS = new Set([ + "$schema", + "$comment", + "title", + "description", + "default", + "examples", + "deprecated", + "readOnly", + "writeOnly", + "$defs", + "definitions", + "contentEncoding", + "contentMediaType", + "minContains", + "maxContains", + "$vocabulary", +]); + +// The object keywords besides the core (properties, required, patternProperties, additionalProperties). +const OBJECT_EXTRAS = [ + "propertyNames", + "minProperties", + "maxProperties", + "dependentRequired", + "dependentSchemas", + "dependencies", +]; + +// The keywords an annotating check handles itself; every other keyword compiles into its plain part. +const ANNOTATING = new Set([ + "properties", + "patternProperties", + "additionalProperties", + "prefixItems", + "items", + "additionalItems", + "contains", + "minContains", + "maxContains", + "allOf", + "anyOf", + "oneOf", + "if", + "then", + "else", + "$ref", + "$dynamicRef", + "dependentSchemas", + "unevaluatedProperties", + "unevaluatedItems", +]); + +/** anyOf/oneOf whose branches are all `{ const: primitive }`: an enum (distinct values make oneOf the same). */ +function constUnion(branches: readonly unknown[], exactlyOne: boolean): Check | undefined { + const values: unknown[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch) || !Object.hasOwn(branch, "const")) return undefined; + for (const k of Object.keys(branch)) if (k !== "const" && !NEUTRAL_KEYWORDS.has(k)) return undefined; + if (typeof branch.const === "object" && branch.const !== null) return undefined; + if (exactlyOne && values.includes(branch.const)) return undefined; + values.push(branch.const); + } + return enumCheck(values); +} + +// The tag of a tagged-union branch: a primitive const on a required property of an object schema. +function tagOf(branch: SchemaObject, key: string): unknown { + const properties = branch.properties; + if (branch.type !== "object" || !Array.isArray(branch.required) || !branch.required.includes(key)) return undefined; + const tag = isSchemaObject(properties) ? properties[key] : undefined; + if (!isSchemaObject(tag) || !Object.hasOwn(tag, "const")) return undefined; + return typeof tag.const === "object" && tag.const !== null ? undefined : tag.const; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" ? value : undefined; +} + +function entries(value: unknown): [string, unknown][] { + return isSchemaObject(value) ? Object.entries(value) : []; +} diff --git a/packages/schema/src/explain.ts b/packages/schema/src/explain.ts new file mode 100644 index 000000000..3ab363986 --- /dev/null +++ b/packages/schema/src/explain.ts @@ -0,0 +1,390 @@ +// Errors of a failed value: a walk over the schema that reports each failing keyword at the value it failed on. +// It runs only for values that failed the check, using the compiler's checks for subschemas. + +import { newAnnotations } from "./annotations.ts"; +import { + codePoints, + constCheck, + containsCheck, + enumCheck, + isMultiple, + isObject, + NEVER, + TYPE_CHECKS, + uniqueCheck, +} from "./checks.ts"; +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject, ValidationError } from "./types.ts"; + +/** Explains failures of one compiled schema. */ +export class Explainer { + private readonly c: Compiler; + private readonly schema: unknown; + /** + * The errors of `$ref` targets for object values, within one errors() call: a recursive union explains each branch + * to pick the one to blame, and every branch would explain the same children again, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + compiler.root(); + } + + /** The errors of `value`, at least one (`value` failed the check). */ + errors(value: unknown): ValidationError[] { + const out: ValidationError[] = []; + try { + this.walk(this.schema, this.c.r.rootBase, value, "", out); + } finally { + this.memo = undefined; + } + if (out.length === 0) out.push({ path: "", keyword: "schema", message: "invalid" }); + return out; + } + + private ok(node: unknown, base: string, v: unknown): boolean { + return this.c.check(node, base)(v); + } + + // For $dynamicRef schemas this check runs outside the original dynamic scope, so their errors are approximate. + private walk(node: unknown, base: string, v: unknown, path: string, out: ValidationError[]): void { + if (node === true) return; + if (node === false) { + out.push({ path, keyword: "false", message: "no value is allowed here" }); + return; + } + if (!isSchemaObject(node)) return; + const b = this.c.r.baseFor(node, base); + if (this.ok(node, b, v)) return; + const start = out.length; + const push = (keyword: string, message: string, at = path) => { + out.push({ path: at, keyword, message }); + }; + if (this.c.draft === "7" && typeof node.$ref === "string") { + this.walkRef(node.$ref, b, v, path, out); + return; + } + if (node.type !== undefined) { + const types = Array.isArray(node.type) ? node.type : [node.type]; + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (!checks.some((check) => check(v))) { + push("type", `expected ${types.join(" or ")}, got ${typeName(v)}`); + return; + } + } + if (Array.isArray(node.enum) && !enumCheck(node.enum)(v)) + push("enum", `expected one of ${listValues(node.enum)}`); + if (Object.hasOwn(node, "const") && !constCheck(node.const)(v)) + push("const", `expected ${JSON.stringify(node.const)}`); + if (typeof v === "string" && node.type !== "bytes") this.walkString(node, v, push); + if (v instanceof Uint8Array && node.type === "bytes") this.walkBytes(node, v, push); + if (typeof v === "number") this.walkNumber(node, v, push); + if (isObject(v)) this.walkObject(node, b, v, path, out, push); + if (Array.isArray(v)) this.walkArray(node, b, v, path, out, push); + if (Array.isArray(node.allOf)) for (const s of node.allOf) this.walk(s, b, v, path, out); + if (Array.isArray(node.anyOf)) this.walkUnion(node.anyOf, b, v, path, out, false); + if (Array.isArray(node.oneOf)) this.walkUnion(node.oneOf, b, v, path, out, true); + if (node.not !== undefined && this.ok(node.not, b, v)) push("not", "must not match the schema in not"); + if (node.if !== undefined) { + const branch = this.ok(node.if, b, v) ? node.then : node.else; + if (branch !== undefined) this.walk(branch, b, v, path, out); + } + if (typeof node.$ref === "string") this.walkRef(node.$ref, b, v, path, out); + if (out.length === start && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + this.walkUnevaluated(node, b, v, path, out); + } + if (out.length === start && typeof node.$dynamicRef === "string") { + push("$dynamicRef", "does not match the dynamically referenced schema"); + } + } + + private walkRef(ref: string, base: string, v: unknown, path: string, out: ValidationError[]): void { + const target = this.c.r.resolve(ref, base); + if (target.node === false && this.c.r.isUnresolved(ref, base)) { + out.push({ path, keyword: "$ref", message: `unresolved reference ${ref}` }); + return; + } + if (!isSchemaObject(target.node) || typeof v !== "object" || v === null) { + this.walk(target.node, target.base, v, path, out); + return; + } + this.memo ??= new Map(); + let values = this.memo.get(target.node); + const known = values?.get(v); + if (known !== undefined && known.path === path) { + out.push(...known.errors); + return; + } + const errors: ValidationError[] = []; + this.walk(target.node, target.base, v, path, errors); + if (values === undefined) { + values = new Map(); + this.memo.set(target.node, values); + } + values.set(v, { path, errors }); + out.push(...errors); + } + + private walkString(node: SchemaObject, v: string, push: Push): void { + const n = codePoints(v); + if (typeof node.minLength === "number" && n < node.minLength) { + push("minLength", `expected at least ${node.minLength} characters`); + } + if (typeof node.maxLength === "number" && n > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} characters`); + } + if (typeof node.pattern === "string" && !sharedRegex(node.pattern).test(v)) { + push("pattern", `expected to match ${node.pattern}`); + } + const formats = this.c.formats; + const format = + formats !== undefined && typeof node.format === "string" && Object.hasOwn(formats, node.format) + ? formats[node.format] + : undefined; + if (format !== undefined && !format(v)) push("format", `expected format ${node.format}`); + } + + private walkBytes(node: SchemaObject, v: Uint8Array, push: Push): void { + if (typeof node.minLength === "number" && v.length < node.minLength) { + push("minLength", `expected at least ${node.minLength} bytes`); + } + if (typeof node.maxLength === "number" && v.length > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} bytes`); + } + } + + private walkNumber(node: SchemaObject, v: number, push: Push): void { + if (typeof node.minimum === "number" && v < node.minimum) push("minimum", `expected >= ${node.minimum}`); + if (typeof node.maximum === "number" && v > node.maximum) push("maximum", `expected <= ${node.maximum}`); + if (typeof node.exclusiveMinimum === "number" && v <= node.exclusiveMinimum) { + push("exclusiveMinimum", `expected > ${node.exclusiveMinimum}`); + } + if (typeof node.exclusiveMaximum === "number" && v >= node.exclusiveMaximum) { + push("exclusiveMaximum", `expected < ${node.exclusiveMaximum}`); + } + if (typeof node.multipleOf === "number" && !isMultiple(v, node.multipleOf)) { + push("multipleOf", `expected a multiple of ${node.multipleOf}`); + } + } + + private walkObject( + node: SchemaObject, + b: string, + v: Record, + path: string, + out: ValidationError[], + push: Push, + ): void { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required : []; + for (const k of required) { + if (typeof k === "string" && (!Object.hasOwn(v, k) || v[k] === undefined)) + push("required", "missing", pointer(path, k)); + } + for (const k of Object.keys(properties)) { + if (Object.hasOwn(v, k) && v[k] !== undefined) this.walk(properties[k], b, v[k], pointer(path, k), out); + } + const patterns = isSchemaObject(node.patternProperties) ? Object.entries(node.patternProperties) : []; + for (const k of Object.keys(v)) { + let matched = Object.hasOwn(properties, k); + for (const [p, s] of patterns) { + if (sharedRegex(p).test(k)) { + matched = true; + this.walk(s, b, v[k], pointer(path, k), out); + } + } + if (!matched && node.additionalProperties !== undefined) { + if (node.additionalProperties === false) push("additionalProperties", "unknown field", pointer(path, k)); + else this.walk(node.additionalProperties, b, v[k], pointer(path, k), out); + } + if (node.propertyNames !== undefined && !this.ok(node.propertyNames, b, k)) { + push("propertyNames", "invalid property name", pointer(path, k)); + } + } + const count = Object.keys(v).length; + if (typeof node.minProperties === "number" && count < node.minProperties) { + push("minProperties", `expected at least ${node.minProperties} properties`); + } + if (typeof node.maxProperties === "number" && count > node.maxProperties) { + push("maxProperties", `expected at most ${node.maxProperties} properties`); + } + const dependencies = { + ...(this.c.draft === "7" || !isSchemaObject(node.dependentRequired) ? {} : node.dependentRequired), + ...(isSchemaObject(node.dependencies) ? node.dependencies : {}), + }; + for (const [k, d] of Object.entries(dependencies)) { + if (!Object.hasOwn(v, k)) continue; + if (!Array.isArray(d)) this.walk(d, b, v, path, out); + else + for (const r of d) + if (!Object.hasOwn(v, r)) push("dependentRequired", `required when ${k} is present`, pointer(path, r)); + } + if (this.c.draft !== "7" && isSchemaObject(node.dependentSchemas)) { + for (const [k, s] of Object.entries(node.dependentSchemas)) + if (Object.hasOwn(v, k)) this.walk(s, b, v, path, out); + } + } + + private walkArray( + node: SchemaObject, + b: string, + v: unknown[], + path: string, + out: ValidationError[], + push: Push, + ): void { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.c.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + for (let i = 0; i < v.length; i++) { + const s = i < prefix.length ? prefix[i] : rest; + if (s !== undefined) this.walk(s, b, v[i], `${path}/${i}`, out); + } + if (typeof node.minItems === "number" && v.length < node.minItems) { + push("minItems", `expected at least ${node.minItems} items`); + } + if (typeof node.maxItems === "number" && v.length > node.maxItems) { + push("maxItems", `expected at most ${node.maxItems} items`); + } + if (node.uniqueItems === true && !uniqueCheck(v)) push("uniqueItems", "expected unique items"); + if (node.contains !== undefined) { + const draft7 = this.c.draft === "7"; + const min = draft7 || typeof node.minContains !== "number" ? 1 : node.minContains; + const max = draft7 || typeof node.maxContains !== "number" ? Number.POSITIVE_INFINITY : node.maxContains; + if (!containsCheck(this.c.check(node.contains, b), min, max)(v)) { + push("contains", containsMessage(min, max)); + } + } + } + + private walkUnion( + branches: readonly unknown[], + b: string, + v: unknown, + path: string, + out: ValidationError[], + exactlyOne: boolean, + ): void { + const matches = branches.filter((s) => this.ok(s, b, v)).length; + if (matches === 1 || (!exactlyOne && matches > 0)) return; + const keyword = exactlyOne ? "oneOf" : "anyOf"; + if (matches > 1) { + out.push({ path, keyword, message: "matches more than one alternative" }); + return; + } + // A tagged union reports the branch its tag selects; otherwise the branch whose first error is deepest. + const tag = this.tagOf(branches, b); + if (tag !== undefined && isObject(v)) { + const i = tag.values.indexOf(v[tag.key]); + if (i < 0) + out.push({ + path: pointer(path, tag.key), + keyword, + message: `expected one of ${listValues(tag.values)}`, + }); + else this.walk(branches[i], b, v, path, out); + return; + } + // An object is blamed on the branch that declares most of its properties (the one it was most likely meant for), + // then on the branch whose first error is deepest. + let best: { errors: ValidationError[]; declared: number } | undefined; + for (const s of branches) { + const errors: ValidationError[] = []; + this.walk(s, b, v, path, errors); + if (errors.length === 0) continue; + const declared = isObject(v) ? this.declaredKeys(s, b, v) : 0; + const better = + best === undefined || + declared > best.declared || + (declared === best.declared && errors[0].path.length > best.errors[0].path.length); + if (better) best = { errors, declared }; + } + out.push(...(best?.errors ?? [{ path, keyword, message: "matches no alternative" }])); + } + + /** How many of the keys of `v` the branch `s` (through a `$ref`) declares in `properties`. */ + private declaredKeys(s: unknown, b: string, v: Record): number { + const node = isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s; + const properties = isSchemaObject(node) && isSchemaObject(node.properties) ? node.properties : {}; + return Object.keys(v).filter((k) => Object.hasOwn(properties, k)).length; + } + + private tagOf(branches: readonly unknown[], b: string): { key: string; values: unknown[] } | undefined { + const resolved = branches.map((s) => + isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s, + ); + const first = resolved[0]; + if (!isSchemaObject(first) || !isSchemaObject(first.properties)) return undefined; + for (const key of Object.keys(first.properties)) { + const values = resolved.map((s) => { + const tag = isSchemaObject(s) && isSchemaObject(s.properties) ? s.properties[key] : undefined; + return isSchemaObject(tag) ? tag.const : undefined; + }); + const primitive = values.every((x) => x !== undefined && (typeof x !== "object" || x === null)); + if (primitive && new Set(values).size === values.length) return { key, values }; + } + return undefined; + } + + private walkUnevaluated(node: SchemaObject, b: string, v: unknown, path: string, out: ValidationError[]): void { + const { unevaluatedProperties, unevaluatedItems, ...rest } = node; + const a = newAnnotations(); + this.c.annotating(rest, b)(v, a); + if (isObject(v) && unevaluatedProperties !== undefined && !a.allProps) { + for (const k of Object.keys(v)) { + if (a.props?.has(k)) continue; + if (unevaluatedProperties === false) { + out.push({ path: pointer(path, k), keyword: "unevaluatedProperties", message: "unknown field" }); + } else this.walk(unevaluatedProperties, b, v[k], pointer(path, k), out); + } + } + if (Array.isArray(v) && unevaluatedItems !== undefined && !a.allItems) { + for (let i = a.items; i < v.length; i++) { + if (a.indexes?.has(i)) continue; + if (unevaluatedItems === false) { + out.push({ path: `${path}/${i}`, keyword: "unevaluatedItems", message: "unexpected item" }); + } else this.walk(unevaluatedItems, b, v[i], `${path}/${i}`, out); + } + } + } +} + +type Push = (keyword: string, message: string, at?: string) => void; + +// Enum errors name the allowed values; a long list (an enum of thousands) is cut, so the message stays readable. +const LISTED_VALUES = 20; + +/** `values` as JSON, at most LISTED_VALUES of them, then how many more there are. */ +function listValues(values: readonly unknown[]): string { + if (values.length <= LISTED_VALUES) return JSON.stringify(values); + const listed = JSON.stringify(values.slice(0, LISTED_VALUES)); + return `${listed.slice(0, -1)}, ... and ${values.length - LISTED_VALUES} more]`; +} + +function containsMessage(min: number, max: number): string { + if (max === Number.POSITIVE_INFINITY) return `expected at least ${min} matching ${min === 1 ? "item" : "items"}`; + if (min === 0) return `expected at most ${max} matching ${max === 1 ? "item" : "items"}`; + return `expected between ${min} and ${max} matching items`; +} + +function pointer(path: string, key: string | number): string { + return `${path}/${String(key).replace(/~/g, "~0").replace(/\//g, "~1")}`; +} + +function typeName(v: unknown): string { + if (v === null) return "null"; + if (Array.isArray(v)) return "array"; + if (v instanceof Uint8Array) return "bytes"; + if (typeof v === "number" && Number.isInteger(v)) return "integer"; + return typeof v; +} diff --git a/packages/schema/src/formats/index.ts b/packages/schema/src/formats/index.ts new file mode 100644 index 000000000..05d8824ed --- /dev/null +++ b/packages/schema/src/formats/index.ts @@ -0,0 +1,53 @@ +// The format checks of JSON Schema 2020-12, for compile()'s `formats` option. An optional entry point: schemas +// compiled without it treat `format` as an annotation and do not bundle these checks. + +import type { FormatCheck } from "../types.ts"; +import { IsDate } from "./typebox/date.ts"; +import { IsDateTime } from "./typebox/date_time.ts"; +import { IsDuration } from "./typebox/duration.ts"; +import { IsEmail } from "./typebox/email.ts"; +import { IsHostname } from "./typebox/hostname.ts"; +import { IsIdnEmail } from "./typebox/idn_email.ts"; +import { IsIdnHostname } from "./typebox/idn_hostname.ts"; +import { IsIPv4 } from "./typebox/ipv4.ts"; +import { IsIPv6 } from "./typebox/ipv6.ts"; +import { IsIri } from "./typebox/iri.ts"; +import { IsIriReference } from "./typebox/iri_reference.ts"; +import { IsJsonPointer } from "./typebox/json_pointer.ts"; +import { IsJsonPointerUriFragment } from "./typebox/json_pointer_uri_fragment.ts"; +import { IsRegex } from "./typebox/regex.ts"; +import { IsRelativeJsonPointer } from "./typebox/relative_json_pointer.ts"; +import { IsTime } from "./typebox/time.ts"; +import { IsUri } from "./typebox/uri.ts"; +import { IsUriReference } from "./typebox/uri_reference.ts"; +import { IsUriTemplate } from "./typebox/uri_template.ts"; +import { IsUrl } from "./typebox/url.ts"; +import { IsUuid } from "./typebox/uuid.ts"; + +/** + * Every format JSON Schema 2020-12 defines, by name, plus `url` and `json-pointer-uri-fragment`, which TypeBox and + * ajv-formats also check. The checks are adapted from TypeBox 1.3.27 (MIT). + */ +export const formats: Readonly> = Object.freeze({ + date: IsDate, + "date-time": IsDateTime, + duration: IsDuration, + email: IsEmail, + hostname: IsHostname, + "idn-email": IsIdnEmail, + "idn-hostname": IsIdnHostname, + ipv4: IsIPv4, + ipv6: IsIPv6, + iri: IsIri, + "iri-reference": IsIriReference, + "json-pointer": IsJsonPointer, + "json-pointer-uri-fragment": IsJsonPointerUriFragment, + regex: IsRegex, + "relative-json-pointer": IsRelativeJsonPointer, + time: IsTime, + uri: IsUri, + "uri-reference": IsUriReference, + "uri-template": IsUriTemplate, + url: IsUrl, + uuid: IsUuid, +}); diff --git a/packages/schema/src/formats/typebox/date.ts b/packages/schema/src/formats/typebox/date.ts new file mode 100644 index 000000000..e35cf0770 --- /dev/null +++ b/packages/schema/src/formats/typebox/date.ts @@ -0,0 +1,47 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; +const DATE = /^(\d\d\d\d)-(\d\d)-(\d\d)$/; + +function IsLeapYear(year: number): boolean { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); +} +/** + * Returns true if the value is a ISO8601 Date component string + * @description https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12` + */ +export function IsDate(value: string): boolean { + const matches: string[] | null = DATE.exec(value); + if (!matches) return false; + const year: number = +matches[1]; + const month: number = +matches[2]; + const day: number = +matches[3]; + return month >= 1 && month <= 12 && day >= 1 && day <= (month === 2 && IsLeapYear(year) ? 29 : DAYS[month]); +} diff --git a/packages/schema/src/formats/typebox/date_time.ts b/packages/schema/src/formats/typebox/date_time.ts new file mode 100644 index 000000000..df4174a40 --- /dev/null +++ b/packages/schema/src/formats/typebox/date_time.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import { IsDate } from "./date.ts"; +import { IsTime } from "./time.ts"; + +/** + * Returns true if the value is a ISO8601 DateTime string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12T20:20:40+00:00` + */ +export function IsDateTime(value: string): boolean { + const dateTime: string[] = value.split(/T/i); + return dateTime.length === 2 && IsDate(dateTime[0]) && IsTime(dateTime[1]); +} diff --git a/packages/schema/src/formats/typebox/duration.ts b/packages/schema/src/formats/typebox/duration.ts new file mode 100644 index 000000000..20d812c4f --- /dev/null +++ b/packages/schema/src/formats/typebox/duration.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Duration = + /^P((\d+Y(\d+M(\d+D)?)?|\d+M(\d+D)?|\d+D)(T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S))?|T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S)|\d+W)$/; + +/** + * Returns true if the value is a valid ISO-8601 duration. + * @specification https://tools.ietf.org/html/rfc3339 + */ +export function IsDuration(value: string): boolean { + return Duration.test(value); +} diff --git a/packages/schema/src/formats/typebox/email.ts b/packages/schema/src/formats/typebox/email.ts new file mode 100644 index 000000000..a6840daed --- /dev/null +++ b/packages/schema/src/formats/typebox/email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Email = + /^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[^"\\]|\\[\x20-\x7e])*")@(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*|\[(?:IPv6:[a-f0-9:]+|(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\])$/i; + +/** + * Returns true if the value is an Email + * @specification https://datatracker.ietf.org/doc/html/rfc5322 + */ +export function IsEmail(value: string): boolean { + return Email.test(value); +} diff --git a/packages/schema/src/formats/typebox/hostname.ts b/packages/schema/src/formats/typebox/hostname.ts new file mode 100644 index 000000000..68c3710fb --- /dev/null +++ b/packages/schema/src/formats/typebox/hostname.ts @@ -0,0 +1,39 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Idna from "./idna/index.ts"; + +/** + * Returns true if the value is a valid hostname. + * @specification https://tools.ietf.org/html/rfc1123 + * @specification https://tools.ietf.org/html/rfc5891 + * @specification https://tools.ietf.org/html/rfc5892 + */ +export function IsHostname(value: string): boolean { + return Idna.IsHostname(value); +} diff --git a/packages/schema/src/formats/typebox/idn_email.ts b/packages/schema/src/formats/typebox/idn_email.ts new file mode 100644 index 000000000..92e3abdaf --- /dev/null +++ b/packages/schema/src/formats/typebox/idn_email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IdnEmail = + /^(?:[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+)*|"(?:[^"\\]|\\.)*")@[\p{L}\p{N}](?:[\p{L}\p{N}-]{0,62})(?> 1; + delta += Math.floor(delta / numPoints); + let k = 0; + while (delta > ((PUNYCODE_BASE - PUNYCODE_TMIN) * PUNYCODE_TMAX) >> 1) { + delta = Math.floor(delta / (PUNYCODE_BASE - PUNYCODE_TMIN)); + k += PUNYCODE_BASE; + } + return k + Math.floor(((PUNYCODE_BASE - PUNYCODE_TMIN + 1) * delta) / (delta + PUNYCODE_SKEW)); +} +// ------------------------------------------------------------------ +// Decode +// ------------------------------------------------------------------ +export function Decode(value: string): string { + const output: number[] = []; + let n = PUNYCODE_INITIAL_N; + let i = 0; + let bias = PUNYCODE_INITIAL_BIAS; + const delimIdx = value.lastIndexOf("-"); + if (delimIdx > 0) { + for (let j = 0; j < delimIdx; j++) { + const cp = value.charCodeAt(j); + if (cp >= 128) ThrowDontCare(); // throw new Error('Invalid punycode: non-basic before delimiter') + output.push(cp); + } + } + let inIdx = delimIdx < 0 ? 0 : delimIdx + 1; + while (inIdx < value.length) { + const oldi = i; + let w = 1; + let k = PUNYCODE_BASE; + while (true) { + if (inIdx >= value.length) ThrowDontCare(); // throw new Error('Invalid punycode: unexpected end of input') + const ch = value.charCodeAt(inIdx++); + let digit: number; + if (ch >= 0x61 && ch <= 0x7a) + digit = ch - 0x61; // a-z => 0-25 + else if (ch >= 0x30 && ch <= 0x39) + digit = ch - 0x30 + 26; // 0-9 => 26-35 + // else if (ch >= 0x41 && ch <= 0x5a) digit = ch - 0x41 // A-Z => 0-25 // (no-spec-coverage) + else ThrowDontCare(); // throw new Error('Invalid punycode: bad digit character') + i += digit * w; + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (digit < t) break; + w *= PUNYCODE_BASE - t; + k += PUNYCODE_BASE; + } + const outLen = output.length + 1; + bias = Adapt(i - oldi, outLen, oldi === 0); + n += Math.floor(i / outLen); + i %= outLen; + output.splice(i, 0, n); + i++; + } + return String.fromCodePoint(...output); +} +// ------------------------------------------------------------------ +// DigitToChar +// ------------------------------------------------------------------ +function DigitToChar(digit: number): string { + return digit < 26 ? String.fromCharCode(digit + 0x61) : String.fromCharCode(digit - 26 + 0x30); +} +// ------------------------------------------------------------------ +// Encode +// +// Encodes a Unicode string into its Punycode payload (the part that +// follows the 'xn--' ACE prefix). Used to compute the true A-label +// length of a U-label, since the 63-octet label length limit (RFC +// 5891 §4.2.4) applies to the encoded form, not the code point count. +// ------------------------------------------------------------------ +const RE_REPLACE_NON_ASCII = new RegExp(Pattern.RE_NON_ASCII.source, "gu"); +export function Encode(input: string): string { + // Extract basic ASCII characters and set initial output array state + const basic = input.replace(RE_REPLACE_NON_ASCII, ""); + const basicLength = basic.length; + const result = basicLength > 0 ? [basic, "-"] : []; + // Extract unicode codepoints + const codePoints = Array.from(input, (char) => char.codePointAt(0)!); + let n = PUNYCODE_INITIAL_N; + let delta = 0; + let bias = PUNYCODE_INITIAL_BIAS; + let handledCPCount = basicLength; + while (handledCPCount < codePoints.length) { + let m = Infinity; + for (const cp of codePoints) { + if (cp >= n && cp < m) m = cp; + } + delta += (m - n) * (handledCPCount + 1); + n = m; + for (const cp of codePoints) { + if (cp < n) delta++; + if (cp === n) { + let q = delta; + for (let k = PUNYCODE_BASE; ; k += PUNYCODE_BASE) { + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (q < t) break; + const digit = t + ((q - t) % (PUNYCODE_BASE - t)); + result.push(DigitToChar(digit)); + q = Math.floor((q - t) / (PUNYCODE_BASE - t)); + } + result.push(DigitToChar(q)); + bias = Adapt(delta, handledCPCount + 1, handledCPCount === basicLength); + delta = 0; + handledCPCount++; + } + } + delta++; + n++; + } + return result.join(""); +} diff --git a/packages/schema/src/formats/typebox/idna/hostname.ts b/packages/schema/src/formats/typebox/idna/hostname.ts new file mode 100644 index 000000000..ade2322b4 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/hostname.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as LabelAscii from "./label/ascii.ts"; +import * as LabelPuny from "./label/puny.ts"; + +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelAscii.IsAsciiLabel(value)); +} +export function IsHostname(value: string): boolean { + if (value.length === 0 || value.length > 253) return false; + if (value.charCodeAt(value.length - 1) === 46) return false; + return value.split(".").every((label) => IsLabel(label)); +} diff --git a/packages/schema/src/formats/typebox/idna/idn-hostname.ts b/packages/schema/src/formats/typebox/idna/idn-hostname.ts new file mode 100644 index 000000000..d4517faac --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/idn-hostname.ts @@ -0,0 +1,72 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "./format/bidi.ts"; +import * as LabelPuny from "./label/puny.ts"; +import * as LabelUnicode from "./label/unicode.ts"; + +// ------------------------------------------------------------------ +// IsLabel +// ------------------------------------------------------------------ +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelUnicode.IsUnicodeLabel(value)); +} +// ------------------------------------------------------------------ +// NormalizeHostname +// +// Normalizes a hostname for label validation. Maps fullwidth +// codepoints to their ASCII equivalent (e.g. 'ａ' U+FF41 maps to +// 'a'), then normalizes for NFC such that equivalent codepoint +// sequences can be uniformly compared. It also removes codepoints +// that IDNA mapping ignores, and converts full stop variants to +// ASCII '.' so the hostname can be split into labels. +// ------------------------------------------------------------------ +function NormalizeHostname(value: string): string { + return ( + value + .replace(/[\uff01-\uff5e]/g, (char) => String.fromCharCode(char.charCodeAt(0) - 0xfee0)) // RE_FULLWIDTH_MAPPING + .normalize("NFC") + // biome-ignore lint/suspicious/noMisleadingCharacterClass: a set of single code points to remove (u flag), by design. + .replace(/[\u00ad\u034f\u180b-\u180d\u200b\ufe00-\ufe0f\u{e0100}-\u{e01ef}]/gu, "") // RE_IGNORED_MAPPING + .replace(/[\u002E\u3002\uFF0E\uFF61]/g, ".") + ); // RE_FULL_STOP_MAPPING +} +// ------------------------------------------------------------------ +// IsIdnHostname +// ------------------------------------------------------------------ +export function IsIdnHostname(value: string): boolean { + if (value.length === 0 || value.includes(" ")) return false; + const normalized = NormalizeHostname(value); + if (normalized.length > 253) return false; + const labels = normalized.split("."); + const hasBidiChars = labels.some((label) => FormatBidi.HasBidiChars(label)); + return labels.every((label) => IsLabel(label) && (!hasBidiChars || FormatBidi.SatisfiesBidiRule(label))); +} diff --git a/packages/schema/src/formats/typebox/idna/index.ts b/packages/schema/src/formats/typebox/idna/index.ts new file mode 100644 index 000000000..c5d10175e --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/index.ts @@ -0,0 +1,30 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +export * from "./hostname.ts"; +export * from "./idn-hostname.ts"; diff --git a/packages/schema/src/formats/typebox/idna/label/ascii.ts b/packages/schema/src/formats/typebox/idna/label/ascii.ts new file mode 100644 index 000000000..704e8f944 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/ascii.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------- +// Validates an ASCII LDH label per RFC 5891 §4.2.3.1 +// ------------------------------------------------------------------- +export function IsAsciiLabel(value: string): boolean { + return ( + Pattern.RE_RULE_HYPHEN_PLACEMENT.test(value) && + Pattern.RE_RULE_NOT_RESERVED_ACE.test(value) && + Pattern.RE_ASCII_LDH.test(value) + ); +} diff --git a/packages/schema/src/formats/typebox/idna/label/puny.ts b/packages/schema/src/formats/typebox/idna/label/puny.ts new file mode 100644 index 000000000..23b288ce2 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/puny.ts @@ -0,0 +1,53 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; +import * as LabelUnicode from "./unicode.ts"; + +// ------------------------------------------------------------------ +// IsPunyLabel +// ------------------------------------------------------------------ +export function IsPunyLabel(value: string): boolean { + if (!FormatPuny.IsAcePrefixed(value)) return false; + try { + const body = value.slice(4).toLowerCase(); + // A payload consisting of only a hyphen, or starting with the delimiter + // (like "-9uc"), has 0 basic ASCII characters preceding it and is malformed. + if (body.lastIndexOf("-") === 0) return false; + const decoded = FormatPuny.Decode(body); + // RFC 5890 §2.3.2.1 - a U-label must contain at least one non-ASCII + // character. Code-unit level check: any non-ASCII codepoint has at least + // one UTF-16 unit >= 0x80 (surrogate halves included), so no need to + // decode codepoints or allocate a character array to answer this. + if (!Pattern.RE_NON_ASCII.test(decoded)) return false; + return LabelUnicode.IsUnicodeLabel(decoded); + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/idna/label/unicode.ts b/packages/schema/src/formats/typebox/idna/label/unicode.ts new file mode 100644 index 000000000..615942427 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/unicode.ts @@ -0,0 +1,122 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "../format/bidi.ts"; +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------ +// ExceedsMaxALabelLength +// +// RFC 5890 §2.3.2.1: an A-label (the ACE-encoded form of a label, +// "xn--" + punycode) must not exceed 63 octets. Only labels +// containing non-ASCII characters need checking - a pure-ASCII label +// has no "xn--" prefix and is used as-is, so its punycode-encoded +// length is irrelevant. The ASCII check is a cheap short-circuit that +// avoids running the punycode encoder on labels that don't need it. +// ------------------------------------------------------------------ +function ExceedsMaxALabelLength(value: string): boolean { + return Pattern.RE_NON_ASCII.test(value) && FormatPuny.Encode(value).length + 4 > 63; +} +// ------------------------------------------------------------------ +// HasInvalidHyphens +// +// RFC 5891 §4.2.3.2: label must not start or end with a hyphen, and +// must not have hyphens in the 3rd and 4th codepoint positions (the +// ACE prefix reservation, e.g. "xn--"). Operates on `chars` (the +// per-codepoint string array) rather than raw string indices so +// supplementary-plane characters earlier in the label don't throw +// off the codepoint-position count. +// ------------------------------------------------------------------ +function HasInvalidHyphens(chars: string[]): boolean { + if (chars[0] === "-" || chars[chars.length - 1] === "-") return true; + return chars.slice(2).join("").startsWith("--"); +} +// ------------------------------------------------------------------ +// IsUnicodeLabel +// ------------------------------------------------------------------ +export function IsUnicodeLabel(value: string): boolean { + // RFC 5890 §2.3.2.1: Ensure Max Length + if (ExceedsMaxALabelLength(value)) return false; + // RFC 5893 - Bidi Rule (applies whenever this label itself is RTL) + if (FormatBidi.HasRightToLeftCharacters(value) && !FormatBidi.SatisfiesBidiRule(value)) return false; + const chars = [...value]; + const codePoints = chars.map((c) => c.codePointAt(0)!); + const length = codePoints.length; + // RFC 5891 §4.2.3.2: Hyphen rules + if (HasInvalidHyphens(chars)) return false; + // RFC 5891 §4.2.3.2 - Must not begin with a combining mark + if (Pattern.RE_COMBINING_MARK.test(chars[0])) return false; + let hasJapanese = false; + // let hasArabicIndic = false // (no-spec-coverage) + // let hasExtendedArabicIndic = false // (no-spec-coverage) + for (let i = 0; i < length; i++) { + const codePoint = codePoints[i]; + const char = chars[i]; + // 1. DISALLOWED exceptions + if (Pattern.RE_RFC5892_DISALLOWED.test(char)) return false; + // 1b. General category (PVALID) enforcement + if (!Pattern.RE_PERMITTED_CATEGORY.test(char)) return false; + // 2. Collect Flags + if (Pattern.RE_SCRIPT_JAPANESE.test(char)) hasJapanese = true; + // if (Pattern.RE_ARABIC_INDIC_DIGIT.test(char)) hasArabicIndic = true // (no-spec-coverage) + // if (Pattern.RE_EXT_ARABIC_INDIC_DIGIT.test(char)) hasExtendedArabicIndic = true // (no-spec-coverage) + // 3. CONTEXTO / CONTEXTJ Neighbor Rules + const prev = codePoints[i - 1], + next = codePoints[i + 1]; + switch (codePoint) { + case 0x00b7: + if (prev !== 0x006c || next !== 0x006c) return false; + break; // MIDDLE DOT (Catalan) + case 0x0375: + if (!next || !Pattern.RE_SCRIPT_GREEK.test(chars[i + 1])) return false; + break; // Greek KERAIA + case 0x05f3: + case 0x05f4: + if (!prev || !Pattern.RE_SCRIPT_HEBREW.test(chars[i - 1])) return false; + break; // Hebrew GERESH + case 0x200c: // ZWNJ - RFC 5892 Appendix A.1 + // Note: Full validation requires a Unicode joining type table. We reject + // only when preceded by ASCII (U+0000-U+007F), which can never satisfy + // the Virama or cursive-joining rules. + if (!prev || (prev < 0x0080 && !Pattern.RE_VIRAMA.test(chars[i - 1]))) return false; + break; + case 0x200d: // ZWJ: RFC 5892 Appendix A.2 + if (!prev || !Pattern.RE_VIRAMA.test(chars[i - 1])) return false; + break; + case 0x30fb: // Checked at end via hasJapanese + break; // KATAKANA MIDDLE DOT + } + } + // 4. Global Context Validations (Post-loop) + // RFC 5892 Appendix A.7 - Katakana Middle Dot requirement + if (value.includes("\u30fb") && !hasJapanese) return false; + // RFC 5892 Appendix A.8/A.9 - Mixing Arabic Digits + // if (hasArabicIndic && hasExtendedArabicIndic) return false // (no-spec-coverage) + return true; +} diff --git a/packages/schema/src/formats/typebox/idna/pattern/pattern.ts b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts new file mode 100644 index 000000000..dfb55c1d6 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts @@ -0,0 +1,116 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-fmt-ignore-file + +// ------------------------------------------------------------------- +// Validation Rules +// ------------------------------------------------------------------- +export const RE_RULE_HYPHEN_PLACEMENT = /^(?!-).*(? regexp.source).join("|"), + "u", +); + +export const RE_PERMITTED_CATEGORY = new RegExp( + [ + RE_LETTER, + RE_EUROPEAN_SEPARATOR, + RE_COMMON_SEPARATOR, + RE_NUMBER_DECIMAL, + RE_MARK_NONSPACING, + RE_MARK_SPACING_COMBINING, + RE_CONTEXTO_EXCEPTIONS, + RE_PVALID_EXCEPTIONS, + ] + .map((regexp) => regexp.source) + .join("|"), + "u", +); diff --git a/packages/schema/src/formats/typebox/ipv4.ts b/packages/schema/src/formats/typebox/ipv4.ts new file mode 100644 index 000000000..2c6bf3c7b --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv4.ts @@ -0,0 +1,37 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; + +/** + * Returns true if the value is a IPV4 address + * @specification http://tools.ietf.org/html/rfc2673#section-3.2 + */ +export function IsIPv4(value: string): boolean { + return IPv4.test(value); +} diff --git a/packages/schema/src/formats/typebox/ipv6.ts b/packages/schema/src/formats/typebox/ipv6.ts new file mode 100644 index 000000000..4cc46deac --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv6.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv6 = + /^(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:)?[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)$/i; + +/** + * Returns true if the value is an IPv6 address + * @specification http://tools.ietf.org/html/rfc2373#section-2.2 + */ +export function IsIPv6(value: string): boolean { + return IPv6.test(value); +} diff --git a/packages/schema/src/formats/typebox/iri.ts b/packages/schema/src/formats/typebox/iri.ts new file mode 100644 index 000000000..8ee1d0540 --- /dev/null +++ b/packages/schema/src/formats/typebox/iri.ts @@ -0,0 +1,63 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const IpvFutureMatchMaxLength = 2048; +const IpvFutureMatch = /\[[vV][0-9a-fA-F]+\.[^\]]+\]/; // Guarded By IpvFutureMatchMaxLength +const InvalidIriChars = /[\x00-\x20<>^`{|}\\]/; +const InvalidPercentEncoding = /%(?![0-9a-fA-F]{2})/; + +// ------------------------------------------------------------------ +// NarrowIpvFuture +// +// Substitutes an IPvFuture address with a standard IPv6 loopback +// address ([::1]). We do this because the native URL.canParse +// (WHATWG standard) rejects IPvFuture literals, which are +// otherwise valid in RFC 3987. Because regex substitution can +// be expensive on large strings, this operation is strictly +// limited to inputs under a defined length threshold. +// +// (review-optimization) +// +// ------------------------------------------------------------------ +function NarrowIpvFuture(value: string): string { + return value.length < IpvFutureMatchMaxLength ? value.replace(IpvFutureMatch, "[::1]") : value; +} +/** + * Returns true if the value is a valid Internationalized Resource Identifier. + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsIri(value: string): boolean { + // 1. Reject strings containing unencoded whitespace or illegal control characters. + if (InvalidIriChars.test(value)) return false; + // 2. Reject malformed percent-encoding triplets. + if (InvalidPercentEncoding.test(value)) return false; + // 3. Delegate to the native URL parser, patching the IPvFuture edge case beforehand. + return URL.canParse(NarrowIpvFuture(value)); +} diff --git a/packages/schema/src/formats/typebox/iri_reference.ts b/packages/schema/src/formats/typebox/iri_reference.ts new file mode 100644 index 000000000..5e45c75ee --- /dev/null +++ b/packages/schema/src/formats/typebox/iri_reference.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +// Rejects unencoded spaces (0x20), backslashes, control characters (0x00-0x1F, 0x7F), or malformed percent-encodings +const InvalidIriChars = /[\x00-\x20\x7F\\]|%(?![0-9a-fA-F]{2})/; +// Detects malformed scheme/authority patterns lacking a colon (e.g. "httpx//example.com") +const MalformedScheme = /^[a-zA-Z][a-zA-Z0-9+\-.]*\/\//; + +/** + * Returns true if the value is an IRI reference + * @specification https://tools.ietf.org/html/rfc3987 + */ +export function IsIriReference(value: string): boolean { + return !InvalidIriChars.test(value) && !MalformedScheme.test(value) && URL.canParse(value, "http://example.com"); +} diff --git a/packages/schema/src/formats/typebox/json_pointer.ts b/packages/schema/src/formats/typebox/json_pointer.ts new file mode 100644 index 000000000..c2d171aef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointer = /^(?:\/(?:[^~/]|~0|~1)*)*$/; + +/** + * Returns true if the value is a json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointer(value: string): boolean { + return JsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts new file mode 100644 index 000000000..cac809cef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointerUriFragment = /^#(?:\/(?:[a-z0-9_\-.!$&'()*+,;:=@]|%[0-9a-f]{2}|~0|~1)*)*$/i; + +/** + * Returns true if the value is a json pointer uri fragment + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointerUriFragment(value: string): boolean { + return JsonPointerUriFragment.test(value); +} diff --git a/packages/schema/src/formats/typebox/regex.ts b/packages/schema/src/formats/typebox/regex.ts new file mode 100644 index 000000000..52cff8d2a --- /dev/null +++ b/packages/schema/src/formats/typebox/regex.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a regular expression string pattern + * @specification https://ecma-international.org/ecma-262 + */ +export function IsRegex(value: string): boolean { + try { + new RegExp(value, "u"); + return true; + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/relative_json_pointer.ts b/packages/schema/src/formats/typebox/relative_json_pointer.ts new file mode 100644 index 000000000..f2dfd6fbd --- /dev/null +++ b/packages/schema/src/formats/typebox/relative_json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const RelativeJsonPointer = /^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$/; + +/** + * Returns true if the value is a relative json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsRelativeJsonPointer(value: string): boolean { + return RelativeJsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/time.ts b/packages/schema/src/formats/typebox/time.ts new file mode 100644 index 000000000..bbc9c6113 --- /dev/null +++ b/packages/schema/src/formats/typebox/time.ts @@ -0,0 +1,56 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const TIME = /^(\d\d):(\d\d):(\d\d)(?:\.\d+)?(?:([Zz])|([+-])(\d\d):(\d\d))?$/; + +/** + * Returns true if the value is an ISO time string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + */ +export function IsTime(value: string, strictTimeZone: boolean = true): boolean { + const matches = TIME.exec(value); + if (!matches) return false; + // Require timezone offset or 'Z'/'z' when strictTimeZone is true + if (strictTimeZone && !matches[4] && !matches[5]) return false; + const hr = +matches[1]; + const min = +matches[2]; + const sec = +matches[3]; + if (hr > 23 || min > 59 || sec > 60) return false; + if (matches[5]) { + const tzH = +matches[6]; + const tzM = +matches[7]; + if (tzH > 23 || tzM > 59) return false; + } + if (sec < 60) return true; + // Leap second handling: must normalize to 23:59:60 UTC (1439 total UTC minutes) + const tzSign = matches[5] === "-" ? -1 : 1; + const tzH = +(matches[6] || 0); + const tzM = +(matches[7] || 0); + const totalUtcMin = hr * 60 + min - tzSign * (tzH * 60 + tzM); + return ((totalUtcMin % 1440) + 1440) % 1440 === 1439; +} diff --git a/packages/schema/src/formats/typebox/uri.ts b/packages/schema/src/formats/typebox/uri.ts new file mode 100644 index 000000000..5aadeab17 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uri = + /^[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uniform Resource Identifier. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUri(value: string): boolean { + return Uri.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_reference.ts b/packages/schema/src/formats/typebox/uri_reference.ts new file mode 100644 index 000000000..215058ded --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_reference.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const UriReference = + /^(?:[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?)(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uri Reference. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUriReference(value: string): boolean { + return UriReference.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_template.ts b/packages/schema/src/formats/typebox/uri_template.ts new file mode 100644 index 000000000..ca364fe15 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_template.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const UriTemplate = + /^(?:(?:[^\x00-\x20"<>%\\^`{|}\x7f]|%[0-9a-f]{2})|\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?(?:,(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?)*\})*$/i; + +/** + * Returns true if the value is a valid Uri Template + * @specification https://datatracker.ietf.org/doc/html/rfc6570 + */ +export function IsUriTemplate(value: string): boolean { + return UriTemplate.test(value); +} diff --git a/packages/schema/src/formats/typebox/url.ts b/packages/schema/src/formats/typebox/url.ts new file mode 100644 index 000000000..dfe00c4b8 --- /dev/null +++ b/packages/schema/src/formats/typebox/url.ts @@ -0,0 +1,36 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a valid Uniform Resource Locator + * @specification https://datatracker.ietf.org/doc/html/rfc3986 + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsUrl(value: string): boolean { + return URL.canParse(value); +} diff --git a/packages/schema/src/formats/typebox/uuid.ts b/packages/schema/src/formats/typebox/uuid.ts new file mode 100644 index 000000000..cf26ffa6e --- /dev/null +++ b/packages/schema/src/formats/typebox/uuid.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uuid = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i; + +/** + * Returns true if the value is a uuid + * @specification https://www.rfc-editor.org/info/rfc4122/ + * @specification https://www.rfc-editor.org/info/rfc9562/ + */ +export function IsUuid(value: string): boolean { + return Uuid.test(value); +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts new file mode 100644 index 000000000..c57ede2e9 --- /dev/null +++ b/packages/schema/src/index.ts @@ -0,0 +1,31 @@ +// JSON Schema validation (drafts 2020-12 and 7) without eval: compile() builds a schema into plain closures once. +// The builders return plain JSON Schema objects typed with the values they accept. + +export { + type Annotations, + arr, + bool, + bytes, + int, + type LengthBounds, + lit, + map, + type NumberBounds, + nul, + nullable, + num, + Optional, + obj, + oneOf, + opt, + type Properties, + type Schema, + type Static, + type StaticObject, + str, + tuple, + union, + unknown, +} from "./builders.ts"; +export type { Check, CompileOptions, FormatCheck, JsonSchema, ParseResult, ValidationError } from "./types.ts"; +export { CoercingValidator, compile, Validator } from "./validator.ts"; diff --git a/packages/schema/src/resolver.ts b/packages/schema/src/resolver.ts new file mode 100644 index 000000000..5c8b97543 --- /dev/null +++ b/packages/schema/src/resolver.ts @@ -0,0 +1,203 @@ +// Reference resolution of one compile: schema resources, anchors and JSON Pointers over the root schema and the +// registered documents. A Resolver lives only while a schema compiles (or while its errors are explained). + +import type { CompileOptions, JsonSchema, SchemaObject } from "./types.ts"; + +/** A schema and the base URI its references resolve against. */ +export interface Resolved { + readonly node: unknown; + readonly base: string; +} + +/** The base URI of a root schema without `$id`; relative references resolve against it. */ +export const DEFAULT_BASE = "https://json-schema.invalid/root.json"; + +/** + * Resolves references in the root schema and the registered documents. The schema is indexed on the first + * reference, so a schema without references is never walked for identifiers. + */ +export class Resolver { + readonly draft: "2020-12" | "7"; + readonly rootBase: string; + /** Schema object to the base URI it was indexed under. */ + readonly baseOf = new Map(); + /** URI (with a `#name` fragment) of every `$dynamicAnchor`, to its schema. */ + readonly dynamicAnchors = new Map(); + private readonly root: JsonSchema; + private readonly options: CompileOptions; + private readonly registry = new Map(); + private readonly rejectUnresolved: boolean; + /** The references that named nothing and resolved to `false` (with `unresolvedRefs: "reject"`). */ + private readonly unresolved = new Set(); + private readonly resources = new Map(); + private readonly anchors = new Map(); + private indexed = false; + + constructor(root: JsonSchema, options: CompileOptions) { + this.draft = options.draft === "7" ? "7" : "2020-12"; + this.root = root; + this.options = options; + this.rejectUnresolved = options.unresolvedRefs === "reject"; + this.rootBase = isSchemaObject(root) ? this.baseFor(root, DEFAULT_BASE) : DEFAULT_BASE; + } + + /** + * The base URI of `node`, whose parent has `base`: its `$id` changes it, unless draft 7 makes the `$id` inert. Once + * the schema is indexed, the indexed base (a resolved reference target already has its own). + */ + baseFor(node: SchemaObject, base: string): string { + const indexed = this.baseOf.get(node); + if (indexed !== undefined) return indexed; + const id = node.$id; + if (typeof id !== "string") return base; + if (this.draft === "7" && (typeof node.$ref === "string" || id.startsWith("#"))) return base; + return documentOf(resolveUri(id, base)); + } + + /** Indexes the root schema and the registered documents' URIs (once). */ + ensureIndexed(): void { + if (this.indexed) return; + this.indexed = true; + for (const [uri, schema] of Object.entries(this.options.schemas ?? {})) { + this.registry.set(documentOf(resolveUri(uri, DEFAULT_BASE)), schema); + } + this.resources.set(DEFAULT_BASE, this.root); + this.index(this.root, DEFAULT_BASE); + } + + /** + * The schema a reference names, with the base URI it compiles under. Throws if it names nothing, or, with + * `unresolvedRefs: "reject"`, resolves to `false`, the schema no value passes. + */ + resolve(ref: string, base: string): Resolved { + this.ensureIndexed(); + if (!this.rejectUnresolved) return this.find(ref, base); + try { + return this.find(ref, base); + } catch { + this.unresolved.add(resolveUri(ref, base)); + return { node: false, base }; + } + } + + /** Whether `ref`, from `base`, named nothing and resolved to `false`. */ + isUnresolved(ref: string, base: string): boolean { + return this.unresolved.has(resolveUri(ref, base)); + } + + private find(ref: string, base: string): Resolved { + const uri = resolveUri(ref, base); + const doc = documentOf(uri); + const fragment = uri.length > doc.length ? decodeURIComponent(uri.slice(doc.length + 1)) : ""; + const root = this.resources.has(doc) ? this.resources.get(doc) : this.load(doc); + if (root === undefined) throw new Error(`Unresolved $ref ${ref} (no schema registered for ${doc})`); + if (fragment === "") return { node: root, base: doc }; + if (fragment.startsWith("/")) return this.pointer(root, doc, fragment); + const target = this.anchors.get(`${doc}#${fragment}`); + if (target === undefined) throw new Error(`Unresolved $ref ${ref} (no anchor ${fragment} in ${doc})`); + return { node: target, base: this.baseOf.get(target) ?? doc }; + } + + /** Whether `node` is the root of the schema resource at `base`. */ + isResource(node: object, base: string): boolean { + this.ensureIndexed(); + return this.resources.get(base) === node; + } + + // Walks only keywords that hold subschemas: an `$id` inside `enum`, `const` or an unknown keyword names nothing. + private index(node: unknown, base: string): void { + if (!isSchemaObject(node) || this.baseOf.has(node)) return; + let b = base; + // In draft 7 a `$ref` makes every sibling keyword, `$id` included, inert. + const refOnly = this.draft === "7" && typeof node.$ref === "string"; + if (!refOnly && typeof node.$id === "string") { + if (this.draft === "7" && node.$id.startsWith("#")) this.anchors.set(documentOf(b) + node.$id, node); + else { + const uri = resolveUri(node.$id, b); + b = documentOf(uri); + this.resources.set(b, node); + if (uri.length > b.length + 1) this.anchors.set(uri, node); + } + } + this.baseOf.set(node, b); + if (refOnly) return; + if (this.draft !== "7") { + if (typeof node.$anchor === "string") this.anchors.set(`${b}#${node.$anchor}`, node); + if (typeof node.$dynamicAnchor === "string") { + this.anchors.set(`${b}#${node.$dynamicAnchor}`, node); + this.dynamicAnchors.set(`${b}#${node.$dynamicAnchor}`, node); + } + } + for (const key of SCHEMA_KEYWORDS) this.index(node[key], b); + for (const key of MAP_KEYWORDS) { + const map = node[key]; + if (isSchemaObject(map)) for (const k of Object.keys(map)) this.index(map[k], b); + } + for (const key of ARRAY_KEYWORDS) { + const list = node[key]; + if (Array.isArray(list)) for (const s of list) this.index(s, b); + } + } + + private load(doc: string): JsonSchema | undefined { + const schema = this.registry.get(doc); + if (schema === undefined) return undefined; + this.resources.set(doc, schema); + this.index(schema, doc); + return schema; + } + + private pointer(root: unknown, base: string, fragment: string): Resolved { + let node = root; + let b = base; + for (const raw of fragment.split("/").slice(1)) { + const key = raw.replace(/~1/g, "/").replace(/~0/g, "~"); + node = + typeof node === "object" && node !== null && Object.hasOwn(node, key) + ? (node as SchemaObject)[key] + : undefined; + if (node === undefined) throw new Error(`Unresolved $ref #${fragment} in ${base}`); + if (typeof node === "object" && node !== null) b = this.baseOf.get(node) ?? b; + } + return { node, base: b }; + } +} + +const SCHEMA_KEYWORDS = [ + "additionalProperties", + "propertyNames", + "contains", + "not", + "if", + "then", + "else", + "unevaluatedProperties", + "unevaluatedItems", + "additionalItems", + "contentSchema", + "items", +]; + +const MAP_KEYWORDS = ["properties", "patternProperties", "$defs", "definitions", "dependentSchemas", "dependencies"]; + +const ARRAY_KEYWORDS = ["allOf", "anyOf", "oneOf", "prefixItems", "items"]; + +/** Whether `value` is a schema object (not `true`/`false`, an array or a primitive). */ +export function isSchemaObject(value: unknown): value is SchemaObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** `ref` resolved against `base`. URNs and other bases a URL parser rejects take fragment-only references. */ +export function resolveUri(ref: string, base: string): string { + try { + return new URL(ref, base).href; + } catch { + return ref.startsWith("#") ? documentOf(base) + ref : ref; + } +} + +/** `uri` without its fragment. */ +export function documentOf(uri: string): string { + const hash = uri.indexOf("#"); + return hash < 0 ? uri : uri.slice(0, hash); +} diff --git a/packages/schema/src/shared.ts b/packages/schema/src/shared.ts new file mode 100644 index 000000000..42c30152f --- /dev/null +++ b/packages/schema/src/shared.ts @@ -0,0 +1,33 @@ +// Regular expressions shared across all compiled schemas while one still uses them: tool schemas repeat a few +// patterns many times, and a RegExp is large once it has run. The cache holds its values weakly and drops an entry +// once its RegExp is collected, so it keeps nothing alive. + +/** + * The regular expression of a `pattern` source, shared by source. Patterns are ECMAScript regular expressions with + * the `u` flag; a source that is invalid with it (some older schemas escape characters `u` rejects) compiles + * without it. + */ +export function sharedRegex(source: string): RegExp { + const hit = regexes.get(source)?.deref(); + if (hit !== undefined) return hit; + let re: RegExp; + try { + re = new RegExp(source, "u"); + } catch { + re = new RegExp(source); + } + regexes.set(source, new WeakRef(re)); + regexCleanup.register(re, source); + return re; +} + +const regexes = new Map>(); + +const regexCleanup = new FinalizationRegistry((source) => { + if (regexes.get(source)?.deref() === undefined) regexes.delete(source); +}); + +/** The number of cached regular expressions (for tests). */ +export function sharedRegexCount(): number { + return regexes.size; +} diff --git a/packages/schema/src/typebox-static.ts b/packages/schema/src/typebox-static.ts new file mode 100644 index 000000000..ce9f29b51 --- /dev/null +++ b/packages/schema/src/typebox-static.ts @@ -0,0 +1,76 @@ +// The static types of TypeBox-built schemas, read from the type shapes TypeBox 1.x declares (`'~kind'`, `'~optional'`, +// `'~readonly'`, `'~unsafe'`) without importing TypeBox, so `Static` also types tool arguments of TypeBox schemas. +// Only the common kinds are mapped; every other kind (Enum, Tuple, Intersect, Cyclic, Ref, ...) is `unknown`. + +import type { Static } from "./builders.ts"; + +/** The value type of a TypeBox schema type by its `'~kind'`, or its `Type.Unsafe` type; `unknown` for other kinds. */ +export type TypeBoxStatic = S extends { readonly "~kind": infer K } + ? KindStatic + : S extends { readonly "~unsafe": infer T } + ? T + : unknown; + +interface Primitives { + String: string; + Number: number; + Integer: number; + Boolean: boolean; + Null: null; + Unknown: unknown; + Any: any; +} + +type KindStatic = K extends keyof Primitives + ? Primitives[K] + : K extends "Literal" + ? S extends { readonly const: infer V } + ? V + : unknown + : K extends "Array" + ? S extends { readonly items: infer I } + ? Static[] + : unknown + : K extends "Object" + ? S extends { readonly properties: infer P } + ? ObjectStatic

+ : unknown + : K extends "Union" + ? S extends { readonly anyOf: readonly (infer B)[] } + ? Static + : unknown + : K extends "Record" + ? S extends { readonly patternProperties: infer P } + ? Record, Static> + : unknown + : unknown; + +type Optional = { readonly "~optional": true }; + +type Readonly = { readonly "~readonly": true }; + +// As TypeBox: no properties is `object`; `'~optional'` properties may be omitted, `'~readonly'` ones are readonly. +type ObjectStatic

= keyof P extends never + ? object + : Simplify< + { -readonly [K in keyof P as P[K] extends Optional | Readonly ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? never : K) : never]?: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Readonly ? (P[K] extends Optional ? never : K) : never]: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? K : never) : never]?: Static< + P[K] + >; + } + >; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +// TypeBox's key patterns for integer and number keys; every other pattern types its keys as strings. +type RecordKey = Pattern extends "^-?(?:0|[1-9][0-9]*)$" | "^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$" + ? number + : string; diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts new file mode 100644 index 000000000..d3f635adb --- /dev/null +++ b/packages/schema/src/types.ts @@ -0,0 +1,49 @@ +// Public types: the schema accepted by compile(), its options, and the errors a failed value produces. + +/** + * A JSON Schema: `true`, `false` or a schema object. Schemas come from tool definitions, MCP servers and other + * untrusted sources, so compile() reads every keyword defensively instead of trusting a static shape. + */ +export type JsonSchema = boolean | object; + +/** A format check: whether `value` (a string) has the format. */ +export type FormatCheck = (value: string) => boolean; + +/** Options of compile(). */ +export interface CompileOptions { + /** The draft whose semantics apply: "2020-12" (default) or "7". */ + readonly draft?: "2020-12" | "7"; + /** + * Format name to check. Without it `format` is an annotation, as draft 2020-12 specifies. The standard formats are + * in `@earendil-works/pi-schema/formats`. + */ + readonly formats?: Readonly>; + /** Schema documents by URI, for `$ref`s to other documents. */ + readonly schemas?: Readonly>; + /** Add `coerce()` and `parse()` to the validator, for loosely typed input (see the README's Coercion section). */ + readonly coerce?: boolean; + /** + * What a `$ref` that names nothing (a missing definition, a document not in `schemas`) does: `"throw"` (default) + * makes compile() throw; `"reject"` compiles it as `false`, so only values at that place fail, with the error + * "unresolved reference". + */ + readonly unresolvedRefs?: "throw" | "reject"; +} + +/** The result of `parse()`: the coerced value when it is valid, else the errors of the coerced value. */ +export type ParseResult = + | { readonly ok: true; readonly value: T } + | { readonly ok: false; readonly errors: ValidationError[] }; + +/** One failure of a value: where it is (a JSON Pointer, "" for the value itself), the failing keyword and a message. */ +export interface ValidationError { + readonly path: string; + readonly keyword: string; + readonly message: string; +} + +/** A schema object as compile() reads it: any keyword, of any value, checked where it is used. */ +export type SchemaObject = { readonly [keyword: string]: unknown }; + +/** A compiled boolean check. */ +export type Check = (value: unknown) => boolean; diff --git a/packages/schema/src/validator.ts b/packages/schema/src/validator.ts new file mode 100644 index 000000000..65781e5dd --- /dev/null +++ b/packages/schema/src/validator.ts @@ -0,0 +1,93 @@ +// compile(): a schema to a Validator, whose `check` is the fast boolean path and whose `errors` explains failures. + +import type { Static } from "./builders.ts"; +import { Coercer } from "./coerce.ts"; +import { Compiler, RESTART } from "./compiler.ts"; +import { Explainer } from "./explain.ts"; +import type { Check, CompileOptions, JsonSchema, ParseResult, ValidationError } from "./types.ts"; + +/** + * Compiles `schema` into a Validator. Throws if the schema is malformed or a `$ref` names nothing (register other + * documents with `options.schemas`). The schema is read, never modified. With `coerce: true` the validator also + * coerces and parses loosely typed input. + */ +export function compile( + schema: S, + options: CompileOptions & { readonly coerce: true }, +): CoercingValidator>; +export function compile(schema: S, options?: CompileOptions): Validator>; +export function compile( + schema: S, + options: CompileOptions = NO_OPTIONS, +): Validator> | CoercingValidator> { + const check = compileCheck(schema, options); + return options.coerce === true + ? new CoercingValidator(schema, options, check) + : new Validator(schema, options, check); +} + +const NO_OPTIONS: CompileOptions = Object.freeze({}); + +/** A compiled schema whose valid values have type `T` (`unknown` unless the schema was built with the builders). */ +export class Validator { + /** Whether `value` is valid. A plain function: it may be passed around without its validator. */ + readonly check: (value: unknown) => value is T; + protected readonly schema: JsonSchema; + protected readonly options: CompileOptions; + private explainer: Explainer | undefined = undefined; + + constructor(schema: JsonSchema, options: CompileOptions, check: Check) { + // A compiled check is exactly the test that a value has the schema's static type. + this.check = check as (value: unknown) => value is T; + this.schema = schema; + this.options = options; + } + + /** + * Why `value` is invalid: one error per failing keyword, `[]` when it is valid. Built on the first failure and + * kept, so a validator whose values always pass never pays for it. + */ + errors(value: unknown): ValidationError[] { + if (this.check(value)) return []; + this.explainer ??= new Explainer(compiler(this.schema, this.options), this.schema); + return this.explainer.errors(value); + } +} + +/** A validator compiled with `coerce: true`: it also coerces loosely typed input towards the schema. */ +export class CoercingValidator extends Validator { + private coercer: Coercer | undefined = undefined; + + /** + * `value` coerced towards the schema: numeric strings to numbers, `"true"` to `true`, a single value to a one-item + * array, an optional `null` dropped, the first union branch the value converts to (the README lists every rule). + * Only the changed paths are copied; `value` comes back itself when nothing changes, and is never modified. Values + * that cannot be coerced stay as they are. The rules are built on the first call. + */ + coerce(value: unknown): unknown { + this.coercer ??= new Coercer(compiler(this.schema, this.options), this.schema); + return this.coercer.coerce(value); + } + + /** Coerces `value`, then checks it: the coerced value when it is valid, else its errors. */ + parse(value: unknown): ParseResult { + const coerced = this.coerce(value); + return this.check(coerced) ? { ok: true, value: coerced } : { ok: false, errors: this.errors(coerced) }; + } +} + +function compileCheck(schema: JsonSchema, options: CompileOptions): Check { + return compiler(schema, options).root(); +} + +// Most schemas compile without tracking the dynamic scope; one with a bookended $dynamicRef compiles again with it. +function compiler(schema: JsonSchema, options: CompileOptions): Compiler { + const plain = new Compiler(schema, options, false); + try { + plain.root(); + return plain; + } catch (error) { + if (error !== RESTART) throw error; + return new Compiler(schema, options, true); + } +} diff --git a/packages/schema/test/bench/all.ts b/packages/schema/test/bench/all.ts new file mode 100644 index 000000000..96ce485c4 --- /dev/null +++ b/packages/schema/test/bench/all.ts @@ -0,0 +1,40 @@ +// Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables. +// Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP. +// The children run with ATA_NO_NATIVE=1 (see candidates.ts). +// Run: npm run bench [-- --out ] (default: pi-schema-bench-results.json in the temp directory) + +import { execFileSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = import.meta.dirname; +const outIndex = process.argv.indexOf("--out"); +const out = outIndex >= 0 ? process.argv[outIndex + 1] : join(tmpdir(), "pi-schema-bench-results.json"); + +const runs: [string, string, string[]][] = [ + ["conformance", "conformance.ts", []], + ["speed", "speed.ts", []], + ["speedNoEval", "speed.ts", ["--disallow-code-generation-from-strings"]], + ["setup", "setup.ts", []], + ["memory", "memory.ts", []], + ["size", "size.ts", []], +]; + +const results: Record = { + node: process.version, + cpu: cpus()[0]?.model ?? "unknown", + date: new Date().toISOString(), +}; +for (const [key, file, flags] of runs) { + const json = join(tmpdir(), `pi-schema-bench-${key}.json`); + console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`); + execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { + stdio: "inherit", + env: { ...process.env, ATA_NO_NATIVE: "1" }, + }); + results[key] = JSON.parse(readFileSync(json, "utf8")); + rmSync(json); +} +writeFileSync(out, JSON.stringify(results, null, "\t")); +console.log(`\nResults: ${out}`); diff --git a/packages/schema/test/bench/candidates.ts b/packages/schema/test/bench/candidates.ts new file mode 100644 index 000000000..ce0c6d4cf --- /dev/null +++ b/packages/schema/test/bench/candidates.ts @@ -0,0 +1,205 @@ +// The validators the benchmarks compare, each behind the same interface: compile a schema, then check values. +// Every library is used through its fastest boolean path and keeps the object it returns, as an application would. +// Run the benchmarks through `npm run bench`, which sets ATA_NO_NATIVE=1: ata-validator's optional native addon only +// serves its buffer APIs, and the verdicts measured here are JavaScript either way. + +import { Validator as CfValidator } from "@cfworker/json-schema"; +import { fromJsonSchema } from "@oh-my-pi/omptype"; +import Ajv7Module from "ajv"; +import Ajv2020Module from "ajv/dist/2020.js"; +import draft07MetaSchema from "ajv/dist/refs/json-schema-draft-07.json" with { type: "json" }; +import addFormatsModule from "ajv-formats"; +import { Validator as AtaValidator } from "ata-validator"; +import { Compile } from "typebox/compile"; +import { ResetExternal, ResetFunctions } from "typebox/schema"; +import { Settings } from "typebox/system"; +import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts"; +import { nativeCases, nativeZod, zodFromJsonSchema } from "./zod.ts"; + +/** How a benchmark compiles a schema. */ +export interface PrepareOptions { + readonly draft: "2020-12" | "7"; + readonly schemas?: Readonly>; + readonly formats?: Readonly>; + /** The schema is a JSON-Schema-Test-Suite schema (for libraries that take native schemas otherwise). */ + readonly suite?: boolean; +} + +/** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */ +export interface Prepared { + readonly check: (value: unknown) => boolean; + readonly validator: unknown; +} + +/** A library under comparison. */ +export interface Candidate { + readonly name: string; + readonly prepare: (schema: JsonSchema, options: PrepareOptions) => Prepared; +} + +/** Whether this process runs with code generation from strings disabled, as under a strict CSP. */ +export const noEval = process.execArgv.includes("--disallow-code-generation-from-strings"); + +const piSchema: Candidate = { + name: "pi-schema", + prepare(schema, options) { + const validator = compile(schema, options); + return { check: validator.check, validator }; + }, +}; + +// TypeBox keeps its last build in module-level accumulators; reset them so no build state is retained between schemas. +function typeboxCompile(schema: JsonSchema, options: PrepareOptions, accelerate: boolean) { + Settings.Set({ useAcceleration: accelerate }); + try { + return Compile((options.schemas ?? {}) as never, schema as never); + } finally { + ResetFunctions(); + ResetExternal(); + Settings.Set({ useAcceleration: true }); + } +} + +const typebox: Candidate = { + name: noEval ? "typebox (no eval: interpreter)" : "typebox", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, true); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const typeboxInterpreter: Candidate = { + name: "typebox interpreter", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, false); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const cfworker: Candidate = { + name: "cfworker", + prepare(schema, options) { + const validator = new CfValidator(schema as never, options.draft, true); + for (const [uri, remote] of Object.entries(options.schemas ?? {})) validator.addSchema(remote as never, uri); + return { check: (v) => validator.validate(v).valid, validator }; + }, +}; + +// omptype compiles with new Function on the first allows() call. Without eval that call throws EvalError once per +// schema; omptype then keeps validating through its interpreter, so the benchmark absorbs that one throw. +const omptype: Candidate = { + name: noEval ? "omptype (no eval: interpreter)" : "omptype", + prepare(schema) { + const validator = fromJsonSchema(schema); + if (noEval) { + try { + validator.allows(null); + } catch (error) { + if (!(error instanceof EvalError)) throw error; + } + } + return { check: (v) => validator.allows(v), validator }; + }, +}; + +// Ajv's CommonJS modules export the class as module.exports and as `default`; the latter is what its types describe. +const Ajv2020 = Ajv2020Module.default; +const Ajv7 = Ajv7Module.default; +const addFormats = addFormatsModule.default; +type AjvInstance = InstanceType; + +// One Ajv per draft, as an application keeps it. Ajv2020 also gets the draft-07 meta-schema, so tool schemas that +// declare draft-07 compile. strict: false so schemas written for other tools compile; logger: false keeps Ajv from +// printing generated code when compiling fails without eval. +function newAjv(draft: "2020-12" | "7", formats: boolean): AjvInstance { + const options = { strict: false, validateFormats: formats, logger: false as const }; + const ajv = draft === "7" ? new Ajv7(options) : new Ajv2020(options); + if (draft !== "7") ajv.addMetaSchema(draft07MetaSchema); + if (formats) addFormats(ajv); + return ajv; +} + +const sharedAjv = new Map(); + +// Ajv compiles with new Function and has no interpreter: without eval, compile() throws EvalError. +const ajv: Candidate = { + name: noEval ? "ajv (no eval: fails)" : "ajv", + prepare(schema, options) { + let instance: AjvInstance; + if (options.schemas === undefined) { + const key = `${options.draft} ${options.formats !== undefined}`; + instance = sharedAjv.get(key) ?? newAjv(options.draft, options.formats !== undefined); + sharedAjv.set(key, instance); + } else { + // The suite reuses $ids across groups, which one shared instance would reject: a fresh one per schema. + instance = newAjv(options.draft, options.formats !== undefined); + for (const [uri, remote] of Object.entries(options.schemas)) { + // Ajv ships its own meta-schemas; adding them again under the same id throws. + if (instance.getSchema(uri) === undefined) { + try { + instance.addSchema(remote as object, uri); + } catch { + // A remote whose $id Ajv already registered under another key. + } + } + } + } + const validator = instance.compile(schema as object); + return { check: (v) => validator(v) as boolean, validator }; + }, +}; + +// ata-validator: its defaults fill `default` values into the input and assert `format`; both are turned off to match +// the other candidates (format checks only in the format run, with its own checks). It reads the draft only from +// $schema, so a draft 7 schema without one gets it declared. +function ataValidator(schema: JsonSchema, options: PrepareOptions, engine: "interpreter" | undefined) { + const declared = + options.draft === "7" && typeof schema === "object" && !("$schema" in schema) + ? { $schema: "http://json-schema.org/draft-07/schema#", ...schema } + : schema; + return new AtaValidator(declared, { + useDefaults: false, + assertFormat: options.formats !== undefined, + schemas: (options.schemas ?? {}) as Record, + logger: false, + ...(engine === undefined ? {} : { engine }), + }); +} + +// Generated code, or its interpreted engine where code generation is blocked. +const ata: Candidate = { + name: noEval ? "ata (no eval: interpreter)" : "ata", + prepare(schema, options) { + const validator = ataValidator(schema, options, undefined); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Its eval-free engine, a tree of closures, chosen explicitly: what it runs on for schemas from outside. +const ataInterpreter: Candidate = { + name: "ata interpreter", + prepare(schema, options) { + const validator = ataValidator(schema, options, "interpreter"); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Zod: native Zod schemas for the speed cases and the corpus, z.fromJSONSchema only for the suite. Its object parser +// generates code with new Function when it can and parses without it otherwise; safeParse is its boolean path. +const zod: Candidate = { + name: noEval ? "zod (no eval: no JIT)" : "zod", + prepare(schema, options) { + const validator = options.suite + ? zodFromJsonSchema(schema, options.draft) + : (nativeCases.get(schema) ?? nativeZod(schema as Record)); + return { check: (v) => validator.safeParse(v).success, validator }; + }, +}; + +/** Every candidate, ours first. */ +export const candidates: readonly Candidate[] = [piSchema, typebox, ajv, ata, ataInterpreter, zod, cfworker, omptype]; + +/** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */ +export const byName: ReadonlyMap = new Map( + [...candidates, typeboxInterpreter].map((c) => [c.name.split(" (")[0], c]), +); diff --git a/packages/schema/test/bench/cases.ts b/packages/schema/test/bench/cases.ts new file mode 100644 index 000000000..d6821df1e --- /dev/null +++ b/packages/schema/test/bench/cases.ts @@ -0,0 +1,261 @@ +// Speed cases: realistic schemas with a valid and an invalid value each. Argument checks shaped like Chord's +// contracts (closed objects, tagged unions), pi's built-in tool schemas, and MCP tool schemas with $ref and anyOf. + +import type { JsonSchema } from "../../src/index.ts"; + +/** A schema, the draft it is written in, and one valid and one invalid value. */ +export interface Case { + readonly name: string; + readonly schema: JsonSchema; + readonly valid: unknown; + readonly invalid: unknown; +} + +type Props = Record; +const closed = (properties: Props, optional: string[] = []) => ({ + type: "object", + properties, + required: Object.keys(properties).filter((k) => !optional.includes(k)), + additionalProperties: false, +}); +const str = (o: object = {}) => ({ type: "string", ...o }); +const int = (o: object = {}) => ({ type: "integer", ...o }); +const num = (o: object = {}) => ({ type: "number", ...o }); +const lit = (v: string) => ({ const: v }); +const arr = (items: object, o: object = {}) => ({ type: "array", items, ...o }); +const uuid = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + +const block = { + anyOf: [ + closed({ kind: lit("text"), text: str() }), + closed({ kind: lit("image"), src: str(), width: int({ minimum: 0 }), height: int({ minimum: 0 }), alt: str() }, [ + "alt", + ]), + closed({ kind: lit("code"), lang: { anyOf: [lit("ts"), lit("js"), lit("py")] }, source: str() }), + ], +}; + +const document = closed( + { + id: str({ pattern: uuid }), + title: str({ minLength: 1, maxLength: 200 }), + tags: arr(str(), { maxItems: 16 }), + author: closed({ name: str(), email: str() }, ["email"]), + blocks: arr(block), + score: num({ minimum: 0, maximum: 1 }), + draft: { type: "boolean" }, + }, + ["score", "draft"], +); + +const documentValue = { + id: "01890a5d-ac96-774b-bcce-b302099a8057", + title: "Quarterly notes", + tags: ["a", "b", "c"], + author: { name: "Mario", email: "m@example.com" }, + blocks: [ + { kind: "text", text: "Hello" }, + { kind: "image", src: "a.png", width: 640, height: 480 }, + { kind: "code", lang: "ts", source: "const x = 1;" }, + { kind: "text", text: "World" }, + { kind: "image", src: "b.png", width: 10, height: 20, alt: "b" }, + ], + score: 0.5, +}; + +const id = int({ minimum: 0 }); +const message = { + anyOf: [ + closed({ + kind: lit("hello"), + protocol: int({ minimum: 0 }), + incarnation: str({ pattern: uuid }), + calls: int({ minimum: 1 }), + }), + closed( + { + kind: lit("hold"), + hold: id, + type: str(), + version: int({ minimum: 1 }), + fingerprint: str({ pattern: "^[0-9a-f]{16}$" }), + params: {}, + address: closed({ kind: str(), params: {} }), + chain: arr(str({ pattern: uuid }), { maxItems: 64 }), + }, + ["chain"], + ), + closed({ kind: lit("release"), hold: id }), + closed({ kind: lit("call"), call: id, hold: id, method: id, args: {} }, ["args"]), + closed({ kind: lit("abort"), call: id }), + ], +}; + +const tool = (properties: Props, required: string[]) => ({ type: "object", properties, required }); + +const mcpIssue = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + owner: str({ minLength: 1, pattern: "^[A-Za-z0-9-]+$" }), + repo: str({ minLength: 1, pattern: "^[A-Za-z0-9._-]+$" }), + title: str({ minLength: 1 }), + body: str(), + labels: arr(str()), + assignees: arr(str()), + milestone: { anyOf: [{ type: "number" }, { type: "null" }] }, + state: { type: "string", enum: ["open", "closed"] }, + }, + required: ["owner", "repo", "title"], + additionalProperties: false, +}; + +const mcpEdit = { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + $defs: { + Range: closed({ start: int({ minimum: 1 }), end: int({ minimum: 1 }) }), + Replace: closed({ oldText: str(), newText: str() }), + LineEdit: closed({ range: { $ref: "#/$defs/Range" }, text: str() }), + }, + properties: { + path: str({ pattern: "^(/|\\./|[A-Za-z]:\\\\)?[^\\u0000]+$" }), + edits: arr({ anyOf: [{ $ref: "#/$defs/Replace" }, { $ref: "#/$defs/LineEdit" }] }, { minItems: 1 }), + dryRun: { type: "boolean" }, + }, + required: ["path", "edits"], + additionalProperties: false, +}; + +const mcpPage = { + type: "object", + $defs: { + RichText: closed( + { + type: { enum: ["text", "mention"] }, + content: str({ maxLength: 2000 }), + annotations: closed({ bold: { type: "boolean" }, italic: { type: "boolean" }, color: str() }, [ + "bold", + "italic", + "color", + ]), + }, + ["annotations"], + ), + Block: { + type: "object", + properties: { + type: { enum: ["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"] }, + rich_text: arr({ $ref: "#/$defs/RichText" }), + checked: { type: "boolean" }, + children: arr({ $ref: "#/$defs/Block" }), + }, + required: ["type", "rich_text"], + additionalProperties: false, + }, + }, + properties: { + parent: { oneOf: [closed({ page_id: str({ pattern: uuid }) }), closed({ database_id: str({ pattern: uuid }) })] }, + title: str({ minLength: 1 }), + due: str({ pattern: "^\\d{4}-\\d{2}-\\d{2}$" }), + children: arr({ $ref: "#/$defs/Block" }, { maxItems: 100 }), + }, + required: ["parent", "title"], + additionalProperties: false, +}; + +const rt = (content: string) => ({ + type: "text", + content, + annotations: { bold: false, italic: false, color: "default" }, +}); +const pageValue = { + parent: { page_id: "01890a5d-ac96-774b-bcce-b302099a8057" }, + title: "Plan", + due: "2026-11-01", + children: [ + { type: "heading_1", rich_text: [rt("Plan")] }, + { + type: "bulleted_list_item", + rich_text: [rt("one")], + children: [{ type: "to_do", rich_text: [rt("sub")], checked: true }], + }, + { type: "paragraph", rich_text: [rt("a"), rt("b"), rt("c")] }, + ], +}; +const pageInvalid = structuredClone(pageValue); +pageInvalid.children[1].children = [{ type: "to_do", rich_text: [rt("sub")], checked: "yes" as unknown as boolean }]; + +/** The speed cases. */ +export const cases: readonly Case[] = [ + { name: "small closed object", schema: closed({ text: str() }), valid: { text: "hello" }, invalid: { text: 1 } }, + { + name: "nested object with tagged unions", + schema: document, + valid: documentValue, + invalid: { + ...documentValue, + blocks: [...documentValue.blocks.slice(0, 4), { kind: "image", src: "b.png", width: -1, height: 20 }], + }, + }, + { + name: "message union of 5", + schema: message, + valid: { kind: "call", call: 7, hold: 3, method: 1, args: { a: 1 } }, + invalid: { kind: "call", call: 7, hold: 3, method: 1, extra: true }, + }, + { + name: "tool: read", + schema: tool({ path: str({ description: "Path to the file" }), offset: num(), limit: num() }, ["path"]), + valid: { path: "src/a.ts", offset: 10, limit: 50 }, + invalid: { offset: 1 }, + }, + { + name: "tool: edit", + schema: tool({ path: str(), edits: arr(tool({ oldText: str(), newText: str() }, ["oldText", "newText"])) }, [ + "path", + "edits", + ]), + valid: { + path: "a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { oldText: "c", newText: "d" }, + ], + }, + invalid: { path: "a.ts", edits: [{ oldText: "a" }] }, + }, + { + name: "tool: bash", + schema: tool({ command: str(), timeout: num() }, ["command"]), + valid: { command: "ls -la", timeout: 30 }, + invalid: { command: 5 }, + }, + { + name: "MCP create_issue (pattern, anyOf, enum)", + schema: mcpIssue, + valid: { + owner: "earendil-works", + repo: "pi", + title: "Bug", + body: "...", + labels: ["bug"], + milestone: null, + state: "open", + }, + invalid: { owner: "earendil works", repo: "pi", title: "Bug" }, + }, + { + name: "MCP edit_file ($ref, anyOf)", + schema: mcpEdit, + valid: { + path: "./src/a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { range: { start: 1, end: 3 }, text: "x" }, + ], + }, + invalid: { path: "./src/a.ts", edits: [{ range: { start: 0, end: 3 }, text: "x" }] }, + }, + { name: "MCP create_page (recursive $ref, oneOf)", schema: mcpPage, valid: pageValue, invalid: pageInvalid }, +]; diff --git a/packages/schema/test/bench/conformance.ts b/packages/schema/test/bench/conformance.ts new file mode 100644 index 000000000..9d8977d76 --- /dev/null +++ b/packages/schema/test/bench/conformance.ts @@ -0,0 +1,56 @@ +// Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section: +// core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run +// with pi-schema's format checks (the other libraries check formats on their own or not at all). Zod goes through +// z.fromJSONSchema, which refuses some keywords and every remote $ref: a schema it refuses fails all its tests. +// Run: node test/bench/conformance.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { formats } from "../../src/formats/index.ts"; +import { suiteFiles, suiteSchemas } from "../suite.ts"; +import { candidates } from "./candidates.ts"; + +/** Passed and total tests by section ("draft2020-12 core", ...) and candidate name. */ +export type ConformanceResults = Record }>; + +const results: ConformanceResults = {}; +for (const file of suiteFiles()) { + const kind = file.name.includes("/optional/format/") + ? "format" + : file.name.includes("/optional/") + ? "optional" + : "core"; + const section = `${file.name.split("/")[0]} ${kind}`; + results[section] ??= { total: 0, passed: Object.fromEntries(candidates.map((c) => [c.name, 0])) }; + for (const group of file.groups) { + results[section].total += group.tests.length; + for (const candidate of candidates) { + let check: ((v: unknown) => boolean) | undefined; + try { + check = candidate.prepare(structuredClone(group.schema), { + draft: file.draft, + schemas: suiteSchemas(), + formats: kind === "format" ? formats : undefined, + suite: true, + }).check; + } catch { + continue; // A schema the library cannot compile fails all its tests. + } + for (const test of group.tests) { + try { + if (check(test.data) === test.valid) results[section].passed[candidate.name]++; + } catch { + // A throw while checking fails the test. + } + } + } + } +} + +const names = candidates.map((c) => c.name); +console.log(`| section | tests | ${names.join(" | ")} |`); +console.log(`|---|---:|${names.map(() => "---:").join("|")}|`); +for (const [section, r] of Object.entries(results)) { + console.log(`| ${section} | ${r.total} | ${names.map((n) => r.passed[n]).join(" | ")} |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/corpus.ts b/packages/schema/test/bench/corpus.ts new file mode 100644 index 000000000..2a985db8e --- /dev/null +++ b/packages/schema/test/bench/corpus.ts @@ -0,0 +1,122 @@ +// A deterministic corpus of distinct tool-like JSON Schemas with one valid value each: random property names, +// descriptions, leaf constraints, enums, nested objects, arrays and some $defs/$ref. No two schemas are copies. + +/** A schema and a value it accepts. */ +export interface CorpusItem { + readonly schema: Record; + readonly value: unknown; +} + +const SYLLABLES = [ + "ka", + "lo", + "mi", + "re", + "su", + "ta", + "ne", + "vo", + "pi", + "da", + "ru", + "xe", + "bo", + "fi", + "gu", + "ha", + "jo", + "ze", +]; +const PATTERNS: readonly (readonly [string, string])[] = [ + ["^[a-z0-9-]+$", "a-1"], + ["^[A-Za-z0-9._-]+$", "A.b_c"], + ["^\\d{4}-\\d{2}-\\d{2}$", "2026-10-31"], + ["^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", "01890a5d-ac96-774b-bcce-b302099a8057"], + ["^#[0-9a-f]{6}$", "#00ff00"], +]; + +/** `count` distinct schemas, the same for the same seed. */ +export function corpus(count: number, seed = 42): CorpusItem[] { + const random = mulberry32(seed); + const int = (lo: number, hi: number) => lo + Math.floor(random() * (hi - lo + 1)); + const pick = (list: readonly T[]): T => list[Math.floor(random() * list.length)]; + const word = () => Array.from({ length: int(2, 4) }, () => pick(SYLLABLES)).join(""); + const sentence = () => `${Array.from({ length: int(4, 12) }, word).join(" ")}.`; + + function leaf(depth: number): [Record, unknown] { + const r = random(); + if (r < 0.3) return [{ type: "string", description: sentence() }, word()]; + if (r < 0.4) { + const min = int(0, 3); + return [ + { type: "string", minLength: min, maxLength: min + int(5, 200), description: sentence() }, + "x".repeat(min + 1), + ]; + } + if (r < 0.47) { + const [pattern, value] = pick(PATTERNS); + return [{ type: "string", pattern, description: sentence() }, value]; + } + if (r < 0.55) { + const values = Array.from({ length: int(2, 5) }, word); + return [{ type: "string", enum: values, description: sentence() }, values[0]]; + } + if (r < 0.65) { + const min = int(0, 10); + return [{ type: "integer", minimum: min, maximum: min + int(10, 10000), description: sentence() }, min + 1]; + } + if (r < 0.72) return [{ type: "number", description: sentence() }, random() * 100]; + if (r < 0.8) return [{ type: "boolean", description: sentence() }, random() < 0.5]; + if (r < 0.88) { + return [{ type: "array", items: { type: "string" }, maxItems: int(1, 50), description: sentence() }, [word()]]; + } + if (depth < 2) return object(depth + 1, int(2, 5)); + return [{ type: "string", description: sentence() }, word()]; + } + + function object(depth: number, n: number): [Record, Record] { + const properties: Record> = {}; + const value: Record = {}; + const required: string[] = []; + for (let i = 0; i < n; i++) { + const name = word() + (i > 0 ? i : ""); + const [schema, v] = leaf(depth); + properties[name] = schema; + if (random() < 0.6) { + required.push(name); + value[name] = v; + } else if (random() < 0.5) value[name] = v; + } + const schema: Record = { type: "object", properties, required }; + if (random() < 0.5) schema.additionalProperties = false; + return [schema, value]; + } + + const items: CorpusItem[] = []; + for (let i = 0; i < count; i++) { + const [schema, value] = object(0, int(2, 10)); + const properties = schema.properties as Record>; + if (random() < 0.2) { + // Move one nested object into $defs and reference it, as MCP servers built with zod often do. + const key = Object.keys(properties).find((k) => properties[k].type === "object"); + if (key !== undefined) { + const name = `${word()}Def`; + schema.$defs = { [name]: properties[key] }; + properties[key] = { $ref: `#/$defs/${name}` }; + } + } + schema.description = sentence(); + items.push({ schema, value }); + } + return items; +} + +function mulberry32(seed: number): () => number { + let state = seed; + return () => { + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/packages/schema/test/bench/memory.ts b/packages/schema/test/bench/memory.ts new file mode 100644 index 000000000..9f2c1546b --- /dev/null +++ b/packages/schema/test/bench/memory.ts @@ -0,0 +1,73 @@ +// Retained heap of 3,200 distinct compiled schemas per candidate. Each candidate runs in its own process with +// --expose-gc: the schemas and values are built before the baseline, every library's returned validator is kept, +// and the heap is read after forced collections, once after compiling and once after checking each value 3 times +// (when libraries that compile lazily, like omptype, have compiled). +// Run: node test/bench/memory.ts [--json ] + +import { execFileSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { byName, candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 3200); + +/** MB retained by the schemas themselves, and by each candidate after compiling and after checking. */ +export interface MemoryResults { + readonly count: number; + readonly schemasMB: number; + readonly candidates: Record; +} + +const child = process.argv.indexOf("--child"); +if (child >= 0) measure(process.argv[child + 1]); +else run(); + +function run(): void { + const results: MemoryResults = { count: COUNT, schemasMB: 0, candidates: {} }; + const names = [...candidates.map((c) => c.name), "typebox interpreter"]; + for (const name of names) { + const output = execFileSync(process.execPath, ["--expose-gc", import.meta.filename, "--child", name], { + encoding: "utf8", + env: { ...process.env, COUNT: String(COUNT) }, + }); + const result = JSON.parse(output); + Object.assign(results, { schemasMB: result.schemasMB }); + results.candidates[name] = { compiledMB: result.compiledMB, checkedMB: result.checkedMB }; + } + console.log(`${COUNT} distinct schemas take ${results.schemasMB.toFixed(1)} MB themselves (not counted below).\n`); + console.log("| candidate | after compiling | after checking |"); + console.log("|---|---:|---:|"); + for (const [name, r] of Object.entries(results.candidates)) { + console.log(`| ${name} | ${r.compiledMB.toFixed(1)} MB | ${r.checkedMB.toFixed(1)} MB |`); + } + const out = process.argv.indexOf("--json"); + if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +} + +function measure(name: string): void { + const candidate = byName.get(name); + if (candidate === undefined) throw new Error(`Unknown candidate ${name}`); + const gc = (globalThis as { gc?: () => void }).gc; + if (gc === undefined) throw new Error("Run with --expose-gc"); + const heap = () => { + for (let i = 0; i < 4; i++) gc(); + return process.memoryUsage().heapUsed; + }; + // Warm the library's own code on another corpus, so its lazily compiled functions are not counted. + for (const { schema, value } of corpus(100, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 4; i++) check(value); + } + const empty = heap(); + const items = corpus(COUNT, 42); + const base = heap(); + const prepared = items.map(({ schema }) => candidate.prepare(schema, { draft: "2020-12" })); + const compiled = heap(); + for (let i = 0; i < items.length; i++) for (let k = 0; k < 3; k++) prepared[i].check(items[i].value); + const checked = heap(); + (globalThis as { keep?: unknown }).keep = [items, prepared]; + const mb = (bytes: number) => bytes / 1048576; + console.log( + JSON.stringify({ schemasMB: mb(base - empty), compiledMB: mb(compiled - base), checkedMB: mb(checked - base) }), + ); +} diff --git a/packages/schema/test/bench/setup.ts b/packages/schema/test/bench/setup.ts new file mode 100644 index 000000000..ccf188c2b --- /dev/null +++ b/packages/schema/test/bench/setup.ts @@ -0,0 +1,51 @@ +// Setup cost: compiling a schema the library has not seen, and the time of its first checks. Uses distinct +// schemas from the corpus after warming each library's own code, so it measures a new schema, not a new process. +// Run: node test/bench/setup.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 1000); + +/** Mean µs per schema: compile, then each of the first three checks and the mean of checks 4-10. */ +export type SetupResults = Record; + +const results: SetupResults = {}; +for (const candidate of candidates) { + for (const { schema, value } of corpus(200, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 10; i++) check(value); + } + const items = corpus(COUNT, 99); + let compile = 0; + const checks = [0, 0, 0, 0]; + for (const { schema, value } of items) { + let start = process.hrtime.bigint(); + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + compile += Number(process.hrtime.bigint() - start); + for (let k = 0; k < 3; k++) { + start = process.hrtime.bigint(); + check(value); + checks[k] += Number(process.hrtime.bigint() - start); + } + start = process.hrtime.bigint(); + for (let k = 3; k < 10; k++) check(value); + checks[3] += Number(process.hrtime.bigint() - start) / 7; + } + const us = (ns: number) => ns / COUNT / 1000; + results[candidate.name] = { + compileUs: us(compile), + checkUs: [us(checks[0]), us(checks[1]), us(checks[2]), us(checks[3])], + }; +} + +console.log("| candidate | compile | check 1 | check 2 | check 3 | checks 4-10 |"); +console.log("|---|---:|---:|---:|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log( + `| ${name} | ${r.compileUs.toFixed(1)} µs | ${r.checkUs.map((x) => `${x.toFixed(2)} µs`).join(" | ")} |`, + ); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/size.ts b/packages/schema/test/bench/size.ts new file mode 100644 index 000000000..46a09e3d3 --- /dev/null +++ b/packages/schema/test/bench/size.ts @@ -0,0 +1,50 @@ +// Bundle size of what a validator needs from each library, as a browser app ships it: esbuild bundles and minifies the +// import for the browser platform (one file, no code splitting), then gzip -9. +// Run: node test/bench/size.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +import { build } from "esbuild"; + +const src = new URL("../../src/", import.meta.url).pathname; + +const entries: Record = { + "pi-schema": `export { compile } from ${JSON.stringify(`${src}index.ts`)};`, + "pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`, + typebox: `export { Compile } from "typebox/compile";`, + cfworker: `export { Validator } from "@cfworker/json-schema";`, + ajv: `export { default } from "ajv/dist/2020.js";`, + "ajv + ajv-formats": `export { default } from "ajv/dist/2020.js"; export { default as addFormats } from "ajv-formats";`, + "ata-validator": `export { Validator } from "ata-validator";`, + // Zod's documented `import * as z` (a named `{ z }` import keeps every locale, about 90 KB gzipped). Its core with + // native schemas, and with z.fromJSONSchema for JSON Schema. + zod: `import * as z from "zod"; export const check = (v: unknown) => z.object({ a: z.string() }).safeParse(v).success;`, + "zod + fromJSONSchema": `import * as z from "zod"; export const fromJSONSchema = z.fromJSONSchema;`, + omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`, +}; + +/** Minified and gzipped bytes by entry. */ +export type SizeResults = Record; + +const results: SizeResults = {}; +for (const [name, contents] of Object.entries(entries)) { + const output = await build({ + stdin: { contents, resolveDir: src, loader: "ts" }, + bundle: true, + minify: true, + format: "esm", + platform: "browser", + write: false, + logLevel: "silent", + }); + const code = output.outputFiles[0].contents; + results[name] = { minified: code.length, gzip: gzipSync(code, { level: 9 }).length }; +} + +console.log("| library | minified | gzip |"); +console.log("|---|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log(`| ${name} | ${(r.minified / 1024).toFixed(1)} KB | ${(r.gzip / 1024).toFixed(1)} KB |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/speed.ts b/packages/schema/test/bench/speed.ts new file mode 100644 index 000000000..ba2f17b23 --- /dev/null +++ b/packages/schema/test/bench/speed.ts @@ -0,0 +1,79 @@ +// Hot validation speed: ns per check for each case and candidate, after warm-up. Each round times every +// candidate once, in a rotated order; the result is the median of the rounds. +// Run: node test/bench/speed.ts [--json ] (add --disallow-code-generation-from-strings for no eval) + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { cases } from "./cases.ts"; + +const ROUNDS = Number(process.env.ROUNDS ?? 15); +const ROUND_MS = Number(process.env.ROUND_MS ?? 30); +let sink = 0; + +function time(check: (v: unknown) => boolean, value: unknown, n: number): number { + const start = process.hrtime.bigint(); + for (let i = 0; i < n; i++) if (check(value)) sink++; + return Number(process.hrtime.bigint() - start) / n; +} + +function median(values: number[]): number { + return [...values].sort((a, b) => a - b)[values.length >> 1]; +} + +/** + * Median ns per check, by case name, input ("valid"/"invalid") and candidate name; null where the candidate cannot + * compile the schema (Ajv without eval). + */ +export type SpeedResults = Record>>; + +const results: SpeedResults = {}; +for (const c of cases) { + results[c.name] = { valid: {}, invalid: {} }; + for (const input of ["valid", "invalid"] as const) { + const value = c[input]; + const runs: { + candidate: (typeof candidates)[number]; + check: (v: unknown) => boolean; + n: number; + samples: number[]; + }[] = []; + for (const candidate of candidates) { + let check: (v: unknown) => boolean; + try { + check = candidate.prepare(c.schema, { draft: "2020-12" }).check; + } catch (error) { + if (!(error instanceof EvalError)) throw error; + results[c.name][input][candidate.name] = null; + continue; + } + if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`); + let n = 1000; + while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2; + runs.push({ candidate, check, n, samples: [] }); + } + for (let round = 0; round < ROUNDS; round++) { + for (let i = 0; i < runs.length; i++) { + const run = runs[(i + round) % runs.length]; + run.samples.push(time(run.check, value, run.n)); + } + } + for (const run of runs) results[c.name][input][run.candidate.name] = median(run.samples); + } +} + +const names = candidates.map((c) => c.name); +console.log(`| case | input | ${names.join(" | ")} |`); +console.log(`|---|---|${names.map(() => "---:").join("|")}|`); +for (const [name, inputs] of Object.entries(results)) { + for (const [input, ns] of Object.entries(inputs)) { + console.log(`| ${name} | ${input} | ${names.map((n) => format(ns[n])).join(" | ")} |`); + } +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +if (sink < 0) console.log(sink); + +function format(ns: number | null): string { + if (ns === null) return "fails"; + return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`; +} diff --git a/packages/schema/test/bench/zod.ts b/packages/schema/test/bench/zod.ts new file mode 100644 index 000000000..c0d4d8371 --- /dev/null +++ b/packages/schema/test/bench/zod.ts @@ -0,0 +1,187 @@ +// Zod in the comparison. Zod is not a JSON Schema validator: the speed cases are written here as the native Zod +// schemas a developer would write (z.strictObject for closed objects, z.discriminatedUnion for tagged unions, z.lazy +// for recursion), and the corpus schemas are built into the same native calls by `nativeZod`. Only the conformance run +// goes through z.fromJSONSchema, Zod's (semi-experimental) JSON Schema import. + +import * as z from "zod"; +import type { JsonSchema } from "../../src/index.ts"; +import { cases } from "./cases.ts"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const block = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("text"), text: z.string() }), + z.strictObject({ + kind: z.literal("image"), + src: z.string(), + width: z.number().int().min(0), + height: z.number().int().min(0), + alt: z.string().optional(), + }), + z.strictObject({ kind: z.literal("code"), lang: z.enum(["ts", "js", "py"]), source: z.string() }), +]); + +const id = z.number().int().min(0); + +const range = z.strictObject({ start: z.number().int().min(1), end: z.number().int().min(1) }); +const replace = z.strictObject({ oldText: z.string(), newText: z.string() }); +const lineEdit = z.strictObject({ range, text: z.string() }); + +const richText = z.strictObject({ + type: z.enum(["text", "mention"]), + content: z.string().max(2000), + annotations: z.strictObject({ bold: z.boolean(), italic: z.boolean(), color: z.string() }).optional(), +}); + +type PageBlock = { + type: "paragraph" | "heading_1" | "heading_2" | "bulleted_list_item" | "to_do"; + rich_text: z.infer[]; + checked?: boolean; + children?: PageBlock[]; +}; + +const pageBlock: z.ZodType = z.lazy(() => + z.strictObject({ + type: z.enum(["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"]), + rich_text: z.array(richText), + checked: z.boolean().optional(), + children: z.array(pageBlock).optional(), + }), +); + +/** The speed cases as native Zod schemas, by case name (the JSON Schema cases in cases.ts, one for one). */ +const native: Record = { + "small closed object": z.strictObject({ text: z.string() }), + "nested object with tagged unions": z.strictObject({ + id: z.string().regex(uuid), + title: z.string().min(1).max(200), + tags: z.array(z.string()).max(16), + author: z.strictObject({ name: z.string(), email: z.string().optional() }), + blocks: z.array(block), + score: z.number().min(0).max(1).optional(), + draft: z.boolean().optional(), + }), + "message union of 5": z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("hello"), + protocol: z.number().int().min(0), + incarnation: z.string().regex(uuid), + calls: z.number().int().min(1), + }), + z.strictObject({ + kind: z.literal("hold"), + hold: id, + type: z.string(), + version: z.number().int().min(1), + fingerprint: z.string().regex(/^[0-9a-f]{16}$/), + params: z.unknown(), + address: z.strictObject({ kind: z.string(), params: z.unknown() }), + chain: z.array(z.string().regex(uuid)).max(64).optional(), + }), + z.strictObject({ kind: z.literal("release"), hold: id }), + z.strictObject({ kind: z.literal("call"), call: id, hold: id, method: id, args: z.unknown().optional() }), + z.strictObject({ kind: z.literal("abort"), call: id }), + ]), + "tool: read": z.object({ path: z.string(), offset: z.number().optional(), limit: z.number().optional() }), + "tool: edit": z.object({ path: z.string(), edits: z.array(z.object({ oldText: z.string(), newText: z.string() })) }), + "tool: bash": z.object({ command: z.string(), timeout: z.number().optional() }), + "MCP create_issue (pattern, anyOf, enum)": z.strictObject({ + owner: z + .string() + .min(1) + .regex(/^[A-Za-z0-9-]+$/), + repo: z + .string() + .min(1) + .regex(/^[A-Za-z0-9._-]+$/), + title: z.string().min(1), + body: z.string().optional(), + labels: z.array(z.string()).optional(), + assignees: z.array(z.string()).optional(), + milestone: z.number().nullable().optional(), + state: z.enum(["open", "closed"]).optional(), + }), + "MCP edit_file ($ref, anyOf)": z.strictObject({ + path: z.string().regex(/^(\/|\.\/|[A-Za-z]:\\)?[^\u0000]+$/), + edits: z.array(z.union([replace, lineEdit])).min(1), + dryRun: z.boolean().optional(), + }), + "MCP create_page (recursive $ref, oneOf)": z.strictObject({ + parent: z.union([ + z.strictObject({ page_id: z.string().regex(uuid) }), + z.strictObject({ database_id: z.string().regex(uuid) }), + ]), + title: z.string().min(1), + due: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .optional(), + children: z.array(pageBlock).max(100).optional(), + }), +}; + +/** The native Zod schema of each speed case's JSON Schema. */ +export const nativeCases: ReadonlyMap = new Map( + cases.map((c) => { + const schema = native[c.name]; + if (schema === undefined) throw new Error(`No native Zod schema for the case ${c.name}`); + return [c.schema, schema]; + }), +); + +type Json = Record; + +/** + * The native Zod schema a developer would write for a corpus schema (corpus.ts): objects (closed ones strict), + * strings with bounds, patterns and enums, bounded integers, numbers, booleans, string arrays, descriptions, and + * $defs reused through their constant. Throws for any other shape. + */ +export function nativeZod(schema: Json, root: Json = schema): z.ZodType { + if (typeof schema.$ref === "string") { + const name = schema.$ref.replace("#/$defs/", ""); + return nativeZod((root.$defs as Record)[name], root); + } + const built = nativeType(schema, root); + return typeof schema.description === "string" ? built.describe(schema.description) : built; +} + +function nativeType(schema: Json, root: Json): z.ZodType { + switch (schema.type) { + case "object": { + const required = new Set(schema.required as string[]); + const shape: Record = {}; + for (const [key, value] of Object.entries(schema.properties as Record)) { + const property = nativeZod(value, root); + shape[key] = required.has(key) ? property : property.optional(); + } + return schema.additionalProperties === false ? z.strictObject(shape) : z.object(shape); + } + case "string": { + if (Array.isArray(schema.enum)) return z.enum(schema.enum as [string, ...string[]]); + let string = z.string(); + if (typeof schema.minLength === "number") string = string.min(schema.minLength); + if (typeof schema.maxLength === "number") string = string.max(schema.maxLength); + if (typeof schema.pattern === "string") string = string.regex(new RegExp(schema.pattern)); + return string; + } + case "integer": + return z + .number() + .int() + .min(schema.minimum as number) + .max(schema.maximum as number); + case "number": + return z.number(); + case "boolean": + return z.boolean(); + case "array": + return z.array(z.string()).max(schema.maxItems as number); + default: + throw new Error(`No native Zod builder for ${JSON.stringify(schema).slice(0, 80)}`); + } +} + +/** z.fromJSONSchema with the draft given; it accepts only local $refs, so remote documents are not passed. */ +export function zodFromJsonSchema(schema: JsonSchema, draft: "2020-12" | "7"): z.ZodType { + return z.fromJSONSchema(schema as never, { defaultTarget: draft === "7" ? "draft-7" : "draft-2020-12" }); +} diff --git a/packages/schema/test/builders.test.ts b/packages/schema/test/builders.test.ts new file mode 100644 index 000000000..2d13886c7 --- /dev/null +++ b/packages/schema/test/builders.test.ts @@ -0,0 +1,104 @@ +// The builders: the JSON Schema they produce, the static types they carry, and how they compile. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + arr, + bool, + bytes, + compile, + int, + lit, + map, + nul, + nullable, + num, + obj, + oneOf, + opt, + type Static, + str, + tuple, + union, + unknown, +} from "../src/index.ts"; + +describe("builders", () => { + it("produce plain JSON Schema", () => { + const schema = obj({ + path: str({ description: "File path", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["read", "write"]), + }); + expect(JSON.parse(JSON.stringify(schema))).toEqual({ + type: "object", + properties: { + path: { type: "string", description: "File path", minLength: 1 }, + offset: { type: "integer", minimum: 1 }, + mode: { enum: ["read", "write"] }, + }, + required: ["path", "mode"], + additionalProperties: false, + }); + }); + + it("produce every leaf", () => { + expect([str(), num(), int(), bool(), nul(), unknown(), lit(3), bytes({ maxLength: 8 })]).toEqual([ + { type: "string" }, + { type: "number" }, + { type: "integer" }, + { type: "boolean" }, + { type: "null" }, + {}, + { const: 3 }, + { type: "bytes", maxLength: 8 }, + ]); + expect(arr(str(), { maxItems: 2 })).toEqual({ type: "array", items: { type: "string" }, maxItems: 2 }); + expect(map(num())).toEqual({ type: "object", additionalProperties: { type: "number" } }); + expect(tuple([str(), num()])).toEqual({ + type: "array", + prefixItems: [{ type: "string" }, { type: "number" }], + items: false, + minItems: 2, + }); + expect(nullable(str())).toEqual({ anyOf: [{ type: "string" }, { type: "null" }] }); + expect(obj({ a: str() }, { open: true })).toEqual({ + type: "object", + properties: { a: { type: "string" } }, + required: ["a"], + }); + }); + + it("carry the static type of valid values", () => { + const message = union([ + obj({ kind: lit("text"), text: str() }), + obj({ kind: lit("image"), url: str(), alt: opt(str()) }), + ]); + expectTypeOf>().toEqualTypeOf< + { kind: "text"; text: string } | { kind: "image"; url: string; alt?: string } + >(); + const pair = tuple([str(), arr(int())]); + expectTypeOf>().toEqualTypeOf<[string, number[]]>(); + expectTypeOf>>>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + }); + + it("narrow values through a compiled check", () => { + const point = obj({ x: num(), y: num(), label: opt(str()) }); + const value: unknown = { x: 1, y: 2 }; + const { check } = compile(point); + if (!check(value)) throw new Error("expected a point"); + expectTypeOf(value).toEqualTypeOf<{ x: number; y: number; label?: string }>(); + expect(value.x + value.y).toBe(3); + }); + + it("compile into checks with the declared semantics", () => { + const validator = compile(obj({ name: str({ minLength: 1 }), tags: opt(arr(str(), { uniqueItems: true })) })); + expect(validator.check({ name: "a" })).toBe(true); + expect(validator.check({ name: "a", tags: ["x", "y"] })).toBe(true); + expect(validator.errors({ name: "", tags: ["x", "x"], other: 1 })).toEqual([ + { path: "/name", keyword: "minLength", message: "expected at least 1 characters" }, + { path: "/tags", keyword: "uniqueItems", message: "expected unique items" }, + { path: "/other", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); +}); diff --git a/packages/schema/test/coerce-typebox.test.ts b/packages/schema/test/coerce-typebox.test.ts new file mode 100644 index 000000000..f9a83140d --- /dev/null +++ b/packages/schema/test/coerce-typebox.test.ts @@ -0,0 +1,1344 @@ +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax, as in the original tests. +// biome-ignore-all lint/correctness/noPrecisionLoss: the original tests' literal, kept as written. +// TypeBox 1.3.27's Value.Convert tests (test/typebox/runtime/value/convert, MIT, (c) Haydn Paterson), ported to the +// conversion pass of pi-schema's coercion. Schemas are still built with TypeBox: its schemas are JSON Schema, and the +// port reads only their JSON structure, never TypeBox's kind marks. Run with vitest. + +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { Coercer } from "../src/coerce.ts"; +import { Compiler } from "../src/compiler.ts"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +/** Pass 2 alone, as TypeBox's Value.Convert(schema, value). */ +function convert(schema: unknown, value: unknown): unknown { + return new Coercer(new Compiler(schema as object, { formats }, false), schema).convertOnly(value); +} + +const Value = { + Convert: convert, + Check: (schema: unknown, value: unknown) => compile(schema as object, { formats }).check(value), +}; +const Convert = convert; +const Assert = { + IsEqual: (actual: unknown, expected: unknown) => expect(actual).toEqual(expected), + IsTrue: (value: boolean) => expect(value).toBe(true), + IsFalse: (value: boolean) => expect(value).toBe(false), +}; + +describe("Value.Convert.Any", () => { + it("Should Convert 1", () => { + const T = Type.Any(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Any(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, undefined); + }); + it("Should Convert 3", () => { + const T = Type.Any(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 4", () => { + const T = Type.Any(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 5", () => { + const T = Type.Any(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Any(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Any(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 8", () => { + const S = Symbol("x-symbol"); + const T = Type.Symbol(); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Array", () => { + it("Should Convert 1", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [1, 3.14, 1, 3.14, 1, 0, 1, 0, "hello"]); + }); + it("Should Convert 2", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [true, 3.14, true, "3.14", true, false, true, false, "hello"]); + }); + it("Should Convert 3", () => { + const T = Type.Array(Type.String()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, ["1", "3.14", "1", "3.14", "true", "false", "true", "false", "hello"]); + }); + // ------------------------------------------------------------------ + // Added support for Array coercion + // + // https://github.com/sinclairzx81/typebox/issues/1469 + // ------------------------------------------------------------------ + it("Should Convert 4", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 5", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + it("Should Convert 6", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 7", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, true); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 8", () => { + const T = Type.Array(Type.Null()); + const R = Value.Convert(T, null); + Assert.IsEqual(R, [null]); + }); + it("Should Convert 9", () => { + const T = Type.Array(Type.Unknown()); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, [undefined]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 10", () => { + const T = Type.Array(Type.BigInt()); + const R = Value.Convert(T, 1n); + Assert.IsEqual(R, [1n]); + }); + it("Should Convert 11", () => { + const T = Type.Array(Type.Object({})); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, [{}]); + }); + it("Should Convert 12", () => { + const T = Type.Array(Type.Object({ x: Type.Number() })); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, [{ x: 1 }]); + }); + it("Should Convert 13", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 14", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 15", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, [false]); + }); + it("Should Convert 16", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, ["hello"]); + }); + it("Should Convert 17", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 18", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 19", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, { 0: 1, length: 1 }); + Assert.IsEqual(R, [{ 0: 1, length: 1 }]); + }); + it("Should Convert 20", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Union + // ------------------------------------------------------------------ + it("Should Convert 21", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, 1); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 22", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, "hello"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 23", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, [1, "hello", true, "3.14"]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 24", () => { + const T = Type.Union([Type.Array(Type.Number()), Type.String()]); + const R = Value.Convert(T, "1"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 25", () => { + const T = Type.Array(Type.Union([Type.Array(Type.Number()), Type.String()])); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Embedded Intersect + // ------------------------------------------------------------------ + it("Should Convert 26", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, { x: 1, y: 2 }); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 27", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, [ + { x: "1", y: "2" }, + { x: "3", y: "4" }, + ]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 28", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Array(Type.Number())]); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 29", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Object({ x: Type.Number() })]); + const R = Value.Convert(T, 42); + Assert.IsFalse(Value.Check(T, R)); + }); +}); + +describe("Value.Convert.Boolean", () => { + it("Should Convert 1", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("Should Convert 2", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 3", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("Should Convert 4", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("Should Convert 5", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 7", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1.1); + Assert.IsEqual(R, 1.1); + }); + it("Should Convert 8", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 9", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 10", () => { + const value = "hello"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 11", () => { + const value = "true"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 12", () => { + const value = "TRUE"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 13", () => { + const value = "false"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 14", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 15", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 16", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 17", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 18", () => { + const value = "2"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, "2"); + }); + it("Should Convert 19", () => { + const value = 0; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 20", () => { + const value = 1n; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 21", () => { + const value = 1; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 22", () => { + const value = 2; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 23", () => { + const value = 0; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 24", () => { + const value = 1; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 25", () => { + const value = 2; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 26", () => { + const value = true; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 27", () => { + const value = false; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 28", () => { + const value = {}; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 29", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 17", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, false); + }); + it("Should Convert 18", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, true); + }); + it("Should Convert 19", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(2)); + Assert.IsEqual(R, BigInt(2)); + }); + it("Should Convert 20", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 21", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + it("Should Convert 22", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, false); + }); + it("Should Convert 23", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, false); + }); +}); + +describe("Value.Convert.Enum", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Enum([1, 2, "hello"]), + }); + const R0 = Value.Convert(T, { x: "1" }); + const R1 = Value.Convert(T, { x: "2" }); + const R2 = Value.Convert(T, { x: "hello" }); + const R3 = Value.Convert(T, { x: "world" }); + Assert.IsEqual(R0, { x: 1 }); + Assert.IsEqual(R1, { x: 2 }); + Assert.IsEqual(R2, { x: "hello" }); + Assert.IsEqual(R3, { x: "world" }); + }); +}); + +describe("Value.Convert.Integer", () => { + it("Should Convert 1", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3); + }); + it("Should Convert 2", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Integer(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Integer(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Integer(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const value = "hello"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 7", () => { + const value = "3.14"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 3); + }); + it("Should Convert 8", () => { + const value = "-0"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -0); + }); + it("Should Convert 9", () => { + const value = "-100"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -100); + }); + it("Should Convert 10", () => { + const value = 42; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 42); + }); + it("Should Convert 11", () => { + const value = true; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 1); + }); + it("Should Convert 12", () => { + const value = false; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 0); + }); + it("Should Convert 13", () => { + const value = {}; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 14", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, []); + }); + // ---------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1147 + // ---------------------------------------------------------- + it("Should Convert 15", () => { + const N = 1738213389080; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, N); + }); + it("Should Convert 16", () => { + const N = 1738213389080.5555; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 17", () => { + const N = "1738213389080"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 18", () => { + const N = "1738213389080.555"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); +}); + +describe("Value.Convert.Intersect", () => { + it("Should Convert intersected objects", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + // ---------------------------------------------------------------- + // Intersection Complex + // ---------------------------------------------------------------- + it("Should complex intersect 1", () => { + const T = Type.Intersect([Type.Number(), Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + it("Should complex intersect 2", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() }), Type.Number()]); + const R = Value.Convert(T, { x: "3", y: "4" }); + Assert.IsEqual(R, { x: 3, y: 4 }); + }); + // Differs from TypeBox: allOf converts with each member in turn; TypeBox first merges an intersection into one object and drops the number. + it("Should complex intersect 3", () => { + const T = Type.Intersect([ + Type.Number(), // lost in Evaluate + Type.Object({ x: Type.Number() }), + Type.Object({ y: Type.Number() }), + ]); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); +}); + +describe("Value.Convert.Literal", () => { + it("LiteralString: Should convert 1", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, "1"); + }); + it("LiteralString: Should convert 2", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 2); + Assert.IsEqual(R, 2); + }); + it("LiteralString: Should convert 3", () => { + const T = Type.Literal("true"); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + // ------------------------------------------------------------------ + // LiteralNumber + // ------------------------------------------------------------------ + it("LiteralNumber: Should convert 1", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("LiteralNumber: Should convert 2", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.15"); + Assert.IsEqual(R, "3.15"); + }); + it("LiteralNumber: Should convert 3", () => { + const T = Type.Literal(1); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("LiteralNumber: Should convert 4", () => { + const T = Type.Literal(0); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("LiteralNumber: Should convert 5", () => { + const T = Type.Literal(2); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + // ------------------------------------------------------------------ + // LiteralBoolean + // ------------------------------------------------------------------ + it("LiteralBoolean: Should convert 1", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("LiteralBoolean: Should convert 2", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 3", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 4", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 5", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 6", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 7", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); +}); + +describe("Value.Convert.Never", () => { + it("Should not 1", () => { + const T = Type.Never(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should not 2", () => { + const T = Type.Never(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should not 3", () => { + const T = Type.Never(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, "true"); + }); +}); + +describe("Value.Convert.Null", () => { + it("Should Convert 1", () => { + const T = Type.Null(); + const R = Value.Convert(T, "null"); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Null(); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 3", () => { + const T = Type.Null(); + const R = Value.Convert(T, "nil"); + Assert.IsEqual(R, "nil"); + }); + it("Should Convert 4", () => { + const T = Type.Null(); + const R = Value.Convert(T, "undefined"); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Null(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 6", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 9", () => { + const T = Type.Null(); + const R = Value.Convert(T, ""); + Assert.IsEqual(R, null); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, null); + }); + it("Should Convert 11", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, BigInt(1)); + }); + it("Should Convert 12", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 13", () => { + const T = Type.Null(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 14", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 15", () => { + const T = Type.Null(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, 1); + }); + it("Should Convert 16", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 17", () => { + const T = Type.Null(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, null); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 18", () => { + const T = Type.Null(); + const S = Symbol("hello"); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Number", () => { + it("Should Convert 1", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 2", () => { + const T = Type.Number(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Number(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.Number(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 8", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-0"); + Assert.IsEqual(R, -0); + }); + it("Should Convert 9", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-100"); + Assert.IsEqual(R, -100); + }); + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, 0); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(10000000000000000000000000000000000000000000000000000)); + Assert.IsEqual(R, BigInt(10000000000000000000000000000000000000000000000000000)); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1); + }); + it("Should Convert 15", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 16", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 17", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 18", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123n"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 19", () => { + const T = Type.Number(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, 0); + }); + it("Should Convert 20", () => { + const T = Type.Number(); + const R = Value.Convert(T, `90071992547409910n`); // trailing 0 too large + Assert.IsEqual(R, "90071992547409910n"); + }); +}); + +describe("Value.Convert.Object", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + z: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 3", () => { + const T = Type.Object({ x: Type.Number() }); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Object( + { x: Type.Number() }, + { + additionalProperties: Type.Null(), + }, + ); + const R = Value.Convert(T, { x: "1", y: 0 }); + Assert.IsEqual(R, { x: 1, y: null }); + }); + // ------------------------------------------------------------------ + // IsOptionalUndefined + // + // https://github.com/sinclairzx81/typebox/issues/1336#issuecomment-3312808962 + // ------------------------------------------------------------------ + it("Should Convert 5", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 6", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: undefined }); + Assert.IsEqual(R, { x: undefined }); + }); +}); + +describe("Value.Convert.Record", () => { + it("Should Convert 1", () => { + const T = Type.Record(Type.String(), Type.Number()); + const R = Value.Convert(T, { x: "42", y: "24", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: 24, z: "hello" }); + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/930 + // ---------------------------------------------------------------- + it("Should Convert 2", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 3", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 6", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Record(Type.Number(), Type.Number(), { + additionalProperties: Type.Null(), + }); + const R = Value.Convert(T, { "0": "1", x: 0 }); + Assert.IsEqual(R, { "0": 1, x: null }); + }); +}); + +describe("Value.Convert.String", () => { + it("Should Convert 1", () => { + const T = Type.String(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 2", () => { + const T = Type.String(); + const R = Value.Convert(T, 3); + Assert.IsEqual(R, "3"); + }); + it("Should Convert 3", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 4", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 5", () => { + const T = Type.String(); + const R = Value.Convert(T, BigInt(12345)); + Assert.IsEqual(R, "12345"); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.String(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.String(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 8", () => { + const T = Type.String(); + const R = Value.Convert(T, 42n); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 9", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 10", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 11", () => { + const T = Type.String(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 12", () => { + const T = Type.String(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 13", () => { + const T = Type.String(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, "null"); + }); + it("Should Convert 14", () => { + const T = Type.String(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, ""); + }); +}); + +describe("Value.Convert.TemplateLiteral", () => { + // Differs from TypeBox: a template literal is a string with a pattern in JSON Schema; TypeBox converts a finite one as a union of literals. + it("Should Convert 1", () => { + const T = Type.TemplateLiteral("${1 | 2 | 3}"); + const R0 = Value.Convert(T, 1); + const R1 = Value.Convert(T, 2); + const R2 = Value.Convert(T, 3); + const R3 = Value.Convert(T, 4); + Assert.IsEqual(R0, "1"); + Assert.IsEqual(R1, "2"); + Assert.IsEqual(R2, "3"); + Assert.IsEqual(R3, "4"); + }); + // ------------------------------------------------------------------ + // https://github.com/sinclairzx81/typebox/issues/1510 + // ------------------------------------------------------------------ + it("Should Convert 2", () => { + const T = Type.Object({ + test: Type.TemplateLiteral('${number}${"a"|"b"|"c"}'), + }); + const R0 = Value.Convert(T, { test: "123b" }); + Assert.IsEqual(R0, { test: "123b" }); + }); +}); + +describe("Value.Convert.Tuple", () => { + it("Should Convert 1", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "true"]); + Assert.IsEqual(R, [1, 1]); + }); + it("Should Convert 2", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1"]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 3", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "2", "3"]); + Assert.IsEqual(R, [1, 2, "3"]); + }); + it("Should Convert 4", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, "not an array"); + Assert.IsEqual(R, "not an array"); + }); + it("Should Convert 5", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 6", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, { 0: "1", 1: "2" }); + Assert.IsEqual(R, { 0: "1", 1: "2" }); + }); +}); + +describe("Value.Convert.Union", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Union([Type.Number(), Type.Null()]), + }); + const R0 = Value.Convert(T, { x: "42" }); + const R1 = Value.Convert(T, { x: "null" }); + const R2 = Value.Convert(T, { x: "hello" }); + Assert.IsEqual(R0, { x: 42 }); + Assert.IsEqual(R1, { x: null }); + Assert.IsEqual(R2, { x: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Union([Type.Boolean(), Type.Number()]), + }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: true }); + }); + + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/787 + // ---------------------------------------------------------------- + it("Should Convert 3", () => { + const T = Type.Intersect([ + Type.Union([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Number() })]), + Type.Object({ c: Type.Number() }), + ]); + const R0 = Convert(T, { a: "1", c: "2" }); + const R1 = Convert(T, { b: "1", c: "2" }); + const R2 = Convert(T, { a: "1", b: "2", c: "3" }); + Assert.IsEqual(R0, { a: 1, c: 2 }); + Assert.IsEqual(R1, { b: 1, c: 2 }); + Assert.IsEqual(R2, { a: 1, b: "2", c: 3 }); // note: matching on first + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1295 + // ---------------------------------------------------------------- + it("Should guard against Array conversion in Object", () => { + const T = Type.Union([ + Type.Object({ + type: Type.Literal("A"), + values: Type.Union([Type.String(), Type.Number()]), + }), + Type.Object({ + type: Type.Literal("B"), + values: Type.String(), + }), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); + it("Should guard against Array conversion in Record", () => { + const T = Type.Union([ + Type.Record(Type.String({ pattern: "^values$" }), Type.Union([Type.String(), Type.Number()])), + Type.Record(Type.String({ pattern: "^type$" }), Type.Union([Type.String(), Type.Number()])), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); +}); + +describe("Value.Convert.Unknown", () => { + it("Should Convert 1", () => { + const V = null; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 2", () => { + const V = undefined; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 3", () => { + const V = "hello"; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 4", () => { + const V = 42; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 5", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); +}); diff --git a/packages/schema/test/coerce.test.ts b/packages/schema/test/coerce.test.ts new file mode 100644 index 000000000..4444f0cc9 --- /dev/null +++ b/packages/schema/test/coerce.test.ts @@ -0,0 +1,458 @@ +// compile(schema, { coerce: true }): coerce() and parse(), the three coercion passes (optional nulls, TypeBox's +// conversion rules, primitive coercion by type), their order where they conflict, and the guarantees around them. +// Ports pi-ai's tool argument coercion tests. + +import Type from "typebox"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { + arr, + compile, + int, + map, + nullable, + num, + obj, + opt, + str, + tuple, + type ValidationError, + type Validator, +} from "../src/index.ts"; + +function coerce(schema: object, value: unknown): unknown { + return compile(schema, { formats, coerce: true }).coerce(value); +} + +/** The coerced `value` property of `{ value }` under an object schema with one required property `value`. */ +function coerceValue(schema: unknown, value: unknown): unknown { + const root = { type: "object", properties: { value: schema }, required: ["value"] }; + return (coerce(root, { value }) as { value: unknown }).value; +} + +describe("compile options", () => { + it("adds coerce and parse only with coerce: true", () => { + const plain = compile(obj({ n: num() })); + expectTypeOf(plain).toEqualTypeOf>(); + expect("coerce" in plain).toBe(false); + expect("parse" in plain).toBe(false); + const coercing = compile(obj({ n: num() }), { coerce: true }); + expectTypeOf(coercing.parse).returns.toEqualTypeOf< + | { readonly ok: true; readonly value: { n: number } } + | { readonly ok: false; readonly errors: ValidationError[] } + >(); + expect(coercing.check({ n: 1 })).toBe(true); + }); +}); + +describe("coerce", () => { + it("never modifies the value", () => { + const value = { count: "42", tags: "a" }; + const schema = obj({ count: num(), tags: arr(str()) }); + expect(coerce(schema, value)).toEqual({ count: 42, tags: ["a"] }); + expect(value).toEqual({ count: "42", tags: "a" }); + }); + + it("returns the value itself when nothing changes", () => { + const value = { a: { b: [1, 2] }, c: "x" }; + const schema = obj({ a: obj({ b: arr(num()) }), c: str() }); + expect(coerce(schema, value)).toBe(value); + }); + + it("copies only the changed paths", () => { + const unchanged = { b: [1, 2] }; + const value = { a: unchanged, c: { d: "1" } }; + const schema = obj({ a: obj({ b: arr(num()) }), c: obj({ d: num() }) }); + const coerced = coerce(schema, value) as typeof value; + expect(coerced).toEqual({ a: { b: [1, 2] }, c: { d: 1 } }); + expect(coerced).not.toBe(value); + expect(coerced.a).toBe(unchanged); + expect(value.c).toEqual({ d: "1" }); + }); + + it("works on frozen schemas", () => { + const schema = Object.freeze({ + type: "object", + properties: Object.freeze({ n: Object.freeze({ type: "integer" }) }), + }); + expect(coerce(schema, { n: "7" })).toEqual({ n: 7 }); + }); + + it("still coerces when the Function constructor is unavailable", () => { + const original = globalThis.Function; + globalThis.Function = (() => { + throw new EvalError("Code generation from strings disallowed for this context"); + }) as unknown as FunctionConstructor; + try { + expect(coerce(obj({ count: num() }), { count: "42" })).toEqual({ count: 42 }); + } finally { + globalThis.Function = original; + } + }); + + it("leaves values it cannot coerce for the check to report", () => { + const validator = compile(obj({ count: num() }), { coerce: true }); + const coerced = validator.coerce({ count: "many" }); + expect(coerced).toEqual({ count: "many" }); + expect(validator.check(coerced)).toBe(false); + }); +}); + +describe("parse", () => { + it("returns the coerced value when it is valid", () => { + const validator = compile(obj({ count: num(), tags: arr(str()) }), { coerce: true }); + const result = validator.parse({ count: "42", tags: "a" }); + expect(result).toEqual({ ok: true, value: { count: 42, tags: ["a"] } }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ count: number; tags: string[] }>(); + }); + + it("returns the errors of the coerced value when it is invalid", () => { + const validator = compile(obj({ count: num(), name: str() }), { coerce: true }); + expect(validator.parse({ count: "many", name: 7 })).toEqual({ + ok: false, + errors: [{ path: "/count", keyword: "type", message: "expected number, got string" }], + }); + }); +}); + +describe("pass 1: optional nulls", () => { + it("drops null in optional non-nullable properties, also nested", () => { + const schema = obj( + { + path: str(), + offset: opt(num()), + nullable: opt(nullable(str())), + metadata: obj({ enabled: opt({ type: "boolean" }) }), + }, + { open: true }, + ); + expect(coerce(schema, { path: "file.txt", offset: null, nullable: null, metadata: { enabled: null } })).toEqual({ + path: "file.txt", + nullable: null, + metadata: {}, + }); + }); + + it("keeps a null whose property is a $ref (the reference may allow it)", () => { + const schema = { + type: "object", + properties: { value: { $ref: "#/$defs/value" } }, + $defs: { value: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }; + expect(coerce(schema, { value: null })).toEqual({ value: null }); + }); + + it("walks array items", () => { + const schema = { type: "array", items: obj({ a: str(), b: opt(int()) }) }; + expect(coerce(schema, [{ a: "x", b: null }])).toEqual([{ a: "x" }]); + }); +}); + +describe("pass 2: TypeBox's conversion rules on JSON Schema", () => { + it("converts primitives by type", () => { + expect(coerceValue({ type: "number" }, "42")).toBe(42); + expect(coerceValue({ type: "number" }, "")).toBe(0); + expect(coerceValue({ type: "number" }, "true")).toBe(1); + expect(coerceValue({ type: "integer" }, "3.5")).toBe(3); + expect(coerceValue({ type: "integer" }, 1.5)).toBe(1); + expect(coerceValue({ type: "boolean" }, "1")).toBe(true); + expect(coerceValue({ type: "boolean" }, "FALSE")).toBe(false); + expect(coerceValue({ type: "null" }, "null")).toBe(null); + expect(coerceValue({ type: "string" }, 42)).toBe("42"); + }); + + it("wraps a single value into an array and converts the items", () => { + expect(coerceValue({ type: "array", items: { type: "number" } }, "1")).toEqual([1]); + expect(coerceValue({ type: "array", items: { type: "string" } }, null)).toEqual(["null"]); + }); + + it("converts tuple items in place and never wraps for a tuple", () => { + const tuple = { type: "array", prefixItems: [{ type: "number" }, { type: "boolean" }], items: false }; + expect(coerceValue(tuple, ["1", "true"])).toEqual([1, true]); + expect(coerceValue(tuple, "1")).toBe("1"); + }); + + it("converts const and enum values only when the result is a member", () => { + expect(coerceValue({ const: 1 }, "1")).toBe(1); + expect(coerceValue({ const: 1 }, "2")).toBe("2"); + expect(coerceValue({ enum: [1, 2, "hello"] }, "2")).toBe(2); + expect(coerceValue({ enum: [1, 2, "hello"] }, "world")).toBe("world"); + }); + + it("keeps a value one union branch accepts, else takes the first branch whose conversion the union accepts", () => { + const union = { anyOf: [{ type: "number" }, { type: "null" }] }; + expect(coerceValue(union, null)).toBe(null); + expect(coerceValue(union, "42")).toBe(42); + expect(coerceValue(union, "null")).toBe(null); + expect(coerceValue({ oneOf: [{ type: "boolean" }, { type: "number" }] }, "1")).toBe(true); + }); + + it("converts allOf with each member in turn", () => { + const schema = { allOf: [obj({ a: num() }, { open: true }), obj({ b: { type: "boolean" } }, { open: true })] }; + expect(coerce(schema, { a: "1", b: "true" })).toEqual({ a: 1, b: true }); + }); + + it("follows $ref", () => { + const schema = { + type: "object", + properties: { item: { $ref: "#/$defs/item" } }, + $defs: { item: { type: "object", properties: { id: { type: "integer" } } } }, + }; + expect(coerce(schema, { item: { id: "7" } })).toEqual({ item: { id: 7 } }); + }); + + it("converts patternProperties and additionalProperties", () => { + const schema = { + type: "object", + properties: { a: { type: "number" } }, + patternProperties: { "^n_": { type: "number" } }, + additionalProperties: { type: "boolean" }, + }; + expect(coerce(schema, { a: "1", n_x: "2", other: "true" })).toEqual({ a: 1, n_x: 2, other: true }); + }); +}); + +describe("pass 3: primitive coercion by type", () => { + it("coerces like Ajv's coerceTypes", () => { + const cases: Array<[unknown, unknown, unknown]> = [ + [{ type: "number" }, true, 1], + [{ type: "number" }, null, 0], + [{ type: "integer" }, "42", 42], + [{ type: "boolean" }, "true", true], + [{ type: "boolean" }, 0, false], + [{ type: "string" }, true, "true"], + [{ type: "null" }, "", null], + [{ type: "null" }, 0, null], + [{ type: "null" }, false, null], + ]; + for (const [schema, input, expected] of cases) expect(coerceValue(schema, input)).toEqual(expected); + }); + + it("leaves a value that matches one of a type array's types, else takes the first coercible type", () => { + expect(coerceValue({ type: ["number", "string"] }, "1")).toBe("1"); + expect(coerceValue({ type: ["boolean", "number"] }, "1")).toBe(1); + expect(coerceValue({ type: ["integer", "null"] }, "5")).toBe(5); + }); + + it("accepts null for nullable array schemas with items", () => { + expect(coerceValue({ type: ["array", "null"], items: { type: "string" } }, null)).toBe(null); + }); + + it("coerces nullable unions when the value matches no branch", () => { + expect(coerceValue({ anyOf: [{ type: "number" }, { type: "null" }] }, "42")).toBe(42); + expect(coerceValue({ oneOf: [{ type: "number" }, { type: "null" }] }, null)).toBe(null); + }); + + it("leaves values no rule coerces", () => { + const validator = compile( + { type: "object", properties: { value: { type: "integer" } }, required: ["value"] }, + { coerce: true }, + ); + for (const input of ["abc", "4x", {}, ["1", "2"]]) { + const coerced = validator.coerce({ value: input }); + expect(validator.check(coerced)).toBe(false); + } + }); +}); + +describe("where the passes conflict, the conversion rules win", () => { + it("turns null into 'null' for a required string (pass 3 alone would give '')", () => { + expect(coerceValue({ type: "string" }, null)).toBe("null"); + }); + + it("turns '' into 0 for a number (pass 3 leaves an empty string)", () => { + expect(coerceValue({ type: "number" }, "")).toBe(0); + }); + + it("turns '1' and '0' into booleans (pass 3 takes only 'true', 'false', 1 and 0)", () => { + expect(coerceValue({ type: "boolean" }, "0")).toBe(false); + }); + + it("truncates '42.1' for an integer (pass 3 rejects it)", () => { + expect(coerceValue({ type: "integer" }, "42.1")).toBe(42); + }); + + it("drops an optional null before converting it (pass 2 alone would give 'null')", () => { + expect(coerce(obj({ name: opt(str()) }), { name: null })).toEqual({}); + }); +}); + +describe("recursive schemas", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, + }; + + it("convert through a recursive union by its first accepting branch", () => { + const validator = compile(tree, { coerce: true }); + expect(validator.parse({ t: ["a", ["b"]] })).toEqual({ ok: true, value: { t: ["a", ["b"]] } }); + expect(validator.parse({ t: 5 })).toEqual({ ok: true, value: { t: "5" } }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(compile(json, { coerce: true }).parse({ data: { a: [1, null] } })).toEqual({ + ok: true, + value: { data: { a: [1, "null"] } }, + }); + }); + + it("leave a value a recursive schema cannot convert as it is, for the check to report", () => { + const validator = compile(tree, { coerce: true }); + const result = validator.parse({ t: {} }); + expect(result.ok).toBe(false); + expect(validator.coerce({ t: { a: 1 } })).toEqual({ t: { a: 1 } }); + }); + + it("convert a scalar-or-array schema that refers to itself", () => { + const schema = { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }; + const validator = compile(schema, { coerce: true }); + expect(validator.parse({ value: "42" })).toEqual({ ok: true, value: { value: 42 } }); + expect(validator.parse({ value: ["1", ["2"]] })).toEqual({ ok: true, value: { value: [1, [2]] } }); + expect(validator.parse({ value: {} }).ok).toBe(false); + }); + + it("stop at a schema that cycles through shared JS objects instead of $ref", () => { + const node: { anyOf: unknown[] } = { anyOf: [{ type: "number" }] }; + node.anyOf.push({ type: "array", items: node }); + const validator = compile(node, { coerce: true }); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + expect(validator.parse(["1", ["2"]])).toEqual({ ok: true, value: [1, [2]] }); + expect(validator.parse({}).ok).toBe(false); + }); + + it("reset the re-entry state between calls and leave the input unchanged", () => { + const validator = compile( + { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#" } }] }, + { coerce: true }, + ); + for (let i = 0; i < 20; i++) { + expect(validator.parse({}).ok).toBe(false); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + const input = [1, "2", ["3"]]; + expect(validator.parse(input)).toEqual({ ok: true, value: [1, 2, [3]] }); + expect(input).toEqual([1, "2", ["3"]]); + } + }); + + it("convert and explain a deep value that every branch of a recursive union rejects, in linear time", () => { + // Each object branch converts and explains every child: without memoizing per call this is 2^depth work. + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const validator = compile( + { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }, + { coerce: true }, + ); + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + const result = validator.parse({ s: value }); + expect(result.ok).toBe(false); + const missing = Array.from({ length: 41 }, (_, i) => ({ + path: `/s${"/c".repeat(i)}/a`, + keyword: "required", + message: "missing", + })); + const leaf = { path: `/s${"/c".repeat(40)}/x`, keyword: "type", message: "expected object, got integer" }; + expect(result).toEqual({ ok: false, errors: [...missing, leaf] }); + }); + + it("convert a TypeBox Cyclic union", () => { + const Tree = Type.Cyclic({ Tree: Type.Union([Type.String(), Type.Array(Type.Ref("Tree"))]) }, "Tree"); + const validator = compile(Type.Object({ t: Tree }), { coerce: true }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(validator.parse({ t: { a: 1 } }).ok).toBe(false); + }); +}); + +describe("member keywords across the passes", () => { + it("keep a pattern-matched property from additionalProperties in every pass", () => { + const record = Type.Record(Type.String(), Type.Number(), { additionalProperties: Type.String() }); + expect(compile(record, { coerce: true }).parse({ n1: "1" })).toEqual({ ok: true, value: { n1: 1 } }); + const plain = { + type: "object", + patternProperties: { "^n_": { type: ["integer", "null"] } }, + additionalProperties: { type: "string" }, + }; + expect(coerce(plain, { n_a: "5", other: 7 })).toEqual({ n_a: 5, other: "7" }); + }); + + it("drop optional nulls inside declared, pattern and additional property values alike", () => { + const item = obj({ n: opt(int({ minimum: 1 })) }); + const schemas = [ + obj({ entry: item }), + map(item), + { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + ]; + for (const schema of schemas) { + const input = { entry: { n: null } }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { entry: {} } }); + expect(input).toEqual({ entry: { n: null } }); + } + }); + + it("keep a null under pattern and additional properties: their keys are not optional properties", () => { + const schema = { type: "object", additionalProperties: { type: ["integer", "null"] } }; + expect(compile(schema, { coerce: true }).parse({ n: null })).toEqual({ ok: true, value: { n: null } }); + }); + + it("drop optional nulls in tuple items as in array items", () => { + const item = obj({ n: opt(int()) }); + expect(compile(arr(item), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + expect(compile(tuple([item]), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + }); + + it("coerce tuple items by type in pass 3", () => { + const schema = { type: "array", prefixItems: [{ type: ["integer", "null"] }, { type: ["boolean", "null"] }] }; + expect(coerce(schema, ["5", "true"])).toEqual([5, true]); + }); + + it("convert the properties a union node declares besides its branches", () => { + const schema = { + type: "object", + properties: { a: { type: "integer" }, b: { type: "integer" } }, + anyOf: [{ required: ["a"] }, { required: ["b"] }], + }; + expect(compile(schema, { coerce: true }).parse({ a: "3.5" })).toEqual({ ok: true, value: { a: 3 } }); + }); + + for (const keyword of ["anyOf", "oneOf"]) { + it(`check ${keyword} candidates after the node's own properties convert`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + const input = { a: "3", c: "2" }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { a: [3], c: 2 } }); + expect(input).toEqual({ a: "3", c: "2" }); + }); + } +}); diff --git a/packages/schema/test/conformance.test.ts b/packages/schema/test/conformance.test.ts new file mode 100644 index 000000000..5702220c5 --- /dev/null +++ b/packages/schema/test/conformance.test.ts @@ -0,0 +1,51 @@ +// The official JSON-Schema-Test-Suite, drafts 2020-12 and 7: every test, including optional and format tests. +// A test listed in KNOWN_FAILURES must fail; when one starts passing, remove it from the list. + +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; +import { suiteFiles, suiteSchemas } from "./suite.ts"; + +const KNOWN_FAILURES = new Set([ + // Vocabularies switched off by a custom meta-schema are not supported. + "draft2020-12/vocabulary > schema that uses custom metaschema with with no validation vocabulary > no validation: invalid number, but it still validates", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: false > format-assertion: false: invalid string", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: true > format-assertion: true: invalid string", + // A $ref to a document of another draft is evaluated with the compile's draft. + "draft2020-12/optional/cross-draft > refs to historic drafts are processed as historic drafts > first item not a string is valid", + "draft7/optional/cross-draft > refs to future drafts are processed as future drafts > missing bar is invalid", + // contentMediaType and contentEncoding are annotations. + "draft7/optional/content > validation of string-encoded content based on media type > an invalid JSON document", + "draft7/optional/content > validation of binary string-encoding > an invalid base64 string (% is not a valid character)", + "draft7/optional/content > validation of binary-encoded media type documents > a validly-encoded invalid JSON document", + "draft7/optional/content > validation of binary-encoded media type documents > an invalid base64 string that is valid JSON", +]); + +const schemas = suiteSchemas(); + +for (const file of suiteFiles()) { + // Format tests assert formats; everywhere else `format` is an annotation, as draft 2020-12 specifies. + const options = { + draft: file.draft, + schemas, + formats: file.name.includes("/optional/format/") ? formats : undefined, + }; + describe(file.name, () => { + for (const group of file.groups) { + describe(group.description, () => { + for (const test of group.tests) { + const known = KNOWN_FAILURES.has(`${file.name} > ${group.description} > ${test.description}`); + (known ? it.fails : it)(test.description, () => { + const validator = compile(group.schema, options); + expect(validator.check(test.data)).toBe(test.valid); + }); + it(`${test.description} (errors agree with check)`, () => { + const validator = compile(group.schema, options); + const errors = validator.errors(test.data); + expect(errors.length === 0).toBe(validator.check(test.data)); + }); + } + }); + } + }); +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/LICENSE b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE new file mode 100644 index 000000000..c28adbadd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/schema/test/fixtures/json-schema-test-suite/README.md b/packages/schema/test/fixtures/json-schema-test-suite/README.md new file mode 100644 index 000000000..324ff0a0c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/README.md @@ -0,0 +1,9 @@ +# JSON-Schema-Test-Suite (subset) + +From https://github.com/json-schema-org/JSON-Schema-Test-Suite at commit +`3f05891cac5d51ffa068d7076d1372130e00c829`, unmodified, under its MIT license (`LICENSE`). + +Only what the drafts this package supports use: + +- `tests/draft2020-12/` and `tests/draft7/`, complete (core, `optional/`, `optional/format/`); +- `remotes/`: the documents those tests reference (served as `http://localhost:1234/...` in the tests). diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json new file mode 100644 index 000000000..0d691d964 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/dependentRequired.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "dependentRequired": { + "foo": ["bar"] + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json new file mode 100644 index 000000000..b5ef3928c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/ignore-prefixItems.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json new file mode 100644 index 000000000..07cce1dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-dynamicref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$dynamicRef": "#detached" + }, + "detached": { + "$dynamicAnchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json new file mode 100644 index 000000000..9c2dca93c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-ref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$anchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json new file mode 100644 index 000000000..5d2644d3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "http://localhost:1234/draft2020-12/real-id-ref-string.json", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json new file mode 100644 index 000000000..65bc0c217 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json @@ -0,0 +1,21 @@ +{ + "description": "extendible array", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/extendible-dynamic-ref.json", + "type": "object", + "properties": { + "elements": { + "type": "array", + "items": { + "$dynamicRef": "#elements" + } + } + }, + "required": ["elements"], + "additionalProperties": false, + "$defs": { + "elements": { + "$dynamicAnchor": "elements" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json new file mode 100644 index 000000000..43a711c9d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json new file mode 100644 index 000000000..39c6b0abf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json new file mode 100644 index 000000000..6565a1ee0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$anchor": "foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json new file mode 100644 index 000000000..71be8b5da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/core": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/applicator" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json new file mode 100644 index 000000000..a6963e548 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/core": true, + "http://localhost:1234/draft/2020-12/vocab/custom": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/validation" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json new file mode 100644 index 000000000..67bc33c51 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json new file mode 100644 index 000000000..3d51ded5c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json @@ -0,0 +1,9 @@ +{ + "$defs": { + "bar": { + "$id": "http://localhost:1234/draft2020-12/the-nested-id.json", + "type": "string" + } + }, + "$ref": "http://localhost:1234/draft2020-12/the-nested-id.json" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json new file mode 100644 index 000000000..29661ff9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json new file mode 100644 index 000000000..6607ac534 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json new file mode 100644 index 000000000..acd8293c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2020-12/prefixItems.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json new file mode 100644 index 000000000..16d30fa3a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/ref-and-defs.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json new file mode 100644 index 000000000..1bb4846d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/$defs/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json new file mode 100644 index 000000000..b07555fb3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json @@ -0,0 +1,17 @@ +{ + "description": "tree schema, extensible", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree.json", + "$dynamicAnchor": "node", + + "type": "object", + "properties": { + "data": true, + "children": { + "type": "array", + "items": { + "$dynamicRef": "#node" + } + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json new file mode 100644 index 000000000..796f37021 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "urn:uuid:feebdaed-ffff-0000-2020-1200deadbeef", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json new file mode 100644 index 000000000..27f2ec80a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft7/detached-ref.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$id": "#detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json new file mode 100644 index 000000000..13fd0df68 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft7/ignore-dependentRequired.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "dependentRequired": { + "foo": ["bar"] + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json new file mode 100644 index 000000000..e72815cd5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json @@ -0,0 +1,11 @@ +{ + "definitions": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$id": "#foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json new file mode 100644 index 000000000..fceacb809 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json @@ -0,0 +1,15 @@ +{ + "definitions": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json new file mode 100644 index 000000000..d5929380c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json @@ -0,0 +1,11 @@ +{ + "$id": "http://localhost:1234/draft7/ref-and-definitions.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json new file mode 100644 index 000000000..6e9b3de35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/definitions/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json new file mode 100644 index 000000000..9cd2527ed --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json @@ -0,0 +1,6 @@ +{ + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json new file mode 100644 index 000000000..c2d48c068 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json @@ -0,0 +1,3 @@ +{ + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json new file mode 100644 index 000000000..9618575e2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json @@ -0,0 +1,219 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema. Boolean \"false\" forbids everything." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "specification": [ { "core":"10.3.2.3"} ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": "additionalProperties can exist by itself", + "specification": [ { "core":"10.3.2.3", "quote": "With no other applicator applying to object instances. This validates all the instance values irrespective of their property names" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "specification": [ { "core":"10.3.2.3", "quote": "Omitting this keyword has the same assertion behavior as an empty schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}} + }, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "specification":[ { "core": "10.2", "quote": "Subschemas of applicator keywords evaluate the instance completely independently such that the results of one such subschema MUST NOT impact the results of sibling subschemas." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "specification": [ { "core":"10.3.2.3" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "additionalProperties with propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { + "maxLength": 5 + }, + "additionalProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "Valid against both keywords", + "data": { "apple": 4 }, + "valid": true + }, + { + "description": "Valid against propertyNames, but not additionalProperties", + "data": { "fig": 2, "pear": "available" }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar": {} + } + } + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "additionalProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar even when foo2 is present", + "data": {"foo2": "", "bar": ""}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json new file mode 100644 index 000000000..9e87903fe --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json @@ -0,0 +1,312 @@ +[ + { + "description": "allOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json new file mode 100644 index 000000000..99143fa11 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json @@ -0,0 +1,120 @@ +[ + { + "description": "Location-independent identifier", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#foo", + "$defs": { + "A": { + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with absolute URI", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/bar#foo", + "$defs": { + "A": { + "$id": "http://localhost:1234/draft2020-12/bar", + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/root", + "$ref": "http://localhost:1234/draft2020-12/nested.json#foo", + "$defs": { + "A": { + "$id": "nested.json", + "$defs": { + "B": { + "$anchor": "foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "same $anchor with different base uri", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/foobar", + "$defs": { + "A": { + "$id": "child1", + "allOf": [ + { + "$id": "child2", + "$anchor": "my_anchor", + "type": "number" + }, + { + "$anchor": "my_anchor", + "type": "string" + } + ] + } + }, + "$ref": "child1#my_anchor" + }, + "tests": [ + { + "description": "$ref resolves to /$defs/A/allOf/1", + "data": "a", + "valid": true + }, + { + "description": "$ref does not resolve to /$defs/A/allOf/0", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json new file mode 100644 index 000000000..89b192dbd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json @@ -0,0 +1,203 @@ +[ + { + "description": "anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json new file mode 100644 index 000000000..72ea3f042 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json @@ -0,0 +1,431 @@ +[ + { + "description": "const validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 2 + }, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"foo": "bar", "baz": "bax"} + }, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [{ "foo": "bar" }] + }, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": null + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": false + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": true + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [false] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [true] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": false} + }, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": true} + }, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 0 + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 1 + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": -2.0 + }, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 9007199254740992 + }, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "hello\u0000there" + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json new file mode 100644 index 000000000..08a00a753 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json @@ -0,0 +1,176 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": true + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json new file mode 100644 index 000000000..698f78057 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json @@ -0,0 +1,131 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document; validates true", + "data": "{:}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character); validates true", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents with schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64", + "contentSchema": { "type": "object", "required": ["foo"], "properties": { "foo": { "type": "string" } } } + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "another valid base64-encoded JSON document", + "data": "eyJib28iOiAyMCwgImZvbyI6ICJiYXoifQ==", + "valid": true + }, + { + "description": "an invalid base64-encoded JSON document; validates true", + "data": "eyJib28iOiAyMH0=", + "valid": true + }, + { + "description": "an empty object as a base64-encoded JSON document; validates true", + "data": "e30=", + "valid": true + }, + { + "description": "an empty array as a base64-encoded JSON document", + "data": "W10=", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json new file mode 100644 index 000000000..ceb3ae271 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json @@ -0,0 +1,82 @@ +[ + { + "description": "invalid type for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json new file mode 100644 index 000000000..da2a503bf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json @@ -0,0 +1,21 @@ +[ + { + "description": "validate definition against metaschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "valid definition schema", + "data": {"$defs": {"foo": {"type": "integer"}}}, + "valid": true + }, + { + "description": "invalid definition schema", + "data": {"$defs": {"foo": {"type": 1}}}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json new file mode 100644 index 000000000..2baa38e9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json @@ -0,0 +1,152 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json new file mode 100644 index 000000000..1c5f0574a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json @@ -0,0 +1,171 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json new file mode 100644 index 000000000..eba4b441b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json @@ -0,0 +1,874 @@ +[ + { + "description": "A $dynamicRef to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef to an $anchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-anchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $ref to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/ref-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$ref": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef resolves to the first $dynamicAnchor still in scope that is encountered when the schema is evaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/typical-dynamic-resolution/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef without anchor in fragment behaves identical to $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-without-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#/$defs/items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items", + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is invalid", + "data": ["foo", "bar"], + "valid": false + }, + { + "description": "An array of numbers is valid", + "data": [24, 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with intermediate scopes that don't include a matching $dynamicAnchor does not affect dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-with-intermediate-scopes/root", + "$ref": "intermediate-scope", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "intermediate-scope": { + "$id": "intermediate-scope", + "$ref": "list" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "An $anchor with the same name as a $dynamicAnchor is not used for dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-ignores-anchors/root", + "$ref": "list", + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef without a matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-without-bookend/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with a non-matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/unmatched-dynamic-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items", + "$dynamicAnchor": "foo" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema with a matching $dynamicAnchor resolves to the first $dynamicAnchor in the dynamic scope", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part is valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "pass" } + } + }, + "valid": true + }, + { + "description": "The recursive part is not valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": false + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema without a matching $dynamicAnchor behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference-without-bookend/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$anchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part doesn't need to validate against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": true + } + ] + }, + { + "description": "multiple dynamic paths to the $dynamicRef keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-with-multiple-paths/main", + "if": { + "properties": { + "kindOfList": { "const": "numbers" } + }, + "required": ["kindOfList"] + }, + "then": { "$ref": "numberList" }, + "else": { "$ref": "stringList" }, + + "$defs": { + "genericList": { + "$id": "genericList", + "properties": { + "list": { + "items": { "$dynamicRef": "#itemType" } + } + }, + "$defs": { + "defaultItemType": { + "$comment": "Only needed to satisfy bookending requirement", + "$dynamicAnchor": "itemType" + } + } + }, + "numberList": { + "$id": "numberList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "number" + } + }, + "$ref": "genericList" + }, + "stringList": { + "$id": "stringList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "string" + } + }, + "$ref": "genericList" + } + } + }, + "tests": [ + { + "description": "number list with number values", + "data": { + "kindOfList": "numbers", + "list": [1.1] + }, + "valid": true + }, + { + "description": "number list with string values", + "data": { + "kindOfList": "numbers", + "list": ["foo"] + }, + "valid": false + }, + { + "description": "string list with number values", + "data": { + "kindOfList": "strings", + "list": [1.1] + }, + "valid": false + }, + { + "description": "string list with string values", + "data": { + "kindOfList": "strings", + "list": ["foo"] + }, + "valid": true + } + ] + }, + { + "description": "after leaving a dynamic scope, it is not used by a $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-leaving-dynamic-scope/main", + "if": { + "$id": "first_scope", + "$defs": { + "thingy": { + "$comment": "this is first_scope#thingy", + "$dynamicAnchor": "thingy", + "type": "number" + } + } + }, + "then": { + "$id": "second_scope", + "$ref": "start", + "$defs": { + "thingy": { + "$comment": "this is second_scope#thingy, the final destination of the $dynamicRef", + "$dynamicAnchor": "thingy", + "type": "null" + } + } + }, + "$defs": { + "start": { + "$comment": "this is the landing spot from $ref", + "$id": "start", + "$dynamicRef": "inner_scope#thingy" + }, + "thingy": { + "$comment": "this is the first stop for the $dynamicRef", + "$id": "inner_scope", + "$dynamicAnchor": "thingy", + "type": "string" + } + } + }, + "tests": [ + { + "description": "string matches /$defs/thingy, but the $dynamicRef does not stop here", + "data": "a string", + "valid": false + }, + { + "description": "first_scope is not in dynamic scope for the $dynamicRef", + "data": 42, + "valid": false + }, + { + "description": "/then/$defs/thingy is the final stop for the $dynamicRef", + "data": null, + "valid": true + } + ] + }, + { + "description": "strict-tree schema, guards against misspelled properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-tree.json", + "$dynamicAnchor": "node", + + "$ref": "tree.json", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "instance with misspelled field", + "data": { + "children": [{ + "daat": 1 + }] + }, + "valid": false + }, + { + "description": "instance with correct field", + "data": { + "children": [{ + "data": 1 + }] + }, + "valid": true + } + ] + }, + { + "description": "tests for implementation dynamic anchor and reference link", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible.json", + "$ref": "extendible-dynamic-ref.json", + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $defs first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-defs-first.json", + "allOf": [ + { + "$ref": "extendible-dynamic-ref.json" + }, + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $ref first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-ref-first.json", + "allOf": [ + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + { + "$ref": "extendible-dynamic-ref.json" + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref to $dynamicRef finds detached $dynamicAnchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-dynamicref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$dynamicRef points to a boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "true": true, + "false": false + }, + "properties": { + "true": { + "$dynamicRef": "#/$defs/true" + }, + "false": { + "$dynamicRef": "#/$defs/false" + } + } + }, + "tests": [ + { + "description": "follow $dynamicRef to a true schema", + "data": { "true": 1 }, + "valid": true + }, + { + "description": "follow $dynamicRef to a false schema", + "data": { "false": 1 }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef skips over intermediate resources - direct reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef avoids the root of each schema, but scopes are still registered", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-avoids-root-of-each-schema/base", + "$ref": "first#/$defs/stuff", + "$defs": { + "first": { + "$id": "first", + "$defs": { + "stuff": { + "$ref": "second#/$defs/stuff" + }, + "length": { + "$comment": "unused, because there is no $dynamicAnchor here", + "maxLength": 1 + } + } + }, + "second": { + "$id": "second", + "$defs": { + "stuff": { + "$ref": "third#/$defs/stuff" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 2 + } + } + }, + "third": { + "$id": "third", + "$defs": { + "stuff": { + "$dynamicRef": "#length" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 3 + } + } + } + } + }, + "tests": [ + { + "description": "data is sufficient for schema at second#/$defs/length", + "data": "hi", + "valid": true + }, + { + "description": "data is not sufficient for schema at second#/$defs/length", + "data": "hey", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json new file mode 100644 index 000000000..dbc5e0d31 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json @@ -0,0 +1,419 @@ +[ + { + "description": "simple enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1, 2, 3] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, "foo", [], true, {"foo": 12}] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, null] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [false] + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[false]] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [true] + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[true]] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [0] + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[0]] + }, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1] + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[1]] + }, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [ "hello\u0000there" ] + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "empty enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [] + }, + "tests": [ + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "number is invalid", + "data": 42, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": [], + "valid": false + }, + { + "description": "boolean is invalid", + "data": false, + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json new file mode 100644 index 000000000..05db23351 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json new file mode 100644 index 000000000..00af9d7ff --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json new file mode 100644 index 000000000..01adcbda3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json @@ -0,0 +1,838 @@ +[ + { + "description": "email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid regex string is only an annotation by default", + "data": "^(abc]", + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv4 string is only an annotation by default", + "data": "127.0.0.0.1", + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv6 string is only an annotation by default", + "data": "12345::", + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-hostname string is only an annotation by default", + "data": "〮실례.테스트", + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid hostname string is only an annotation by default", + "data": "-a-host-name-that-starts-with--", + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date string is only an annotation by default", + "data": "06/19/1963", + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date-time string is only an annotation by default", + "data": "1990-02-31T15:59:60.123-08:00", + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid time string is only an annotation by default", + "data": "08:30:06 PST", + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid json-pointer string is only an annotation by default", + "data": "/foo/bar~", + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid relative-json-pointer string is only an annotation by default", + "data": "/foo/bar", + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri string is only an annotation by default", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\filëßåré", + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri string is only an annotation by default", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\fileshare", + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-template string is only an annotation by default", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": true + } + ] + }, + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uuid string is only an annotation by default", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": true + } + ] + }, + { + "description": "duration format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid duration string is only an annotation by default", + "data": "PT1D", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json new file mode 100644 index 000000000..9abf26f71 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json @@ -0,0 +1,308 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json new file mode 100644 index 000000000..46f157a35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json @@ -0,0 +1,37 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/$defs/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/$defs/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json new file mode 100644 index 000000000..6a3e1cf26 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json @@ -0,0 +1,304 @@ +[ + { + "description": "a schema given for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": true + }, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "item": { + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/sub-item" }, + { "$ref": "#/$defs/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "prefixItems with no additional items allowed", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}, {}, {}], + "items": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "items does not look in applicators, valid case", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "prefixItems": [ { "minimum": 3 } ] } + ], + "items": { "minimum": 5 } + }, + "tests": [ + { + "description": "prefixItems in allOf does not constrain items, invalid case", + "data": [ 3, 5 ], + "valid": false + }, + { + "description": "prefixItems in allOf does not constrain items, valid case", + "data": [ 5, 5 ], + "valid": true + } + ] + }, + { + "description": "prefixItems validation adjusts the starting index for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ { "type": "string" } ], + "items": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "items with heterogeneous array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}], + "items": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "items with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json new file mode 100644 index 000000000..9a42b45d6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json @@ -0,0 +1,123 @@ +[ + { + "description": "maxContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "two items still valid against lone maxContains", + "data": [ 1, 2 ], + "valid": true + } + ] + }, + { + "description": "maxContains with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + }, + { + "description": "some elements match, valid maxContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "some elements match, invalid maxContains", + "data": [ 1, 2, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains with contains, value with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1.0 + }, + "tests": [ + { + "description": "one element matches, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains < maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1, + "maxContains": 3 + }, + "tests": [ + { + "description": "actual < minContains < maxContains", + "data": [ ], + "valid": false + }, + { + "description": "minContains < actual < maxContains", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "minContains < maxContains < actual", + "data": [ 1, 1, 1, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains = 0 with minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 0 + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "one matching item", + "data": [ 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json new file mode 100644 index 000000000..f6a6b7c9a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "maxItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json new file mode 100644 index 000000000..7462726d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "maxLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json new file mode 100644 index 000000000..9cb745cad --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json @@ -0,0 +1,94 @@ +[ + { + "description": "maxProperties validation", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 0 + }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json new file mode 100644 index 000000000..7eeab2877 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json @@ -0,0 +1,60 @@ +[ + { + "description": "maximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 3.0 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 300 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json new file mode 100644 index 000000000..ee72d7d62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json @@ -0,0 +1,224 @@ +[ + { + "description": "minContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "zero items still valid against lone minContains", + "data": [], + "valid": true + } + ] + }, + { + "description": "minContains=1 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "no elements match", + "data": [ 2 ], + "valid": false + }, + { + "description": "single element matches, valid minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "all elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "some elements match, invalid minContains", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "all elements match, valid minContains (exactly as needed)", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "all elements match, valid minContains (more than needed)", + "data": [ 1, 1, 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains with a decimal value", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2.0 + }, + "tests": [ + { + "description": "one element matches, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "both elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains = minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 2, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "all elements match, valid maxContains and minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains < minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1, + "minContains": 3 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "invalid maxContains and minContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "minContains = 0 makes contains always pass", + "data": [ 2 ], + "valid": true + } + ] + }, + { + "description": "minContains = 0 with maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "not more than maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many", + "data": [ 1, 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json new file mode 100644 index 000000000..9d6a8b6d2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "minItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json new file mode 100644 index 000000000..5076c5a92 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "minLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json new file mode 100644 index 000000000..12587a5f8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "minProperties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json new file mode 100644 index 000000000..dc4405278 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json @@ -0,0 +1,75 @@ +[ + { + "description": "minimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": 1.1 + }, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -2 + }, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json new file mode 100644 index 000000000..76fa4765b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json @@ -0,0 +1,102 @@ +[ + { + "description": "by int", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 2 + }, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 1.5 + }, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 0.0001 + }, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 0.123456789 + }, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 1e-8 + }, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json new file mode 100644 index 000000000..d0f2b6e84 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json @@ -0,0 +1,301 @@ +[ + { + "description": "not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {} + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": true + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": false + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { "not": {} } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "collect annotations inside a 'not', even if collection is disabled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "$comment": "this subschema must still produce annotations internally, even though the 'not' will ultimately discard them", + "anyOf": [ + true, + { "properties": { "foo": true } } + ], + "unevaluatedProperties": false + } + }, + "tests": [ + { + "description": "unevaluated property", + "data": { "bar": 1 }, + "valid": true + }, + { + "description": "annotations are still collected inside a 'not'", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json new file mode 100644 index 000000000..7a7c7ffe3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json @@ -0,0 +1,293 @@ +[ + { + "description": "oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, true] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, false, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [false, false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json new file mode 100644 index 000000000..6d6713be5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json @@ -0,0 +1,60 @@ +[ + { + "description": "$anchor inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $anchor buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "anchor_in_enum": { + "enum": [ + { + "$anchor": "my_anchor", + "type": "null" + } + ] + }, + "real_identifier_in_schema": { + "$anchor": "my_anchor", + "type": "string" + }, + "zzz_anchor_in_const": { + "const": { + "$anchor": "my_anchor", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/anchor_in_enum" }, + { "$ref": "#my_anchor" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$anchor": "my_anchor", + "type": "null" + }, + "valid": true + }, + { + "description": "in implementations that strip $anchor, this may match either $def", + "data": { + "type": "null" + }, + "valid": false + }, + { + "description": "match $ref to $anchor", + "data": "a string to match #/$defs/anchor_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $anchor", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json new file mode 100644 index 000000000..d69b29e85 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json @@ -0,0 +1,110 @@ +[ + { + "description": "integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json new file mode 100644 index 000000000..5113bd649 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json @@ -0,0 +1,18 @@ +[ + { + "description": "refs to historic drafts are processed as historic drafts", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "$ref": "http://localhost:1234/draft2019-09/ignore-prefixItems.json" + }, + "tests": [ + { + "description": "first item not a string is valid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": [1, 2, 3], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json new file mode 100644 index 000000000..47d5bd79d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json @@ -0,0 +1,282 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "single schema dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "schema dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json new file mode 100644 index 000000000..7e63f209a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json @@ -0,0 +1,56 @@ +[ + { + "description": "$dynamicRef skips over intermediate resources - pointer reference across resource boundary", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/optional/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "bar#/$defs/item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }] \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json new file mode 100644 index 000000000..a4d62e0cf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json @@ -0,0 +1,582 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\p{Letter}cole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\wcole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "[a-z]cole" + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\d+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\p{digit}+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json new file mode 100644 index 000000000..f5ae8b18f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json @@ -0,0 +1,17 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "multipleOf": 0.5 + }, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json new file mode 100644 index 000000000..96f26fefb --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json @@ -0,0 +1,44 @@ +[ + { + "description": "schema that uses custom metaschema with format-assertion: false", + "comment": "The true/false boolean in $vocabulary only controls behavior for implementations that do not recognize the vocabulary. For implementations that do understand format-assertion, the boolean has no impact on validation behavior — hence both test groups produce identical results.", + "schema": { + "$id": "https://schema/using/format-assertion/false", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: false: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: false: invalid string", + "comment": "valid: false is intentional — the false value in $vocabulary only affects unknown vocabulary handling, not validation behavior itself", + "data": "not-an-ipv4", + "valid": false + } + ] + }, + { + "description": "schema that uses custom metaschema with format-assertion: true", + "schema": { + "$id": "https://schema/using/format-assertion/true", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: true: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: true: invalid string", + "data": "not-an-ipv4", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json new file mode 100644 index 000000000..590f2f678 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of date-time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json new file mode 100644 index 000000000..a8a19261a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json @@ -0,0 +1,423 @@ +[ + { + "description": "validation of date strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json new file mode 100644 index 000000000..e3e184161 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json @@ -0,0 +1,268 @@ +[ + { + "description": "validation of duration strings", + "comment": "RFC 3339 Appendix A defines the ABNF grammar for ISO-8601 durations used by JSON Schema format 'duration'. These tests enforce only the syntax defined by that grammar.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid duration string", + "data": "P4DT12H30M5S", + "valid": true + }, + { + "description": "an invalid duration string", + "data": "PT1D", + "valid": false + }, + { + "description": "must start with P", + "data": "4DT12H30M5S", + "valid": false + }, + { + "description": "no elements present", + "data": "P", + "valid": false + }, + { + "description": "no time elements present", + "data": "P1YT", + "valid": false + }, + { + "description": "no date or time elements present", + "data": "PT", + "valid": false + }, + { + "description": "elements out of order", + "data": "P2D1Y", + "valid": false + }, + { + "description": "missing time separator", + "data": "P1D2H", + "valid": false + }, + { + "description": "time element in the date position", + "data": "P2S", + "valid": false + }, + { + "description": "four years duration", + "data": "P4Y", + "valid": true + }, + { + "description": "zero time, in seconds", + "data": "PT0S", + "valid": true + }, + { + "description": "zero time, in days", + "data": "P0D", + "valid": true + }, + { + "description": "one month duration", + "data": "P1M", + "valid": true + }, + { + "description": "one minute duration", + "data": "PT1M", + "valid": true + }, + { + "description": "one and a half days, in hours", + "data": "PT36H", + "valid": true + }, + { + "description": "one and a half days, in days and hours", + "data": "P1DT12H", + "valid": true + }, + { + "description": "two weeks", + "data": "P2W", + "valid": true + }, + { + "description": "weeks cannot be combined with other units", + "data": "P1Y2W", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "P২Y", + "valid": false + }, + { + "description": "element without unit", + "data": "P1", + "valid": false + }, + { + "description": "all date and time components", + "data": "P1Y2M3DT4H5M6S", + "valid": true + }, + { + "description": "date components only", + "data": "P1Y2M3D", + "valid": true + }, + { + "description": "time components only", + "data": "PT1H2M3S", + "valid": true + }, + { + "description": "month and day", + "data": "P1M2D", + "valid": true + }, + { + "description": "hour and minute", + "data": "PT1H30M", + "valid": true + }, + { + "description": "multi-digit values in all components", + "data": "P10Y10M10DT10H10M10S", + "valid": true + }, + { + "description": "fractional duration is not allowed by RFC 3339 ABNF", + "comment": "numeric components use 1*DIGIT where DIGIT = %x30-39; '.' is not allowed", + "data": "PT0.5S", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " P1D", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "P1D ", + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "years and months can appear without days", + "data": "P1Y2M", + "valid": true + }, + { + "description": "years and days cannot appear without months", + "data": "P1Y2D", + "valid": false + }, + { + "description": "hours and minutes can appear without seconds", + "data": "PT1H2M", + "valid": true + }, + { + "description": "hours and seconds cannot appear without minutes", + "data": "PT1H2S", + "valid": false + }, + { + "description": "minutes and seconds can appear without hour", + "data": "PT1M2S", + "valid": true + }, + { + "description": "a leading sign is not allowed", + "data": "-P1D", + "valid": false + }, + { + "description": "a number before the time separator has no unit", + "data": "P1D2T3H", + "valid": false + }, + { + "description": "exponent notation is not allowed in a component", + "data": "P1e2D", + "valid": false + }, + { + "description": "a trailing newline is invalid", + "data": "P1D\n", + "valid": false + }, + { + "description": "weeks cannot be combined with a time component", + "data": "P1WT1H", + "valid": false + }, + { + "description": "weeks cannot be combined with a zero-valued component", + "data": "P0Y1W", + "valid": false + }, + { + "description": "a leading zero in a component is valid", + "data": "P01D", + "valid": true + }, + { + "description": "a component with many digits is valid", + "data": "P999999999999999999999999999999999999999999999999999999999999999999999999999999D", + "valid": true + }, + { + "description": "a comma as the decimal separator is invalid", + "data": "PT0,5S", + "valid": false + }, + { + "description": "a sign inside a component is invalid", + "data": "P-1D", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..b9ae3c049 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json @@ -0,0 +1,116 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json new file mode 100644 index 000000000..50e214622 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json @@ -0,0 +1,368 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json new file mode 100644 index 000000000..7cacfcfd2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json @@ -0,0 +1,386 @@ +[ + { + "description": "validation of host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json new file mode 100644 index 000000000..f46238b6f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json @@ -0,0 +1,140 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json new file mode 100644 index 000000000..62f914d02 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json @@ -0,0 +1,588 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json new file mode 100644 index 000000000..696713e90 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json @@ -0,0 +1,217 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "１９２.１６８.１.１", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json new file mode 100644 index 000000000..e5b46a00f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json @@ -0,0 +1,266 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json new file mode 100644 index 000000000..d31cdbbea --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json @@ -0,0 +1,116 @@ +[ + { + "description": "validation of IRI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json new file mode 100644 index 000000000..db27c829a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of IRIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json new file mode 100644 index 000000000..d5b9a8bb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json @@ -0,0 +1,211 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json new file mode 100644 index 000000000..dc3e72937 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json @@ -0,0 +1,103 @@ +[ + { + "description": "validation of regular expressions", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..413c03fd8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json new file mode 100644 index 000000000..2f4b6353f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json @@ -0,0 +1,303 @@ +[ + { + "description": "validation of time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json new file mode 100644 index 000000000..7fc35f536 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json @@ -0,0 +1,46 @@ +[ + { + "description": "unknown format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "unknown" + }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json new file mode 100644 index 000000000..08c655e00 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json @@ -0,0 +1,182 @@ +[ + { + "description": "validation of URI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json new file mode 100644 index 000000000..58d28f9de --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json @@ -0,0 +1,251 @@ +[ + { + "description": "format: uri-template", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json new file mode 100644 index 000000000..e3736ccaf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json @@ -0,0 +1,253 @@ +[ + { + "description": "validation of URIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json new file mode 100644 index 000000000..4679e996a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json @@ -0,0 +1,156 @@ +[ + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "all upper-case", + "data": "2EB8AA08-AA98-11EA-B4AA-73B441D16380", + "valid": true + }, + { + "description": "all lower-case", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": true + }, + { + "description": "mixed case", + "data": "2eb8aa08-AA98-11ea-B4Aa-73B441D16380", + "valid": true + }, + { + "description": "all zeroes is valid", + "data": "00000000-0000-0000-0000-000000000000", + "valid": true + }, + { + "description": "wrong length", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": false + }, + { + "description": "missing section", + "data": "2eb8aa08-aa98-11ea-73b441d16380", + "valid": false + }, + { + "description": "bad characters (not hex)", + "data": "2eb8aa08-aa98-11ea-b4ga-73b441d16380", + "valid": false + }, + { + "description": "no dashes", + "data": "2eb8aa08aa9811eab4aa73b441d16380", + "valid": false + }, + { + "description": "too few dashes", + "data": "2eb8aa08aa98-11ea-b4aa73b441d16380", + "valid": false + }, + { + "description": "too many dashes", + "data": "2eb8-aa08-aa98-11ea-b4aa73b44-1d16380", + "valid": false + }, + { + "description": "dashes in the wrong spot", + "data": "2eb8aa08aa9811eab4aa73b441d16380----", + "valid": false + }, + { + "description": "shifted dashes", + "data": "2eb8aa0-8aa98-11e-ab4aa7-3b441d16380", + "valid": false + }, + { + "description": "valid version 4", + "data": "98d80576-482e-427f-8434-7f86890ab222", + "valid": true + }, + { + "description": "valid version 5", + "data": "99c17cbb-656f-564a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 6", + "data": "99c17cbb-656f-664a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 15", + "data": "99c17cbb-656f-f64a-940f-1a4568f03487", + "valid": true + }, + { + "description": "URN prefixed UUID is invalid", + "data": "urn:uuid:2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "a UUID wrapped in curly braces is invalid", + "data": "{2eb8aa08-aa98-11ea-b4aa-73b441d16380}", + "valid": false + }, + { + "description": "trailing hyphen after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380-", + "valid": false + }, + { + "description": "non-ASCII digit '২' (a Bengali 2) is invalid", + "data": "২eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "an underscore in place of a hex digit is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1_380", + "valid": false + }, + { + "description": "a trailing newline after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380\n", + "valid": false + }, + { + "description": "a variant nibble not defined by RFC 4122 is valid", + "data": "2eb8aa08-aa98-11ea-f4aa-73b441d16380", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json new file mode 100644 index 000000000..0b7df4e80 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json @@ -0,0 +1,53 @@ +[ + { + "description": "$id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $id buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_enum" }, + { "$ref": "https://localhost:1234/draft2020-12/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to $id", + "data": "a string to match #/$defs/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $id", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json new file mode 100644 index 000000000..676e6b53a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json @@ -0,0 +1,26 @@ +[ + { + "description": "validation without $schema", + "comment": "minLength is the same across all drafts", + "schema": { + "minLength": 2 + }, + "tests": [ + { + "description": "a 3-character string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a 1-character string is not valid", + "data": "a", + "valid": false + }, + { + "description": "a non-string is valid", + "data": 5, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json new file mode 100644 index 000000000..d2efb3ef6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json @@ -0,0 +1,86 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^🐲*$" + }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json new file mode 100644 index 000000000..ab1510dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json @@ -0,0 +1,112 @@ +[ + { + "description": "reference of a root arbitrary keyword ", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown-keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of a root arbitrary keyword with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown/keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown-keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference internals of known non-applicator", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "examples": [ + { "type": "string" } + ], + "$ref": "#/examples/0" + }, + "tests": [ + { + "description": "match", + "data": "a string", + "valid": true + }, + { + "description": "mismatch", + "data": 42, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown/keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json new file mode 100644 index 000000000..28b0c4ce9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json @@ -0,0 +1,57 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_unknown0" }, + { "$ref": "#/$defs/id_in_unknown1" }, + { "$ref": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json new file mode 100644 index 000000000..ed41f79a7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json @@ -0,0 +1,90 @@ +[ + { + "description": "pattern validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^a*$" + }, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "a+" + }, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + }, + { + "description": "pattern with Unicode property escape requires unicode mode", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\p{Letter}+$" + }, + "tests": [ + { + "description": "ASCII letters match", + "data": "Hello", + "valid": true + }, + { + "description": "Non-ASCII letters match", + "data": "π", + "valid": true + }, + { + "description": "Digits do not match", + "data": "123", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json new file mode 100644 index 000000000..5a0ce941b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json @@ -0,0 +1,211 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + }, + { + "description": "patternProperties with Unicode property escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{Letter}+$": { + "type": "number" + } + } + }, + "tests": [ + { + "description": "Unicode letter property name matches", + "data": { + "π": 1 + }, + "valid": true + }, + { + "description": "Unicode letter property name matches but value type is incorrect", + "data": { + "π": "1" + }, + "valid": false + }, + { + "description": "Non-letter property name does not match pattern", + "data": { + "123": 1 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json new file mode 100644 index 000000000..0adfc069e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json @@ -0,0 +1,104 @@ +[ + { + "description": "a schema given for prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "prefixItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "additional items are allowed by default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "integer"}] + }, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "prefixItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json new file mode 100644 index 000000000..eb66fa8bd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json @@ -0,0 +1,242 @@ +[ + { + "description": "object properties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json new file mode 100644 index 000000000..84ca2b84e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json @@ -0,0 +1,178 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": true + }, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": false + }, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"const": "foo"} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"enum": ["foo", "bar"]} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json new file mode 100644 index 000000000..9ca82eeb0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json @@ -0,0 +1,1085 @@ +[ + { + "description": "root pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"$ref": "#/prefixItems/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/$defs/tilde~0field"}, + "slash": {"$ref": "#/$defs/slash~1field"}, + "percent": {"$ref": "#/$defs/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a": {"type": "integer"}, + "b": {"$ref": "#/$defs/a"}, + "c": {"$ref": "#/$defs/b"} + }, + "$ref": "#/$defs/c" + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref applies alongside sibling keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/$defs/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid, maxItems valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems invalid", + "data": { "foo": [1, 2, 3] }, + "valid": false + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"$ref": "#/$defs/is-string"} + }, + "$defs": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "$defs": { + "node": { + "$id": "http://localhost:1234/draft2020-12/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\"bar": {"$ref": "#/$defs/foo%22bar"} + }, + "$defs": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "ref creates new scope when adjacent to keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "A": { + "unevaluatedProperties": false + } + }, + "properties": { + "prop1": { + "type": "string" + } + }, + "$ref": "#/$defs/A" + }, + "tests": [ + { + "description": "referenced subschema doesn't see annotations from properties", + "data": { + "prop1": "match" + }, + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/$defs/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/$defs/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-relative-uri-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-refs-absolute-uris-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/a.json", + "$defs": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "$defs": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id1/base.json", + "$ref": "int.json", + "$defs": { + "bigint": { + "$comment": "canonical uri: https://example.com/ref-and-id1/int.json", + "$id": "int.json", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id1-int.json", + "$id": "/draft2020-12/ref-and-id1-int.json", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $anchor and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id2/base.json", + "$ref": "#bigint", + "$defs": { + "bigint": { + "$comment": "canonical uri: /ref-and-id2/base.json#/$defs/bigint; another valid uri for this location: /ref-and-id2/base.json#bigint", + "$anchor": "bigint", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id2#/$defs/smallint; another valid uri for this location: https://example.com/ref-and-id2/#bigint", + "$id": "https://example.com/draft2020-12/ref-and-id2/", + "$anchor": "bigint", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref on nested schema", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id3/base.json", + "$ref": "nested/foo.json", + "$defs": { + "foo": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/foo.json", + "$id": "nested/foo.json", + "$ref": "./bar.json" + }, + "bar": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/bar.json", + "$id": "nested/bar.json", + "type": "number" + } + } + }, + "tests": [ + { + "description": "data is valid against nested sibling", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against nested sibling", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "$defs": { + "bar": { + "$anchor": "something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN ref with nested pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": { + "foo": { + "$id": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": "bar", + "valid": true + }, + { + "description": "a non-string is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/if", + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/then", + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/else", + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/ref/absref.json", + "$defs": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "$ref": "/absref/foobar.json" + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///c:/folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "": { + "$defs": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs//$defs/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json new file mode 100644 index 000000000..6083aabb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json @@ -0,0 +1,342 @@ +[ + { + "description": "remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/integer.json" + }, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/integer" + }, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "anchor within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#foo" + }, + "tests": [ + { + "description": "remote anchor valid", + "data": 1, + "valid": true + }, + { + "description": "remote anchor invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs1.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolder/"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs2.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolderInSubschema/#/$defs/bar"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "$defs": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/object", + "type": "object", + "properties": { + "name": {"$ref": "name-defs.json#/$defs/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/schema-remote-ref-ref-defs1.json", + "$ref": "ref-and-defs.json" + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/different-id-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different URN $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/urn-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with nested absolute ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/nested-absolute-ref-to-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds detached $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-ref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json new file mode 100644 index 000000000..17db8939e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json @@ -0,0 +1,169 @@ +[ + { + "description": "required validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description":"ignores boolean", + "data": true, + "valid": true + + } + ] + }, + { + "description": "required default validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": ["__proto__", "toString", "constructor"] + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json new file mode 100644 index 000000000..2123c408d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json @@ -0,0 +1,501 @@ +[ + { + "description": "integer type matches integers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object" + }, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array" + }, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "boolean" + }, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "null" + }, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["integer", "string"] + }, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json new file mode 100644 index 000000000..d8a7c1b5b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json @@ -0,0 +1,854 @@ +[ + { + "description": "unevaluatedItems true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": true + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems as schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { "type": "string" } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with valid unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with invalid unevaluated items", + "data": [42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with uniform items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "type": "string" }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with items and prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "items": true, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "number"}, + "unevaluatedItems": {"type": "string"} + }, + "tests": [ + { + "description": "valid under items", + "comment": "no elements are considered by unevaluatedItems", + "data": [5, 6, 7, 8], + "valid": true + }, + { + "description": "invalid under items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "allOf": [ + { + "prefixItems": [ + true, + { "type": "number" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", 42], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", 42, true], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": {"type": "boolean"}, + "anyOf": [ + { "items": {"type": "string"} }, + true + ] + }, + "tests": [ + { + "description": "with only (valid) additional items", + "data": [true, false], + "valid": true + }, + { + "description": "with no additional items", + "data": ["yes", "no"], + "valid": true + }, + { + "description": "with invalid additional item", + "data": ["yes", false], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested prefixItems and items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ], + "items": true + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with nested unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ] + }, + { "unevaluatedItems": true } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "anyOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when one schema matches and has no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "when one schema matches and has unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + }, + { + "description": "when two schemas match and has no unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "when two schemas match and has unevaluated items", + "data": ["foo", "bar", "baz", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "oneOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "not": { + "not": { + "prefixItems": [ + true, + { "const": "bar" } + ] + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "if": { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + "then": { + "prefixItems": [ + true, + true, + { "const": "then" } + ] + }, + "else": { + "prefixItems": [ + true, + true, + true, + { "const": "else" } + ] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when if matches and it has no unevaluated items", + "data": ["foo", "bar", "then"], + "valid": true + }, + { + "description": "when if matches and it has unevaluated items", + "data": ["foo", "bar", "then", "else"], + "valid": false + }, + { + "description": "when if doesn't match and it has no unevaluated items", + "data": ["foo", 42, 42, "else"], + "valid": true + }, + { + "description": "when if doesn't match and it has unevaluated items", + "data": ["foo", 42, 42, "else", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false, + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false, + "prefixItems": [ + { "type": "string" } + ], + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-items-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "prefixItems": [ + true, + { "type": "string" } + ] + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "type": "array", + "prefixItems": [ + { "type": "string" } + ], + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ true ] + }, + { "unevaluatedItems": false } + ] + }, + "tests": [ + { + "description": "always fails", + "data": [ 1 ], + "valid": false + } + ] + }, + { + "description": "item is evaluated in an uncle schema to unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + } + ] + }, + "tests": [ + { + "description": "no extra items", + "data": { + "foo": [ + "test" + ] + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": [ + "test", + "test" + ] + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on adjacent contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true], + "contains": {"type": "string"}, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "second item is evaluated by contains", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "contains fails, second item is not evaluated", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "contains passes, second item is not evaluated", + "data": [ 1, 2, "foo" ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on multiple nested contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "contains": { "multipleOf": 2 } }, + { "contains": { "multipleOf": 3 } } + ], + "unevaluatedItems": { "multipleOf": 5 } + }, + "tests": [ + { + "description": "5 not evaluated, passes unevaluatedItems", + "data": [ 2, 3, 4, 5, 6 ], + "valid": true + }, + { + "description": "7 not evaluated, fails unevaluatedItems", + "data": [ 2, 3, 4, 7, 8 ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems and contains interact to control item dependency relationship", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "contains": {"const": "a"} + }, + "then": { + "if": { + "contains": {"const": "b"} + }, + "then": { + "if": { + "contains": {"const": "c"} + } + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "only a's are valid", + "data": [ "a", "a" ], + "valid": true + }, + { + "description": "a's and b's are valid", + "data": [ "a", "b", "a", "b", "a" ], + "valid": true + }, + { + "description": "a's, b's and c's are valid", + "data": [ "c", "a", "c", "c", "b", "a" ], + "valid": true + }, + { + "description": "only b's are invalid", + "data": [ "b", "b" ], + "valid": false + }, + { + "description": "only c's are invalid", + "data": [ "c", "c" ], + "valid": false + }, + { + "description": "only b's and c's are invalid", + "data": [ "c", "b", "c", "b", "c" ], + "valid": false + }, + { + "description": "only a's and c's are invalid", + "data": [ "c", "a", "c", "a", "c" ], + "valid": false + } + ] + }, + { + "description" : "unevaluatedItems with minContains = 0", + "schema" : { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"type": "string"}, + "minContains": 0, + "unevaluatedItems": false + }, + "tests" : [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "no items evaluated by contains", + "data": [0], + "valid": false + }, + { + "description": "some but not all items evaluated by contains", + "data": ["foo", 0], + "valid": false + }, + { + "description": "all items evaluated by contains", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "non-array instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "prefixItems": [{"const": "a"}] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": [ "a" ], + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": [ "b" ], + "valid": false + } + ] + }, + { + "description": "Evaluated items collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "prefixItems": [ + true, + { "type": "string" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with an unevaluated item that exists at another location", + "data": [ + ["foo", "bar"], + "bar" + ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json new file mode 100644 index 000000000..1995d0ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json @@ -0,0 +1,1681 @@ +[ + { + "description": "unevaluatedProperties true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "string", + "minLength": 3 + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with valid unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with invalid unevaluated properties", + "data": { + "foo": "fo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": true, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with adjacent non-bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": {"type": "string"}, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "properties": { + "bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "patternProperties": { + "^bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "additionalProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": { + "type": "string", + "maxLength": 2 + } + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "anyOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + }, + { + "properties": { + "quux": { "const": "quux" } + }, + "required": ["quux"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when one matches and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when one matches and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "not-baz" + }, + "valid": false + }, + { + "description": "when two match and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": true + }, + { + "description": "when two match and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz", + "quux": "not-quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "oneOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "quux": "quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "not": { + "not": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, then not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, else not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with dependentSchemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [true], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false, + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-properties-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "properties": { + "bar": { "type": "string" } + } + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedProperties comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins (reverse order)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": false + }, + { + "properties": { + "foo": true + } + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, true with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, false with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": true + }, + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "property is evaluated in an uncle schema to unevaluatedProperties", + "comment": "see https://stackoverflow.com/questions/66936884/deeply-nested-unevaluatedproperties-and-their-expectations", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { + "type": "string" + } + }, + "unevaluatedProperties": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "properties": { + "faz": { + "type": "string" + } + } + } + } + } + ] + }, + "tests": [ + { + "description": "no extra properties", + "data": { + "foo": { + "bar": "test" + } + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": { + "bar": "test", + "faz": "test" + } + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, allOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + }, + "unevaluatedProperties": false + } + ], + "anyOf": [ + { + "properties": { + "bar": true + } + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": true + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, anyOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + } + ], + "anyOf": [ + { + "properties": { + "bar": true + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties + single cyclic ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "x": { "$ref": "#" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is valid", + "data": {}, + "valid": true + }, + { + "description": "Single is valid", + "data": { "x": {} }, + "valid": true + }, + { + "description": "Unevaluated on 1st level is invalid", + "data": { "x": {}, "y": {} }, + "valid": false + }, + { + "description": "Nested is valid", + "data": { "x": { "x": {} } }, + "valid": true + }, + { + "description": "Unevaluated on 2nd level is invalid", + "data": { "x": { "x": {}, "y": {} } }, + "valid": false + }, + { + "description": "Deep nested is valid", + "data": { "x": { "x": { "x": {} } } }, + "valid": true + }, + { + "description": "Unevaluated on 3rd level is invalid", + "data": { "x": { "x": { "x": {}, "y": {} } } }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties + ref inside allOf / oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "properties": { "a": true } + }, + "two": { + "required": ["x"], + "properties": { "x": true } + } + }, + "allOf": [ + { "$ref": "#/$defs/one" }, + { "properties": { "b": true } }, + { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { + "required": ["y"], + "properties": { "y": true } + } + ] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid (no x or y)", + "data": {}, + "valid": false + }, + { + "description": "a and b are invalid (no x or y)", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "x and y are invalid", + "data": { "x": 1, "y": 1 }, + "valid": false + }, + { + "description": "a and x are valid", + "data": { "a": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and y are valid", + "data": { "a": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x are valid", + "data": { "a": 1, "b": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and b and y are valid", + "data": { "a": 1, "b": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x and y are invalid", + "data": { "a": 1, "b": 1, "x": 1, "y": 1 }, + "valid": false + } + ] + }, + { + "description": "dynamic evaluation inside nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { "required": ["b"], "properties": { "b": true } }, + { "required": ["xx"], "patternProperties": { "x": true } }, + { "required": ["all"], "unevaluatedProperties": true } + ] + }, + "two": { + "oneOf": [ + { "required": ["c"], "properties": { "c": true } }, + { "required": ["d"], "properties": { "d": true } } + ] + } + }, + "oneOf": [ + { "$ref": "#/$defs/one" }, + { "required": ["a"], "properties": { "a": true } } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid", + "data": {}, + "valid": false + }, + { + "description": "a is valid", + "data": { "a": 1 }, + "valid": true + }, + { + "description": "b is valid", + "data": { "b": 1 }, + "valid": true + }, + { + "description": "c is valid", + "data": { "c": 1 }, + "valid": true + }, + { + "description": "d is valid", + "data": { "d": 1 }, + "valid": true + }, + { + "description": "a + b is invalid", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "a + c is invalid", + "data": { "a": 1, "c": 1 }, + "valid": false + }, + { + "description": "a + d is invalid", + "data": { "a": 1, "d": 1 }, + "valid": false + }, + { + "description": "b + c is invalid", + "data": { "b": 1, "c": 1 }, + "valid": false + }, + { + "description": "b + d is invalid", + "data": { "b": 1, "d": 1 }, + "valid": false + }, + { + "description": "c + d is invalid", + "data": { "c": 1, "d": 1 }, + "valid": false + }, + { + "description": "xx is valid", + "data": { "xx": 1 }, + "valid": true + }, + { + "description": "xx + foox is valid", + "data": { "xx": 1, "foox": 1 }, + "valid": true + }, + { + "description": "xx + foo is invalid", + "data": { "xx": 1, "foo": 1 }, + "valid": false + }, + { + "description": "xx + a is invalid", + "data": { "xx": 1, "a": 1 }, + "valid": false + }, + { + "description": "xx + b is invalid", + "data": { "xx": 1, "b": 1 }, + "valid": false + }, + { + "description": "xx + c is invalid", + "data": { "xx": 1, "c": 1 }, + "valid": false + }, + { + "description": "xx + d is invalid", + "data": { "xx": 1, "d": 1 }, + "valid": false + }, + { + "description": "all is valid", + "data": { "all": 1 }, + "valid": true + }, + { + "description": "all + foo is valid", + "data": { "all": 1, "foo": 1 }, + "valid": true + }, + { + "description": "all + a is invalid", + "data": { "all": 1, "a": 1 }, + "valid": false + } + ] + }, + { + "description": "non-object instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null valued properties", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties not affected by propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 1}, + "unevaluatedProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "allows only number properties", + "data": {"a": 1}, + "valid": true + }, + { + "description": "string property is invalid", + "data": {"a": "b"}, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "patternProperties": { + "foo": { + "type": "string" + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": { + "foo": "a" + }, + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": { + "bar": "a" + }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar":{} + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "unevaluatedProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties doesn't see bar when foo2 is absent", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties sees bar when foo2 is present", + "data": { "foo2": "", "bar": ""}, + "valid": true + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "patternProperties": { + "^bar$": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "additionalProperties": { "type": "string" } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json new file mode 100644 index 000000000..262114529 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json @@ -0,0 +1,399 @@ +[ + { + "description": "uniqueItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": true + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": false + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json new file mode 100644 index 000000000..1acb96a93 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json @@ -0,0 +1,57 @@ +[ + { + "description": "schema that uses custom metaschema with with no validation vocabulary", + "schema": { + "$id": "https://schema/using/no/validation", + "$schema": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "properties": { + "badProperty": false, + "numberProperty": { + "minimum": 10 + } + } + }, + "tests": [ + { + "description": "applicator vocabulary still works", + "data": { + "badProperty": "this property should not exist" + }, + "valid": false + }, + { + "description": "no validation: valid number", + "data": { + "numberProperty": 20 + }, + "valid": true + }, + { + "description": "no validation: invalid number, but it still validates", + "data": { + "numberProperty": 1 + }, + "valid": true + } + ] + }, + { + "description": "ignore unrecognized optional vocabulary", + "schema": { + "$schema": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "type": "number" + }, + "tests": [ + { + "description": "string value", + "data": "foobar", + "valid": false + }, + { + "description": "number value", + "data": 20, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json new file mode 100644 index 000000000..6ef4212b1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json @@ -0,0 +1,190 @@ +[ + { + "description": "additionalItems as schema", + "schema": { + "items": [{}], + "additionalItems": {"type": "integer"} + }, + "tests": [ + { + "description": "additional items match schema", + "data": [ null, 2, 3, 4 ], + "valid": true + }, + { + "description": "additional items do not match schema", + "data": [ null, 2, 3, "foo" ], + "valid": false + } + ] + }, + { + "description": "when items is schema, additionalItems does nothing", + "schema": { + "items": { + "type": "integer" + }, + "additionalItems": { + "type": "string" + } + }, + "tests": [ + { + "description": "valid with a array of type integers", + "data": [1,2,3], + "valid": true + }, + { + "description": "invalid with a array of mixed types", + "data": [1,"2","3"], + "valid": false + } + ] + }, + { + "description": "when items is schema, boolean additionalItems does nothing", + "schema": { + "items": {}, + "additionalItems": false + }, + "tests": [ + { + "description": "all items match schema", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + } + ] + }, + { + "description": "array of items with no additionalItems permitted", + "schema": { + "items": [{}, {}, {}], + "additionalItems": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "additionalItems as false without items", + "schema": {"additionalItems": false}, + "tests": [ + { + "description": + "items defaults to empty schema so everything is valid", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + } + ] + }, + { + "description": "additionalItems are allowed by default", + "schema": {"items": [{"type": "integer"}]}, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "additionalItems does not look in applicators, invalid case", + "schema": { + "allOf": [ + { "items": [ { "type": "integer" }, { "type": "string" } ] } + ], + "items": [ {"type": "integer" } ], + "additionalItems": { "type": "boolean" } + }, + "tests": [ + { + "description": "items defined in allOf are not examined", + "data": [ 1, "hello" ], + "valid": false + } + ] + }, + { + "description": "items validation adjusts the starting index for additionalItems", + "schema": { + "items": [ { "type": "string" } ], + "additionalItems": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "additionalItems with heterogeneous array", + "schema": { + "items": [{}], + "additionalItems": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "additionalItems with null instance elements", + "schema": { + "additionalItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json new file mode 100644 index 000000000..0f8e1627a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json @@ -0,0 +1,147 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "schema": { + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": + "additionalProperties can exist by itself", + "schema": { + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "schema": {"properties": {"foo": {}, "bar": {}}}, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "schema": { + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "schema": { + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json new file mode 100644 index 000000000..ec9319e14 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json @@ -0,0 +1,294 @@ +[ + { + "description": "allOf", + "schema": { + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": {"allOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": {"allOf": [true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": {"allOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json new file mode 100644 index 000000000..ab5eb386b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json @@ -0,0 +1,189 @@ +[ + { + "description": "anyOf", + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": {"anyOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": {"anyOf": [true, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": {"anyOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json new file mode 100644 index 000000000..168fff4e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json @@ -0,0 +1,384 @@ +[ + { + "description": "const validation", + "schema": {"const": 2}, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": {"const": {"foo": "bar", "baz": "bax"}}, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": {"const": [{ "foo": "bar" }]}, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": {"const": null}, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": {"const": false}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": {"const": true}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": {"const": [false]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": {"const": [true]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": {"const": {"a": false}}, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": {"const": {"a": true}}, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": {"const": 0}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": {"const": 1}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": {"const": -2.0}, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": {"const": 9007199254740992}, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "const": "hello\u0000there" }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json new file mode 100644 index 000000000..2b1a51520 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json @@ -0,0 +1,165 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": {"contains": true}, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": {"contains": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json new file mode 100644 index 000000000..289a9b66c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json @@ -0,0 +1,79 @@ +[ + { + "description": "invalid type for default", + "schema": { + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json new file mode 100644 index 000000000..afe396e42 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json @@ -0,0 +1,26 @@ +[ + { + "description": "validate definition against metaschema", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "valid definition schema", + "data": { + "definitions": { + "foo": {"type": "integer"} + } + }, + "valid": true + }, + { + "description": "invalid definition schema", + "data": { + "definitions": { + "foo": {"type": 1} + } + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json new file mode 100644 index 000000000..c0bd809f6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json @@ -0,0 +1,286 @@ +[ + { + "description": "dependencies", + "schema": { + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "dependencies with empty array", + "schema": { + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependencies", + "schema": { + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "multiple dependencies subschema", + "schema": { + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "dependencies with boolean subschemas", + "schema": { + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\tbar": { + "minProperties": 4 + }, + "foo'bar": {"required": ["foo\"bar"]}, + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "valid object 1", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "valid object 2", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "valid object 3", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "invalid object 1", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "invalid object 2", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "invalid object 3", + "data": { + "foo'bar": 1 + }, + "valid": false + }, + { + "description": "invalid object 4", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "properties": { + "foo": {} + }, + "dependencies": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json new file mode 100644 index 000000000..bf1e96fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json @@ -0,0 +1,341 @@ +[ + { + "description": "simple enum validation", + "schema": {"enum": [1, 2, 3]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": {"enum": [6, "foo", [], true, {"foo": 12}]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { "enum": [6, null] }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": {"enum": [false]}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": {"enum": [[false]]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": {"enum": [true]}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": {"enum": [[true]]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": {"enum": [0]}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": {"enum": [[0]]}, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": {"enum": [1]}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": {"enum": [[1]]}, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "enum": [ "hello\u0000there" ] }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json new file mode 100644 index 000000000..dc3cd709d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json new file mode 100644 index 000000000..b38d7ecec --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json new file mode 100644 index 000000000..e2447d60f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json @@ -0,0 +1,614 @@ +[ + { + "description": "email format", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json new file mode 100644 index 000000000..a3bab7c0b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json @@ -0,0 +1,296 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json new file mode 100644 index 000000000..f98c74fc6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json @@ -0,0 +1,36 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "definitions": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/definitions/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/definitions/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json new file mode 100644 index 000000000..7ed6781b6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json @@ -0,0 +1,282 @@ +[ + { + "description": "a schema given for items", + "schema": { + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "an array of schemas for items", + "schema": { + "items": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": {"items": true}, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": {"items": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schemas", + "schema": { + "items": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "definitions": { + "item": { + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/sub-item" }, + { "$ref": "#/definitions/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "single-form items with null instance elements", + "schema": { + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "array-form items with null instance elements", + "schema": { + "items": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json new file mode 100644 index 000000000..f0c36ab28 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "maxItems validation", + "schema": {"maxItems": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": {"maxItems": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json new file mode 100644 index 000000000..be60c5407 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "maxLength validation", + "schema": {"maxLength": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": {"maxLength": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json new file mode 100644 index 000000000..acec14209 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "maxProperties validation", + "schema": {"maxProperties": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "schema": {"maxProperties": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "schema": { "maxProperties": 0 }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json new file mode 100644 index 000000000..c3e727069 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json @@ -0,0 +1,54 @@ +[ + { + "description": "maximum validation", + "schema": {"maximum": 3.0}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": {"maximum": 300}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json new file mode 100644 index 000000000..d3b18720d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "minItems validation", + "schema": {"minItems": 1}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": {"minItems": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json new file mode 100644 index 000000000..23c68fe3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "minLength validation", + "schema": {"minLength": 2}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": {"minLength": 2.0}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json new file mode 100644 index 000000000..416e02010 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json @@ -0,0 +1,64 @@ +[ + { + "description": "minProperties validation", + "schema": {"minProperties": 1}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": {"minProperties": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json new file mode 100644 index 000000000..21ae50e0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json @@ -0,0 +1,69 @@ +[ + { + "description": "minimum validation", + "schema": {"minimum": 1.1}, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": {"minimum": -2}, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json new file mode 100644 index 000000000..da0ce0830 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json @@ -0,0 +1,87 @@ +[ + { + "description": "by int", + "schema": {"multipleOf": 2}, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": {"multipleOf": 1.5}, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": {"multipleOf": 0.0001}, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": {"type": "integer", "multipleOf": 0.123456789}, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": {"type": "integer", "multipleOf": 1e-8}, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json new file mode 100644 index 000000000..b46c4ed05 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json @@ -0,0 +1,259 @@ +[ + { + "description": "not", + "schema": { + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { "not": {} }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { "not": true }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { "not": false }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { "not": { "not": {} } }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json new file mode 100644 index 000000000..c30a65c0d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json @@ -0,0 +1,274 @@ +[ + { + "description": "oneOf", + "schema": { + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": {"oneOf": [true, true, true]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": {"oneOf": [true, false, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": {"oneOf": [true, true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": {"oneOf": [false, false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json new file mode 100644 index 000000000..94b4a4e62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json @@ -0,0 +1,93 @@ +[ + { + "description": "integer", + "schema": { "type": "integer" }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { "type": "number" }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { "type": "string" }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { "maximum": 18446744073709551615 }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { "minimum": -18446744073709551615 }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json new file mode 100644 index 000000000..3f5a7430b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json @@ -0,0 +1,77 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document", + "data": "{:}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character)", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document", + "data": "ezp9Cg==", + "valid": false + }, + { + "description": "an invalid base64 string that is valid JSON", + "data": "{}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json new file mode 100644 index 000000000..8ff537362 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json @@ -0,0 +1,25 @@ +[ + { + "description": "refs to future drafts are processed as future drafts", + "schema": { + "type": "object", + "allOf": [ + { "properties": { "foo": true } }, + { "$ref": "http://localhost:1234/draft2019-09/dependentRequired.json" } + ] + }, + "tests": [ + { + "description": "missing bar is invalid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": {"foo": "any value"}, + "valid": false + }, + { + "description": "present bar is valid", + "data": {"foo": "any value", "bar": "also any value"}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json new file mode 100644 index 000000000..c4886aaa8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json @@ -0,0 +1,552 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { "pattern": "\\p{Letter}cole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { "pattern": "\\wcole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { "pattern": "[a-z]cole" }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { "pattern": "^\\d+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { "pattern": "^\\p{digit}+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json new file mode 100644 index 000000000..52ff9827c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json @@ -0,0 +1,13 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": {"type": "integer", "multipleOf": 0.5}, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json new file mode 100644 index 000000000..07a4e9195 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json @@ -0,0 +1,247 @@ +[ + { + "description": "validation of date-time strings", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json new file mode 100644 index 000000000..4a3847b4f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json @@ -0,0 +1,420 @@ +[ + { + "description": "validation of date strings", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..bf3fae7f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json @@ -0,0 +1,98 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json new file mode 100644 index 000000000..6ef76f3c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json @@ -0,0 +1,365 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json new file mode 100644 index 000000000..bd5dcdea2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json @@ -0,0 +1,380 @@ +[ + { + "description": "validation of host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json new file mode 100644 index 000000000..c88bd9fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json @@ -0,0 +1,137 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json new file mode 100644 index 000000000..13d223d24 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json @@ -0,0 +1,582 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json new file mode 100644 index 000000000..d56d62b33 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json @@ -0,0 +1,214 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "１９２.１６８.１.１", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json new file mode 100644 index 000000000..7f4674be4 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json @@ -0,0 +1,263 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json new file mode 100644 index 000000000..c4b6f1853 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json @@ -0,0 +1,113 @@ +[ + { + "description": "validation of IRI References", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json new file mode 100644 index 000000000..4025e85d5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of IRIs", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json new file mode 100644 index 000000000..134eca586 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json @@ -0,0 +1,208 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json new file mode 100644 index 000000000..ff831fa2e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json @@ -0,0 +1,100 @@ +[ + { + "description": "validation of regular expressions", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..59299e7b3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json new file mode 100644 index 000000000..3182c3ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json @@ -0,0 +1,300 @@ +[ + { + "description": "validation of time strings", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json new file mode 100644 index 000000000..12339ae57 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json @@ -0,0 +1,43 @@ +[ + { + "description": "unknown format", + "schema": { "format": "unknown" }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json new file mode 100644 index 000000000..3741bad01 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json @@ -0,0 +1,179 @@ +[ + { + "description": "validation of URI References", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json new file mode 100644 index 000000000..f0aaa52f7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json @@ -0,0 +1,248 @@ +[ + { + "description": "format: uri-template", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json new file mode 100644 index 000000000..b9d080027 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of URIs", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json new file mode 100644 index 000000000..6be81b8da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json @@ -0,0 +1,114 @@ +[ + { + "description": "id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an id buried in the enum", + "schema": { + "definitions": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_enum" }, + { "$ref": "https://localhost:1234/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to id", + "data": "a string to match #/definitions/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to id", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing a plain-name $id property", + "schema": { + "definitions": { + "const_not_anchor": { + "const": { + "$id": "#not_a_real_anchor" + } + } + }, + "if": { + "const": "skip not_a_real_anchor" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_anchor" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_anchor", + "valid": true + }, + { + "description": "const at const_not_anchor does not match", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing an $id property", + "schema": { + "definitions": { + "const_not_id": { + "const": { + "$id": "not_a_real_id" + } + } + }, + "if": { + "const": "skip not_a_real_id" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_id" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_id", + "valid": true + }, + { + "description": "const at const_not_id does not match", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json new file mode 100644 index 000000000..dd67af2b2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json @@ -0,0 +1,82 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { "pattern": "^🐲*$" }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json new file mode 100644 index 000000000..1f58d97e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json @@ -0,0 +1,56 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "definitions": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_unknown0" }, + { "$ref": "#/definitions/id_in_unknown1" }, + { "$ref": "https://localhost:1234/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json new file mode 100644 index 000000000..92db0f971 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json @@ -0,0 +1,59 @@ +[ + { + "description": "pattern validation", + "schema": {"pattern": "^a*$"}, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": {"pattern": "a+"}, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json new file mode 100644 index 000000000..c276e6479 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json @@ -0,0 +1,171 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json new file mode 100644 index 000000000..5b971ca0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json @@ -0,0 +1,236 @@ +[ + { + "description": "object properties validation", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json new file mode 100644 index 000000000..b86564310 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json @@ -0,0 +1,164 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": {"propertyNames": true}, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": {"propertyNames": false}, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": {"propertyNames": {"const": "foo"}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": {"propertyNames": {"enum": ["foo", "bar"]}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json new file mode 100644 index 000000000..82e1e1672 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json @@ -0,0 +1,1043 @@ +[ + { + "description": "root pointer ref", + "schema": { + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "items": [ + {"type": "integer"}, + {"$ref": "#/items/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "definitions": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/definitions/tilde~0field"}, + "slash": {"$ref": "#/definitions/slash~1field"}, + "percent": {"$ref": "#/definitions/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "definitions": { + "a": {"type": "integer"}, + "b": {"$ref": "#/definitions/a"}, + "c": {"$ref": "#/definitions/b"} + }, + "allOf": [{ "$ref": "#/definitions/c" }] + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref overrides any sibling keywords", + "schema": { + "definitions": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/definitions/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems ignored", + "data": { "foo": [ 1, 2, 3] }, + "valid": true + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "$ref prevents a sibling $id from changing the base uri", + "schema": { + "$id": "http://localhost:1234/sibling_id/base/", + "definitions": { + "foo": { + "$id": "http://localhost:1234/sibling_id/foo.json", + "type": "string" + }, + "base_foo": { + "$comment": "this canonical uri is http://localhost:1234/sibling_id/base/foo.json", + "$id": "foo.json", + "type": "number" + } + }, + "allOf": [ + { + "$comment": "$ref resolves to http://localhost:1234/sibling_id/base/foo.json, not http://localhost:1234/sibling_id/foo.json", + "$id": "http://localhost:1234/sibling_id/", + "$ref": "foo.json" + } + ] + }, + "tests": [ + { + "description": "$ref resolves to /definitions/base_foo, data does not validate", + "data": "a", + "valid": false + }, + { + "description": "$ref resolves to /definitions/base_foo, data validates", + "data": 1, + "valid": true + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "properties": { + "$ref": {"$ref": "#/definitions/is-string"} + }, + "definitions": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$id": "http://localhost:1234/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "definitions": { + "node": { + "$id": "http://localhost:1234/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "properties": { + "foo\"bar": {"$ref": "#/definitions/foo%22bar"} + }, + "definitions": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "Location-independent identifier", + "schema": { + "allOf": [{ + "$ref": "#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Reference an anchor with a non-relative URI", + "schema": { + "$id": "https://example.com/schema-with-anchor", + "allOf": [{ + "$ref": "https://example.com/schema-with-anchor#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$id": "http://localhost:1234/root", + "allOf": [{ + "$ref": "http://localhost:1234/nested.json#foo" + }], + "definitions": { + "A": { + "$id": "nested.json", + "definitions": { + "B": { + "$id": "#foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "definitions": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/definitions/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/definitions/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-relative-uri-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-refs-absolute-uris-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$id": "http://example.com/a.json", + "definitions": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "definitions": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "definitions": { + "bar": { + "$id": "#something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/if"}, + { + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/then"}, + { + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/else"}, + { + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$id": "http://example.com/ref/absref.json", + "definitions": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "allOf": [ + { "$ref": "/absref/foobar.json" } + ] + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$id": "file:///folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$id": "file:///c:/folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "definitions": { + "": { + "definitions": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions//definitions/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json new file mode 100644 index 000000000..450787af6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json @@ -0,0 +1,257 @@ +[ + { + "description": "remote ref", + "schema": {"$ref": "http://localhost:1234/integer.json"}, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": {"$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/integer"}, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$id": "http://localhost:1234/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$id": "http://localhost:1234/scope_change_defs1.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$id": "http://localhost:1234/scope_change_defs2.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz/definitions/bar"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "definitions": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$id": "http://localhost:1234/object", + "type": "object", + "properties": { + "name": {"$ref": "draft7/name.json#/definitions/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to definitions", + "schema": { + "$id": "http://localhost:1234/schema-remote-ref-ref-defs1.json", + "allOf": [ + { "$ref": "draft7/ref-and-definitions.json" } + ] + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/locationIndependentIdentifier.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$id": "http://localhost:1234/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds location-independent $id", + "schema": { + "$ref": "http://localhost:1234/draft7/detached-ref.json#/definitions/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json new file mode 100644 index 000000000..f98c64af8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json @@ -0,0 +1,161 @@ +[ + { + "description": "required validation", + "schema": { + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores boolean", + "data": true, + "valid": true + } + ] + }, + { + "description": "required default validation", + "schema": { + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { "required": ["__proto__", "toString", "constructor"] }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json new file mode 100644 index 000000000..830464702 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json @@ -0,0 +1,474 @@ +[ + { + "description": "integer type matches integers", + "schema": {"type": "integer"}, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": {"type": "number"}, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": {"type": "string"}, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": {"type": "object"}, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": {"type": "array"}, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": {"type": "boolean"}, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": {"type": "null"}, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": {"type": ["integer", "string"]}, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json new file mode 100644 index 000000000..a1da0096e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json @@ -0,0 +1,389 @@ +[ + { + "description": "uniqueItems validation", + "schema": {"uniqueItems": true}, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { "uniqueItems": false }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/meta-schemas/README.md b/packages/schema/test/fixtures/meta-schemas/README.md new file mode 100644 index 000000000..46b120d97 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/README.md @@ -0,0 +1,10 @@ +# Meta-schemas + +The official meta-schemas some suite tests reference, unmodified: + +- `draft2020-12-schema.json`: https://json-schema.org/draft/2020-12/schema +- `draft2020-12-meta-*.json`: https://json-schema.org/draft/2020-12/meta/{core,applicator,unevaluated,validation,meta-data,format-annotation,content} +- `draft-07-schema.json`: http://json-schema.org/draft-07/schema + +Published by the JSON Schema organization (https://github.com/json-schema-org/json-schema-spec), licensed +AFL-3.0 or BSD-3-Clause. diff --git a/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json new file mode 100644 index 000000000..fb92c7f75 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json new file mode 100644 index 000000000..f4775974a --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/applicator", + "$dynamicAnchor": "meta", + + "title": "Applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "prefixItems": { "$ref": "#/$defs/schemaArray" }, + "items": { "$dynamicRef": "#meta" }, + "contains": { "$dynamicRef": "#meta" }, + "additionalProperties": { "$dynamicRef": "#meta" }, + "properties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependentSchemas": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "propertyNames": { "$dynamicRef": "#meta" }, + "if": { "$dynamicRef": "#meta" }, + "then": { "$dynamicRef": "#meta" }, + "else": { "$dynamicRef": "#meta" }, + "allOf": { "$ref": "#/$defs/schemaArray" }, + "anyOf": { "$ref": "#/$defs/schemaArray" }, + "oneOf": { "$ref": "#/$defs/schemaArray" }, + "not": { "$dynamicRef": "#meta" } + }, + "$defs": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$dynamicRef": "#meta" } + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json new file mode 100644 index 000000000..76e3760d2 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/content", + "$dynamicAnchor": "meta", + + "title": "Content vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "contentEncoding": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentSchema": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json new file mode 100644 index 000000000..691862289 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/core", + "$dynamicAnchor": "meta", + + "title": "Core vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "$id": { + "$ref": "#/$defs/uriReferenceString", + "$comment": "Non-empty fragments not allowed.", + "pattern": "^[^#]*#?$" + }, + "$schema": { "$ref": "#/$defs/uriString" }, + "$ref": { "$ref": "#/$defs/uriReferenceString" }, + "$anchor": { "$ref": "#/$defs/anchorString" }, + "$dynamicRef": { "$ref": "#/$defs/uriReferenceString" }, + "$dynamicAnchor": { "$ref": "#/$defs/anchorString" }, + "$vocabulary": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/uriString" }, + "additionalProperties": { + "type": "boolean" + } + }, + "$comment": { + "type": "string" + }, + "$defs": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" } + } + }, + "$defs": { + "anchorString": { + "type": "string", + "pattern": "^[A-Za-z_][-A-Za-z0-9._]*$" + }, + "uriString": { + "type": "string", + "format": "uri" + }, + "uriReferenceString": { + "type": "string", + "format": "uri-reference" + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json new file mode 100644 index 000000000..3479e6695 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/format-annotation", + "$dynamicAnchor": "meta", + + "title": "Format vocabulary meta-schema for annotation results", + "type": ["object", "boolean"], + "properties": { + "format": { "type": "string" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json new file mode 100644 index 000000000..4049ab21b --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/meta-data", + "$dynamicAnchor": "meta", + + "title": "Meta-data vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "deprecated": { + "type": "boolean", + "default": false + }, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json new file mode 100644 index 000000000..93779e54e --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/unevaluated", + "$dynamicAnchor": "meta", + + "title": "Unevaluated applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "unevaluatedItems": { "$dynamicRef": "#meta" }, + "unevaluatedProperties": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json new file mode 100644 index 000000000..ebb75db77 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/validation", + "$dynamicAnchor": "meta", + + "title": "Validation vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "type": { + "anyOf": [ + { "$ref": "#/$defs/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/$defs/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "const": true, + "enum": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/$defs/nonNegativeInteger" }, + "minLength": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "maxItems": { "$ref": "#/$defs/nonNegativeInteger" }, + "minItems": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxContains": { "$ref": "#/$defs/nonNegativeInteger" }, + "minContains": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 1 + }, + "maxProperties": { "$ref": "#/$defs/nonNegativeInteger" }, + "minProperties": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/$defs/stringArray" }, + "dependentRequired": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/stringArray" + } + } + }, + "$defs": { + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 0 + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json new file mode 100644 index 000000000..d5e2d31c3 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": true, + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/meta-data": true, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": true, + "https://json-schema.org/draft/2020-12/vocab/content": true + }, + "$dynamicAnchor": "meta", + + "title": "Core and Validation specifications meta-schema", + "allOf": [ + {"$ref": "meta/core"}, + {"$ref": "meta/applicator"}, + {"$ref": "meta/unevaluated"}, + {"$ref": "meta/validation"}, + {"$ref": "meta/meta-data"}, + {"$ref": "meta/format-annotation"}, + {"$ref": "meta/content"} + ], + "type": ["object", "boolean"], + "$comment": "This meta-schema also defines keywords that have appeared in previous drafts in order to prevent incompatible extensions as they remain in common use.", + "properties": { + "definitions": { + "$comment": "\"definitions\" has been replaced by \"$defs\".", + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "deprecated": true, + "default": {} + }, + "dependencies": { + "$comment": "\"dependencies\" has been split and replaced by \"dependentSchemas\" and \"dependentRequired\" in order to serve their differing semantics.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$dynamicRef": "#meta" }, + { "$ref": "meta/validation#/$defs/stringArray" } + ] + }, + "deprecated": true, + "default": {} + }, + "$recursiveAnchor": { + "$comment": "\"$recursiveAnchor\" has been replaced by \"$dynamicAnchor\".", + "$ref": "meta/core#/$defs/anchorString", + "deprecated": true + }, + "$recursiveRef": { + "$comment": "\"$recursiveRef\" has been replaced by \"$dynamicRef\".", + "$ref": "meta/core#/$defs/uriReferenceString", + "deprecated": true + } + } +} diff --git a/packages/schema/test/shared.test.ts b/packages/schema/test/shared.test.ts new file mode 100644 index 000000000..ba3e4ab30 --- /dev/null +++ b/packages/schema/test/shared.test.ts @@ -0,0 +1,42 @@ +// Values shared across compiled schemas hold nothing alive: a cached pattern goes once its schemas are collected. + +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { describe, expect, it } from "vitest"; +import { compile } from "../src/index.ts"; +import { sharedRegexCount } from "../src/shared.ts"; + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; + +async function collectGarbage(): Promise { + for (let round = 0; round < 6; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } +} + +describe("shared checks", () => { + it("share the module-level check of a bare type", () => { + const a = compile({ type: "string", description: "a" }); + const b = compile({ type: "string", title: "b" }); + expect(a.check).toBe(b.check); + }); + + it("share a pattern's regular expression and drop it with the last schema that uses it", async () => { + await collectGarbage(); + const before = sharedRegexCount(); + let validators: { check: object }[] | undefined = Array.from({ length: 50 }, (_, i) => + compile({ type: "string", pattern: `^unique-${i}$` }), + ); + const again = compile({ type: "string", pattern: "^unique-0$" }); + expect(sharedRegexCount()).toBe(before + 50); + expect(again.check("unique-0")).toBe(true); + const check = new WeakRef(validators[0].check); + validators = undefined; + await collectGarbage(); + // `again` still uses ^unique-0$; the other 49 patterns are gone. + expect(check.deref()).toBeUndefined(); + expect(sharedRegexCount()).toBe(before + 1); + }); +}); diff --git a/packages/schema/test/suite.ts b/packages/schema/test/suite.ts new file mode 100644 index 000000000..fac5bebbd --- /dev/null +++ b/packages/schema/test/suite.ts @@ -0,0 +1,77 @@ +// Loads the vendored JSON-Schema-Test-Suite: its test files, the remote documents and the official meta-schemas. + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import type { JsonSchema } from "../src/index.ts"; + +/** One test file: its path below `tests/` and its groups. */ +export interface SuiteFile { + readonly name: string; + readonly draft: "2020-12" | "7"; + readonly groups: readonly SuiteGroup[]; +} + +/** A schema with values and whether each is valid. */ +export interface SuiteGroup { + readonly description: string; + readonly schema: JsonSchema; + readonly tests: readonly { readonly description: string; readonly data: unknown; readonly valid: boolean }[]; +} + +const fixtures = join(import.meta.dirname, "fixtures"); +const root = join(fixtures, "json-schema-test-suite"); + +/** Every test file of drafts 2020-12 and 7. */ +export function suiteFiles(): SuiteFile[] { + const files: SuiteFile[] = []; + for (const [dir, draft] of [ + ["draft2020-12", "2020-12"], + ["draft7", "7"], + ] as const) { + for (const path of jsonFiles(join(root, "tests", dir))) { + const name = relative(join(root, "tests"), path).replace(/\.json$/, ""); + files.push({ name, draft, groups: JSON.parse(readFileSync(path, "utf8")) }); + } + } + return files; +} + +/** The remote documents and meta-schemas the tests reference, by URI (fresh copies). */ +export function suiteSchemas(): Record { + const schemas: Record = {}; + const remotes = join(root, "remotes"); + for (const path of jsonFiles(remotes)) { + schemas[`http://localhost:1234/${relative(remotes, path).replaceAll("\\", "/")}`] = readJson(path); + } + const meta = join(fixtures, "meta-schemas"); + schemas["http://json-schema.org/draft-07/schema"] = readJson(join(meta, "draft-07-schema.json")); + schemas["https://json-schema.org/draft/2020-12/schema"] = readJson(join(meta, "draft2020-12-schema.json")); + for (const vocabulary of [ + "core", + "applicator", + "unevaluated", + "validation", + "meta-data", + "format-annotation", + "content", + ]) { + schemas[`https://json-schema.org/draft/2020-12/meta/${vocabulary}`] = readJson( + join(meta, `draft2020-12-meta-${vocabulary}.json`), + ); + } + return schemas; +} + +function readJson(path: string): JsonSchema { + return JSON.parse(readFileSync(path, "utf8")); +} + +function jsonFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir).sort()) { + const path = join(dir, entry); + if (statSync(path).isDirectory()) out.push(...jsonFiles(path)); + else if (entry.endsWith(".json")) out.push(path); + } + return out; +} diff --git a/packages/schema/test/typebox-static.test.ts b/packages/schema/test/typebox-static.test.ts new file mode 100644 index 000000000..c62ec03f9 --- /dev/null +++ b/packages/schema/test/typebox-static.test.ts @@ -0,0 +1,72 @@ +// Static for TypeBox-built schemas: the common kinds give TypeBox's own static type, without pi-schema importing +// TypeBox; other kinds give `unknown`. Type-level tests (they pass when this file type-checks), against TypeBox 1.3.27. + +import { Type, type Static as TypeBoxOwn } from "typebox"; +import { describe, expectTypeOf, it } from "vitest"; +import { compile, type Static } from "../src/index.ts"; + +describe("Static of TypeBox schemas", () => { + it("maps the primitive kinds, literals and Unsafe", () => { + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + const any = Type.Any(); + expectTypeOf>().toBeAny(); + const literal = Type.Literal("add"); + expectTypeOf>().toEqualTypeOf<"add">(); + const unsafe = Type.Unsafe<"a" | "b">({ type: "string", enum: ["a", "b"] }); + expectTypeOf>().toEqualTypeOf<"a" | "b">(); + }); + + it("maps objects with optional and readonly properties as TypeBox does", () => { + const schema = Type.Object({ + path: Type.String(), + offset: Type.Optional(Type.Integer()), + id: Type.Readonly(Type.String()), + tag: Type.Readonly(Type.Optional(Type.Literal("x"))), + }); + type Expected = { path: string; offset?: number; readonly id: string; readonly tag?: "x" }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + const empty = Type.Object({}); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("maps arrays, unions and records, nested", () => { + const schema = Type.Object({ + edits: Type.Array(Type.Object({ oldText: Type.String(), newText: Type.String() })), + mode: Type.Union([Type.Literal("text"), Type.Literal("binary"), Type.Null()]), + env: Type.Record(Type.String(), Type.Union([Type.String(), Type.Number()])), + counts: Type.Record(Type.Integer(), Type.Boolean()), + }); + type Expected = { + edits: { oldText: string; newText: string }[]; + mode: "text" | "binary" | null; + env: Record; + counts: Record; + }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("gives unknown for kinds it does not map", () => { + const enumeration = Type.Enum(["a", "b"]); + const tuple = Type.Tuple([Type.String()]); + const intersect = Type.Intersect([Type.Object({ a: Type.String() }), Type.Object({ b: Type.String() })]); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + // Inside a mapped kind, an unmapped kind is unknown only where it stands. + const object = Type.Object({ pair: Type.Tuple([Type.String()]), name: Type.String() }); + expectTypeOf>().toEqualTypeOf<{ pair: unknown; name: string }>(); + }); + + it("types compile() and parse() for TypeBox schemas", () => { + const schema = Type.Object({ n: Type.Number(), s: Type.Optional(Type.String()) }); + const result = compile(schema, { coerce: true }).parse({ n: "1" }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ n: number; s?: string }>(); + }); +}); diff --git a/packages/schema/test/validator.test.ts b/packages/schema/test/validator.test.ts new file mode 100644 index 000000000..c5ebd6edf --- /dev/null +++ b/packages/schema/test/validator.test.ts @@ -0,0 +1,323 @@ +// compile() and Validator: checks, errors, values, options and schemas from other sources. + +import Type from "typebox"; +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +const read = { + type: "object", + properties: { path: { type: "string" }, offset: { type: "integer", minimum: 1 } }, + required: ["path"], + additionalProperties: false, +}; + +const block = { + anyOf: [ + { + type: "object", + properties: { kind: { const: "text" }, text: { type: "string" } }, + required: ["kind", "text"], + additionalProperties: false, + }, + { + type: "object", + properties: { kind: { const: "image" }, width: { type: "integer", minimum: 0 } }, + required: ["kind", "width"], + additionalProperties: false, + }, + ], +}; + +describe("errors", () => { + it("reports a missing field and an unknown field at the field", () => { + expect(compile(read).errors({ offset: 0, extra: true })).toEqual([ + { path: "/path", keyword: "required", message: "missing" }, + { path: "/offset", keyword: "minimum", message: "expected >= 1" }, + { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("is empty for a valid value", () => { + expect(compile(read).errors({ path: "a.ts" })).toEqual([]); + }); + + it("reports the branch a tagged union's tag selects", () => { + const validator = compile({ type: "array", items: block }); + expect( + validator.errors([ + { kind: "text", text: "a" }, + { kind: "image", width: -1 }, + ]), + ).toEqual([{ path: "/1/width", keyword: "minimum", message: "expected >= 0" }]); + expect(validator.errors([{ kind: "video" }])).toEqual([ + { path: "/0/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }, + ]); + }); + + it("escapes ~ and / in paths as JSON Pointer does", () => { + const validator = compile({ type: "object", properties: { "a/b~c": { type: "string" } } }); + expect(validator.errors({ "a/b~c": 1 })).toEqual([ + { path: "/a~1b~0c", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("follows $ref into the referenced schema", () => { + const validator = compile({ + $defs: { point: { type: "object", properties: { x: { type: "number" } }, required: ["x"] } }, + type: "array", + items: { $ref: "#/$defs/point" }, + }); + expect(validator.errors([{ x: 1 }, {}])).toEqual([{ path: "/1/x", keyword: "required", message: "missing" }]); + }); +}); + +describe("values", () => { + it("treats a property set to undefined as absent", () => { + const validator = compile(read); + expect(validator.check({ path: "a.ts", offset: undefined })).toBe(true); + expect(validator.check({ path: undefined })).toBe(false); + }); + + it("checks bytes as Uint8Array with length bounds", () => { + const validator = compile({ type: "object", properties: { data: { type: "bytes", maxLength: 4 } } }); + expect(validator.check({ data: new Uint8Array(4) })).toBe(true); + expect(validator.check({ data: new Uint8Array(5) })).toBe(false); + expect(validator.check({ data: [1, 2] })).toBe(false); + expect(validator.errors({ data: new Uint8Array(5) })).toEqual([ + { path: "/data", keyword: "maxLength", message: "expected at most 4 bytes" }, + ]); + }); + + it("does not treat bytes as an object", () => { + expect(compile({ type: "object" }).check(new Uint8Array(1))).toBe(false); + }); + + it("checks properties named like Object.prototype members as own properties", () => { + const validator = compile({ + type: "object", + properties: { constructor: { type: "string" }, toString: { type: "number" } }, + required: ["constructor"], + additionalProperties: false, + }); + expect(validator.check({})).toBe(false); + expect(validator.check({ constructor: "a" })).toBe(true); + expect(validator.check({ constructor: "a", toString: "b" })).toBe(false); + }); + + it("does not accept every value for a type named like an Object.prototype member", () => { + expect(compile({ type: "constructor" }).check(1)).toBe(false); + }); + + it("checks a detached check function", () => { + const { check } = compile(read); + expect([{ path: "a" }, { path: 1 }].filter(check)).toEqual([{ path: "a" }]); + }); +}); + +describe("options", () => { + it("treats format as an annotation unless formats are given", () => { + const schema = { type: "string", format: "email" }; + expect(compile(schema).check("not an email")).toBe(true); + expect(compile(schema, { formats }).check("not an email")).toBe(false); + expect(compile(schema, { formats }).check("a@example.com")).toBe(true); + }); + + it("resolves $ref to registered documents", () => { + const point = { type: "object", properties: { x: { type: "number" } }, required: ["x"] }; + const validator = compile( + { $ref: "https://example.com/point.json" }, + { schemas: { "https://example.com/point.json": point } }, + ); + expect(validator.check({ x: 1 })).toBe(true); + expect(validator.check({})).toBe(false); + }); + + it("throws for a $ref that names nothing", () => { + expect(() => compile({ $ref: "https://example.com/missing.json" })).toThrow(/Unresolved \$ref/); + expect(() => compile({ $ref: "#/$defs/missing" })).toThrow(/Unresolved \$ref/); + }); + + it("applies draft 7 semantics: $ref overrides its siblings", () => { + const schema = { definitions: { s: { type: "string" } }, $ref: "#/definitions/s", maxLength: 1 }; + expect(compile(schema, { draft: "7" }).check("ab")).toBe(true); + expect(compile(schema).check("ab")).toBe(false); + }); +}); + +describe("schemas", () => { + it("compiles TypeBox schemas as they are", () => { + const validator = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); + expect(validator.check({ name: "a", tags: [] })).toBe(true); + expect(validator.errors({ name: "a", tags: [1] })).toEqual([ + { path: "/tags/0", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("compiles deep-frozen schemas and never modifies a schema", () => { + const schema = deepFreeze(structuredClone({ ...block, $defs: { s: { type: "string", pattern: "^a" } } })); + const before = JSON.stringify(schema); + const validator = compile(schema); + expect(validator.check({ kind: "text", text: "x" })).toBe(true); + expect(validator.errors({ kind: "image", width: -1 })).toHaveLength(1); + expect(JSON.stringify(schema)).toBe(before); + }); + + it("compiles a recursive schema", () => { + const tree = { + type: "object", + properties: { value: { type: "number" }, children: { type: "array", items: { $ref: "#" } } }, + required: ["value"], + }; + const validator = compile(tree); + expect(validator.check({ value: 1, children: [{ value: 2, children: [{ value: 3 }] }] })).toBe(true); + expect(validator.errors({ value: 1, children: [{ value: 2, children: [{}] }] })).toEqual([ + { path: "/children/0/children/0/value", keyword: "required", message: "missing" }, + ]); + }); + + it("throws for a malformed schema", () => { + expect(() => compile({ properties: { a: 1 } })).toThrow(/must be an object or a boolean/); + }); +}); + +function deepFreeze(value: T): T { + if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + } + return value; +} + +describe("unresolved references", () => { + const dangling = { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }; + const external = { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }; + + it("throw when the schema compiles, by default", () => { + expect(() => compile(dangling)).toThrow("Unresolved $ref #/definitions/Missing"); + expect(() => compile(external)).toThrow("Unresolved $ref https://example.com/schemas/t.json"); + }); + + it("reject only values at their place with unresolvedRefs: reject", () => { + for (const schema of [dangling, external]) { + const validator = compile(schema, { unresolvedRefs: "reject" }); + expect(validator.check({})).toBe(true); + expect(validator.check({ t: 1 })).toBe(false); + } + expect(compile(dangling, { unresolvedRefs: "reject" }).errors({ t: 1 })).toEqual([ + { path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }, + ]); + }); + + it("leave values at their place uncoerced", () => { + const validator = compile(dangling, { unresolvedRefs: "reject", coerce: true }); + expect(validator.parse({ t: "1" })).toEqual({ + ok: false, + errors: [{ path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }], + }); + }); + + it("reject a TypeBox Ref without its definition", () => { + const validator = compile(Type.Object({ a: Type.Ref("A") }), { unresolvedRefs: "reject" }); + expect(validator.errors({ a: { x: "1" } })).toEqual([ + { path: "/a", keyword: "$ref", message: "unresolved reference A" }, + ]); + }); +}); + +describe("string lengths", () => { + it("count an unpaired high surrogate as one code point and a pair as one", () => { + expect(compile({ type: "string", minLength: 2 }).check("\ud800A")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800A")).toBe(false); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("😀")).toBe(true); + expect(compile({ type: "string", minLength: 2 }).check("😀")).toBe(false); + expect(compile({ type: "string", minLength: 2 }).check("\udc00\ud800")).toBe(true); + }); +}); + +describe("error messages", () => { + it("list at most 20 enum values", () => { + const values = Array.from({ length: 5000 }, (_, i) => `v${i}`); + const validator = compile({ type: "object", properties: { e: { type: "string", enum: values } } }); + expect(validator.check({ e: "v4999" })).toBe(true); + const [error] = validator.errors({ e: "v5000" }); + expect(error.message).toBe( + `expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 4980 more]`, + ); + expect(compile({ enum: ["a", "b"] }).errors("c")).toEqual([ + { path: "", keyword: "enum", message: 'expected one of ["a","b"]' }, + ]); + }); + + it("accept a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + expect(compile({ enum: values }).check("member9999")).toBe(true); + }); + + it("describe contains bounds in words", () => { + const at = (schema: object) => compile({ type: "array", ...schema }).errors(["a"])[0]?.message; + expect(at({ contains: { type: "number" } })).toBe("expected at least 1 matching item"); + expect(at({ contains: { type: "number" }, minContains: 2 })).toBe("expected at least 2 matching items"); + expect(at({ contains: { type: "string" }, maxContains: 0, minContains: 0 })).toBe( + "expected at most 0 matching items", + ); + expect(at({ contains: { type: "number" }, minContains: 1, maxContains: 3 })).toBe( + "expected between 1 and 3 matching items", + ); + }); + + it("blame a oneOf failure on the branch that declares most of the object's properties", () => { + const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, + }; + expect(compile(filter).errors({ where: { and: [{ field: "a", eq: 1, x: 2 }] } })).toEqual([ + { path: "/where/and/0/eq", keyword: "type", message: "expected string, got integer" }, + { path: "/where/and/0/x", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("explain one object reached through the same reference at two paths at each path", () => { + const schema = { + type: "object", + properties: { a: { $ref: "#/$defs/P" }, b: { $ref: "#/$defs/P" } }, + $defs: { P: { type: "object", properties: { n: { type: "integer" } } } }, + }; + const shared = { n: "x" }; + expect(compile(schema).errors({ a: shared, b: shared })).toEqual([ + { path: "/a/n", keyword: "type", message: "expected integer, got string" }, + { path: "/b/n", keyword: "type", message: "expected integer, got string" }, + ]); + }); +}); + +describe("TypeBox's extra formats", () => { + it("check url and json-pointer-uri-fragment", () => { + const url = compile({ type: "string", format: "url" }, { formats }); + expect(url.check("https://example.com/a?b=1")).toBe(true); + expect(url.check("not a url")).toBe(false); + const fragment = compile({ type: "string", format: "json-pointer-uri-fragment" }, { formats }); + expect(fragment.check("#/a/b~1c")).toBe(true); + expect(fragment.check("#/a b")).toBe(false); + expect(fragment.check("/a")).toBe(false); + }); +}); diff --git a/packages/schema/tsconfig.build.json b/packages/schema/tsconfig.build.json new file mode 100644 index 000000000..695dd9ad2 --- /dev/null +++ b/packages/schema/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +} diff --git a/packages/schema/vitest.config.ts b/packages/schema/vitest.config.ts new file mode 100644 index 000000000..18733b4d4 --- /dev/null +++ b/packages/schema/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"], + }, + resolve: { conditions: ["source"] }, + ssr: { resolve: { conditions: ["source"] } }, +}); diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md index b71b6c86e..a6dd650a8 100644 --- a/packages/telemetry/CHANGELOG.md +++ b/packages/telemetry/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Fixed + +- The package root and `./testing` exports have a `source` condition, like `./render`, so workspace tests that resolve with `source` (Durable's and pi-env's, as the Environments workflow runs them) work without a build. + ## [1.4.0] - 2026-10-21 ### Breaking Changes diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 92b1f04e4..eeb30b234 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -1,12 +1,13 @@ { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "description": "Vendor-neutral telemetry contracts and typed schema utilities for pi", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { ".": { + "source": "./src/index.ts", "types": "./dist/index.d.ts", "import": "./dist/index.js" }, @@ -16,6 +17,7 @@ "import": "./dist/render.js" }, "./testing": { + "source": "./src/testing/index.ts", "types": "./dist/testing/index.d.ts", "import": "./dist/testing/index.js" } @@ -25,7 +27,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index f3e85bc4f..5c7b22acf 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,10 +2,31 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Added + +- Added `replaceControlCharacters()` to replace runs of C0 and C1 control characters with a space +- Added `Component.capturesViewportInput`. While a component that sets it has focus, `TuiAltScreen` sends viewport keys such as PageUp/PageDown to it instead of scrolling the transcript, as it already does for focused overlays. `SelectList`, `SettingsList`, and `Editor` with open autocomplete set it ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Added `getSelectPageIndex()` to compute the selection after a `tui.select.pageUp` or `tui.select.pageDown` key ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) + ### Fixed +- Fixed `Shift+Enter` queueing a follow-up in Apple Terminal setups that send `ESC` + Return. When local native Shift detection is available (Apple Terminal, native Windows), `ProcessTerminal` now treats `ESC` + Return with Shift held as `Shift+Enter`; without Shift it remains `Alt+Enter` ([#11559](https://github.com/earendil-works/pi/issues/11559)) +- Fixed regular-mode `TUI` writing OSC 133 prompt markers from rendered lines to the terminal. Terminals treat `133;A` as a new shell prompt and drop the OSC 7501 program status, so the working indicator disappeared during runs. Fullscreen still uses the markers for prompt navigation and already stripped them ([#11519](https://github.com/earendil-works/pi/issues/11519)) - Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`. Word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) - Fixed `fuzzyMatch()` ranking a query scattered over an earlier part of the text above a contiguous match later in it. It now also scores each contiguous occurrence of the query and keeps the best alignment ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) +- Fixed `ProcessTerminal.setTitle()` writing control characters from the title, which could end the title sequence and send the rest to the terminal as escape sequences +- Fixed display math wider than the content width being wrapped row by row, which split fractions and sums apart. It now falls back to the raw LaTeX source ([#11483](https://github.com/earendil-works/pi/issues/11483)) +- Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored by `SelectList`, `SettingsList`, and editor autocomplete ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Fixed `wrapTextWithAnsi()` starting the next line with a space when a styled word ended exactly at the wrap width, and strikethrough continuing into the right padding of wrapped lines ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `Markdown` styling, highlighting, and wrapping every block on each render. It now reuses the top-level blocks of its last render, so a streaming message renders only its changed last block, and a resize reuses highlighted code. Reused blocks are held weakly and dropped by `invalidate()`, which a theme change must call because `MarkdownTheme.highlightCode` results are reused until then ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed `Text.setText()` with unchanged text dropping the wrapped lines, so renderers that set the same text on every update wrapped it again ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/README.md b/packages/tui/README.md index 1150ebcf5..6ed8a26b3 100644 --- a/packages/tui/README.md +++ b/packages/tui/README.md @@ -251,6 +251,7 @@ interface Component { render(width: number): string[]; handleInput?(data: string): void; handleMouse?(event: TuiMouseEvent): TuiMouseEventResult | undefined; + capturesViewportInput?: boolean; invalidate(): void; } ``` @@ -260,6 +261,7 @@ interface Component { | `render(width)` | Returns an array of strings, one per line. Each line **must not exceed `width`** or the TUI will error. Use `truncateToWidth()` or manual wrapping to ensure this. | | `handleInput?(data)` | Called when the component has focus and receives keyboard input. The `data` string contains raw terminal input (may include ANSI escape sequences). | | `handleMouse?(event)` | Called by `TuiAltScreen` for normalized pointer input targeted at the component. | +| `capturesViewportInput?` | If true while the component has focus, `TuiAltScreen` passes its viewport scrolling and prompt-jump keys (`tui.altScreen.*` except search) to it instead of scrolling the viewport, as it does for a focused overlay. Set by `SelectList`, `SettingsList`, and `Editor` while autocomplete is open, so PageUp/PageDown move the selection. Only the focused component is checked: a wrapper that forwards input to a list must expose the list's value itself. Use `getSelectPageIndex(keybindings, data, selectedIndex, itemCount, pageSize)` to compute the new selection. | | `invalidate()` | Required. Clear any cached render state so the next `render()` starts from scratch. Components without cached render state can use an empty implementation. | The TUI appends a full SGR reset and OSC 8 reset at the end of each rendered line. Styles do not carry across lines. If you emit multi-line text with styling, reapply styles per line or use `wrapTextWithAnsi()` so styles are preserved for each wrapped line. diff --git a/packages/tui/package.json b/packages/tui/package.json index 31dc4744a..74a49f0d3 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,11 +1,11 @@ { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "type": "module", "main": "dist/index.js", "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:native:darwin": "native/darwin/build.sh", "build:native:linux": "native/linux/build.sh", diff --git a/packages/tui/src/components/editor.ts b/packages/tui/src/components/editor.ts index 5d453e3cd..9fd3677e9 100644 --- a/packages/tui/src/components/editor.ts +++ b/packages/tui/src/components/editor.ts @@ -304,6 +304,11 @@ export class Editor implements Component, Focusable { /** Focusable interface - set by TUI when focus changes */ focused: boolean = false; + /** The open autocomplete menu receives fullscreen viewport keys, such as PageUp/PageDown. */ + get capturesViewportInput(): boolean { + return this.autocompleteState !== null; + } + protected tui: TUI; private theme: EditorTheme; private paddingX: number = 0; @@ -761,7 +766,12 @@ export class Editor implements Component, Focusable { return; } - if (kb.matches(data, "tui.select.up") || kb.matches(data, "tui.select.down")) { + if ( + kb.matches(data, "tui.select.up") || + kb.matches(data, "tui.select.down") || + kb.matches(data, "tui.select.pageUp") || + kb.matches(data, "tui.select.pageDown") + ) { this.autocompleteList.handleInput(data); return; } diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index 9194c2251..a21f75181 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -1,13 +1,23 @@ -import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens } from "marked"; +import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens, type TokensList } from "marked"; import markedCjkFriendly from "marked-cjk-friendly"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; import type { Component } from "../tui.ts"; -import { applyBackgroundToLine, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; +import { applyBackgroundToLine, cjkPunctuationRegex, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; const STRICT_STRIKETHROUGH_REGEX = /^(~~)(?=[^\s~])((?:\\.|[^\\])*?(?:\\.|[^\s~\\]))\1(?=[^~]|$)/; -class StrictStrikethroughTokenizer extends Tokenizer { +class MarkdownTokenizer extends Tokenizer { + override url(src: string): Tokens.Link | undefined { + const token = super.url(src); + if (!token) return undefined; + + // Bare URLs stop at CJK punctuation; explicit links and angle autolinks + // use separate tokenizers and can still contain these characters. + const punctuationIndex = token.raw.search(cjkPunctuationRegex); + return punctuationIndex < 0 ? token : super.url(token.raw.slice(0, punctuationIndex)); + } + override del(src: string): Tokens.Del | undefined { const match = STRICT_STRIKETHROUGH_REGEX.exec(src); if (!match) { @@ -189,7 +199,7 @@ function trimPartialClosingFences(tokens: readonly Token[]): void { const markdownParser = new Marked(); markdownParser.setOptions({ - tokenizer: new StrictStrikethroughTokenizer(), + tokenizer: new MarkdownTokenizer(), }); markdownParser.use({ tokenizer: { emStrong: markedCjkFriendly().tokenizer?.emStrong }, @@ -234,6 +244,7 @@ export interface MarkdownTheme { italic: (text: string) => string; strikethrough: (text: string) => string; underline: (text: string) => string; + /** Highlighted lines of a code block. A Markdown component reuses the result for the same code until invalidate(). */ highlightCode?: (code: string, lang?: string) => string[]; /** Prefix applied to each rendered code block line (default: " ") */ codeBlockIndent?: string; @@ -255,6 +266,16 @@ interface InlineStyleContext { stylePrefix: string; } +/** A top-level block of the last render, keyed by its source, reused by the next render. */ +interface RenderedBlock { + /** Width and type of the following block (which decides the spacing) that the lines were rendered for. */ + width: number; + nextTokenType: string | undefined; + lines: string[]; + /** Highlighted lines of the block's code, keyed by language and source; they survive a resize or new spacing. */ + highlightedCode: Map; +} + export class Markdown implements Component { private text: string; private paddingX: number; // Left/right padding @@ -271,7 +292,14 @@ export class Markdown implements Component { // Parsed tokens depend only on the source, so they survive theme and width invalidation. Held weakly: a token tree is // about ten times the size of its source, and every message of a long transcript keeps a Markdown component. The // tokens survive a burst of re-renders, such as a theme preview, and are collected afterwards. - private cachedTokens?: WeakRef<{ source: string; tokens: Token[] }>; + private cachedTokens?: WeakRef<{ source: string; tokens: TokensList }>; + // The last render's top-level blocks. While a message streams only its last block changes, so a render styles and + // wraps only that block. Each render keeps only the blocks it used, which drops a streamed block's earlier versions. + // Held weakly like the tokens: the blocks repeat the source in their keys and the code in their highlighting, and + // only a burst of renders, such as streaming or a resize, needs them. + private blocks?: WeakRef>; + // Highlighted code of the block being rendered, only set while renderBlock runs. + private blockHighlightedCode?: Map; constructor( text: string, @@ -291,10 +319,17 @@ export class Markdown implements Component { setText(text: string): void { this.text = text; - this.invalidate(); + this.clearRenderCache(); } invalidate(): void { + this.clearRenderCache(); + // The theme may have changed, which changes every styled line and the default style's prefix. + this.blocks = undefined; + this.defaultStylePrefix = undefined; + } + + private clearRenderCache(): void { this.cachedText = undefined; this.cachedWidth = undefined; this.cachedLines = undefined; @@ -312,6 +347,7 @@ export class Markdown implements Component { // Don't render anything if there's no actual text if (!text || text.trim() === "") { + this.blocks = undefined; const result: string[] = []; // Update cache this.cachedText = this.text; @@ -332,65 +368,39 @@ export class Markdown implements Component { this.cachedTokens = new WeakRef({ source: normalizedText, tokens }); } - // Convert tokens to styled terminal output - const renderedLines: string[] = []; - + // A reference link resolves through a definition anywhere in the text, so blocks with the same source can render + // differently; such texts are rare in practice and render without reusing blocks. + const reuseBlocks = Object.keys(tokens.links).length === 0; + const previousBlocks = this.blocks?.deref(); + const blocks = new Map(); + this.blocks = new WeakRef(blocks); + const contentLines: string[] = []; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; - const nextToken = tokens[i + 1]; - const tokenLines = this.renderToken(token, contentWidth, nextToken?.type); - for (const tokenLine of tokenLines) { - renderedLines.push(tokenLine); + const nextTokenType = tokens[i + 1]?.type; + const reused = reuseBlocks ? (blocks.get(token.raw) ?? previousBlocks?.get(token.raw)) : undefined; + const block = reused ?? { width: -1, nextTokenType, lines: [], highlightedCode: new Map() }; + if (block.width !== width || block.nextTokenType !== nextTokenType) { + this.renderBlock(block, token, nextTokenType, width, contentWidth); } - } - - // Wrap lines (NO padding, NO background yet) - const wrappedLines: string[] = []; - for (const line of renderedLines) { - if (isImageLine(line)) { - wrappedLines.push(line); - } else { - for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { - wrappedLines.push(wrappedLine); - } - } - } - - // Add margins and background to each wrapped line - const leftMargin = " ".repeat(this.paddingX); - const rightMargin = " ".repeat(this.paddingX); - const bgFn = this.defaultTextStyle?.bgColor; - const contentLines: string[] = []; - - for (const line of wrappedLines) { - if (isImageLine(line)) { + if (reuseBlocks) blocks.set(token.raw, block); + for (const line of block.lines) { contentLines.push(line); - continue; - } - - const lineWithMargins = leftMargin + line + rightMargin; - - if (bgFn) { - contentLines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); - } else { - // No background - just pad to width - const visibleLen = visibleWidth(lineWithMargins); - const paddingNeeded = Math.max(0, width - visibleLen); - contentLines.push(lineWithMargins + " ".repeat(paddingNeeded)); } } // Add top/bottom padding (empty lines) + const bgFn = this.defaultTextStyle?.bgColor; const emptyLine = " ".repeat(width); const emptyLines: string[] = []; for (let i = 0; i < this.paddingY; i++) { const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; emptyLines.push(line); } + flattenLines(emptyLines); // Combine top padding, content, and bottom padding const result = emptyLines.concat(contentLines, emptyLines); - flattenLines(result); // Update cache this.cachedText = this.text; @@ -400,6 +410,42 @@ export class Markdown implements Component { return result.length > 0 ? result : [""]; } + /** Render one top-level block to output lines: wrapped, with margins, and padded or backgrounded to the width. */ + private renderBlock( + block: RenderedBlock, + token: Token, + nextTokenType: string | undefined, + width: number, + contentWidth: number, + ): void { + this.blockHighlightedCode = block.highlightedCode; + const leftMargin = " ".repeat(this.paddingX); + const rightMargin = " ".repeat(this.paddingX); + const bgFn = this.defaultTextStyle?.bgColor; + const lines: string[] = []; + for (const line of this.renderToken(token, contentWidth, nextTokenType)) { + if (isImageLine(line)) { + lines.push(line); + continue; + } + for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { + const lineWithMargins = leftMargin + wrappedLine + rightMargin; + if (bgFn) { + lines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); + } else { + // No background - just pad to width + const paddingNeeded = Math.max(0, width - visibleWidth(lineWithMargins)); + lines.push(lineWithMargins + " ".repeat(paddingNeeded)); + } + } + } + this.blockHighlightedCode = undefined; + flattenLines(lines); + block.lines = lines; + block.width = width; + block.nextTokenType = nextTokenType; + } + /** * Apply default text style to a string. * This is the base styling applied to all text content. @@ -536,10 +582,14 @@ export class Markdown implements Component { case "latexBlock": { const latexToken = token as LatexToken; - const rendered = + let rendered = !latexToken.pending && this.options.renderLatex !== false - ? (renderLatex(latexToken.text, { display: true }) ?? latexToken.raw.trim()) - : latexToken.raw.trim(); + ? renderLatex(latexToken.text, { display: true }) + : undefined; + // A 2D layout cannot be wrapped row by row; fall back to the raw source when it does not fit. + if (rendered === undefined || rendered.split("\n").some((line) => visibleWidth(line) > width)) { + rendered = latexToken.raw.trim(); + } for (const line of rendered.split("\n")) { lines.push(this.applyDefaultStyle(line)); } @@ -553,8 +603,13 @@ export class Markdown implements Component { const indent = this.theme.codeBlockIndent ?? " "; lines.push(this.theme.codeBlockBorder(`\`\`\`${token.lang || ""}`)); if (this.theme.highlightCode) { - const highlightedLines = this.theme.highlightCode(token.text, token.lang); - for (const hlLine of highlightedLines) { + const key = `${token.lang ?? ""}\0${token.text}`; + let highlighted = this.blockHighlightedCode?.get(key); + if (!highlighted) { + highlighted = this.theme.highlightCode(token.text, token.lang); + this.blockHighlightedCode?.set(key, highlighted); + } + for (const hlLine of highlighted) { lines.push(`${indent}${hlLine}`); } } else { diff --git a/packages/tui/src/components/select-list.ts b/packages/tui/src/components/select-list.ts index f4f4354dd..c196ab4ab 100644 --- a/packages/tui/src/components/select-list.ts +++ b/packages/tui/src/components/select-list.ts @@ -1,4 +1,4 @@ -import { getKeybindings } from "../keybindings.ts"; +import { getKeybindings, getSelectPageIndex } from "../keybindings.ts"; import type { Component, TuiMouseEvent, TuiMouseEventResult } from "../tui.ts"; import { truncateToWidth, visibleWidth } from "../utils.ts"; @@ -38,6 +38,7 @@ export interface SelectListLayoutOptions { } export class SelectList implements Component { + readonly capturesViewportInput: boolean = true; private items: SelectItem[] = []; private filteredItems: SelectItem[] = []; private selectedIndex: number = 0; @@ -144,6 +145,7 @@ export class SelectList implements Component { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredItems.length, this.maxVisible); // Up arrow - wrap to bottom when at top if (kb.matches(keyData, "tui.select.up")) { this.selectedIndex = this.selectedIndex === 0 ? this.filteredItems.length - 1 : this.selectedIndex - 1; @@ -154,6 +156,12 @@ export class SelectList implements Component { this.selectedIndex = this.selectedIndex === this.filteredItems.length - 1 ? 0 : this.selectedIndex + 1; this.notifySelectionChange(); } + // Page up/down - move by one visible page, clamped to the list bounds + else if (pageIndex !== undefined) { + if (this.filteredItems.length === 0 || pageIndex === this.selectedIndex) return; + this.selectedIndex = pageIndex; + this.notifySelectionChange(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedItem = this.filteredItems[this.selectedIndex]; diff --git a/packages/tui/src/components/settings-list.ts b/packages/tui/src/components/settings-list.ts index b1b8366bf..b5e4d1b8c 100644 --- a/packages/tui/src/components/settings-list.ts +++ b/packages/tui/src/components/settings-list.ts @@ -1,5 +1,5 @@ import { fuzzyFilter } from "../fuzzy.ts"; -import { getKeybindings } from "../keybindings.ts"; +import { getKeybindings, getSelectPageIndex } from "../keybindings.ts"; import type { Component, TuiMouseEvent, TuiMouseEventResult } from "../tui.ts"; import { truncateToWidth, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; import { Input } from "./input.ts"; @@ -36,6 +36,7 @@ export interface SettingsListOptions { } export class SettingsList implements Component { + readonly capturesViewportInput: boolean = true; private items: SettingItem[]; private filteredItems: SettingItem[]; private theme: SettingsListTheme; @@ -230,12 +231,16 @@ export class SettingsList implements Component { // Main list input handling const kb = getKeybindings(); const displayItems = this.getDisplayItems(); + const pageIndex = getSelectPageIndex(kb, data, this.selectedIndex, displayItems.length, this.maxVisible); if (kb.matches(data, "tui.select.up")) { if (displayItems.length === 0) return; this.selectedIndex = this.selectedIndex === 0 ? displayItems.length - 1 : this.selectedIndex - 1; } else if (kb.matches(data, "tui.select.down")) { if (displayItems.length === 0) return; this.selectedIndex = this.selectedIndex === displayItems.length - 1 ? 0 : this.selectedIndex + 1; + } else if (pageIndex !== undefined) { + if (displayItems.length === 0) return; + this.selectedIndex = pageIndex; } else if ( kb.matches(data, "tui.select.confirm") || (data === " " && (!this.searchEnabled || this.searchInput?.getValue().length === 0)) diff --git a/packages/tui/src/components/text.ts b/packages/tui/src/components/text.ts index 024748912..c9aea3d13 100644 --- a/packages/tui/src/components/text.ts +++ b/packages/tui/src/components/text.ts @@ -23,6 +23,8 @@ export class Text implements Component { } setText(text: string): void { + // Renderers often set the same text on every update; keep the wrapped lines then. + if (text === this.text) return; this.text = text; this.cachedText = undefined; this.cachedWidth = undefined; diff --git a/packages/tui/src/index.ts b/packages/tui/src/index.ts index eb962609f..8b4f9ea4a 100644 --- a/packages/tui/src/index.ts +++ b/packages/tui/src/index.ts @@ -83,6 +83,7 @@ export { type FuzzyMatch, fuzzyFilter, fuzzyMatch } from "./fuzzy.ts"; // Keybindings export { getKeybindings, + getSelectPageIndex, type Keybinding, type KeybindingConflict, type KeybindingDefinition, @@ -186,6 +187,7 @@ export { TuiMainScreen, type TuiMainScreenRenderState } from "./tui-main-screen. // Utilities export { getOsc8LinkAtColumn, + replaceControlCharacters, sliceByColumn, stripTerminalSequences, truncateToWidth, diff --git a/packages/tui/src/keybindings.ts b/packages/tui/src/keybindings.ts index ed6e95171..652399f4b 100644 --- a/packages/tui/src/keybindings.ts +++ b/packages/tui/src/keybindings.ts @@ -318,3 +318,22 @@ export function getKeybindings(): KeybindingsManager { } return globalKeybindings; } + +/** + * Returns the selection index after a `tui.select.pageUp` or `tui.select.pageDown` key, moved by `pageSize` and + * clamped to `0..itemCount - 1`. Returns undefined when `data` is not a selection page key, and 0 for a page key on + * an empty list, so callers must still check that the list has items before selecting. + */ +export function getSelectPageIndex( + keybindings: KeybindingsManager, + data: string, + selectedIndex: number, + itemCount: number, + pageSize: number, +): number | undefined { + let direction: number; + if (keybindings.matches(data, "tui.select.pageUp")) direction = -1; + else if (keybindings.matches(data, "tui.select.pageDown")) direction = 1; + else return undefined; + return Math.max(0, Math.min(itemCount - 1, selectedIndex + direction * pageSize)); +} diff --git a/packages/tui/src/keys.ts b/packages/tui/src/keys.ts index 734fbfae5..a3ce39726 100644 --- a/packages/tui/src/keys.ts +++ b/packages/tui/src/keys.ts @@ -325,8 +325,25 @@ const FUNCTIONAL_CODEPOINTS = { pageDown: -13, home: -14, end: -15, + f1: -21, + f2: -22, + f3: -23, + f4: -24, + f5: -25, + f6: -26, + f7: -27, + f8: -28, + f9: -29, + f10: -30, + f11: -31, + f12: -32, } as const; +// Kitty reserves this Private Use Area range for functional keys (F13-F35, +// media keys, modifier keys, ...). CSI-u codepoints in it are never text. +const KITTY_FUNCTIONAL_PUA_START = 57344; +const KITTY_FUNCTIONAL_PUA_END = 63743; + const KITTY_FUNCTIONAL_KEY_EQUIVALENTS = new Map([ [57399, 48], // KP_0 -> 0 [57400, 49], // KP_1 -> 1 @@ -558,7 +575,10 @@ export function isKeyRelease(data: string): boolean { data.includes(":3C") || data.includes(":3D") || data.includes(":3H") || - data.includes(":3F") + data.includes(":3F") || + data.includes(":3P") || + data.includes(":3Q") || + data.includes(":3S") ) { return true; } @@ -584,7 +604,10 @@ export function isKeyRepeat(data: string): boolean { data.includes(":2C") || data.includes(":2D") || data.includes(":2H") || - data.includes(":2F") + data.includes(":2F") || + data.includes(":2P") || + data.includes(":2Q") || + data.includes(":2S") ) { return true; } @@ -644,6 +667,18 @@ function parseKittySequence(data: string): ParsedKittySequence | null { 6: FUNCTIONAL_CODEPOINTS.pageDown, 7: FUNCTIONAL_CODEPOINTS.home, 8: FUNCTIONAL_CODEPOINTS.end, + 11: FUNCTIONAL_CODEPOINTS.f1, + 12: FUNCTIONAL_CODEPOINTS.f2, + 13: FUNCTIONAL_CODEPOINTS.f3, + 14: FUNCTIONAL_CODEPOINTS.f4, + 15: FUNCTIONAL_CODEPOINTS.f5, + 17: FUNCTIONAL_CODEPOINTS.f6, + 18: FUNCTIONAL_CODEPOINTS.f7, + 19: FUNCTIONAL_CODEPOINTS.f8, + 20: FUNCTIONAL_CODEPOINTS.f9, + 21: FUNCTIONAL_CODEPOINTS.f10, + 23: FUNCTIONAL_CODEPOINTS.f11, + 24: FUNCTIONAL_CODEPOINTS.f12, }; const codepoint = funcCodes[keyNum]; if (codepoint !== undefined) { @@ -662,6 +697,21 @@ function parseKittySequence(data: string): ParsedKittySequence | null { return { codepoint, modifier: modValue - 1, eventType }; } + // F1, F2, F4: \x1b[P/Q/S or \x1b[1;P/Q/S or \x1b[1;:P/Q/S + // F3 is sent as \x1b[13~ because \x1b[1;R collides with cursor position reports. + const fKeyMatch = data.match(/^\x1b\[(?:1;(\d+)(?::(\d+))?)?([PQS])$/); + if (fKeyMatch) { + const modValue = fKeyMatch[1] ? parseInt(fKeyMatch[1], 10) : 1; + const eventType = parseEventType(fKeyMatch[2]); + const fKeyCodes: Record = { + P: FUNCTIONAL_CODEPOINTS.f1, + Q: FUNCTIONAL_CODEPOINTS.f2, + S: FUNCTIONAL_CODEPOINTS.f4, + }; + _lastEventType = eventType; + return { codepoint: fKeyCodes[fKeyMatch[3]!]!, modifier: modValue - 1, eventType }; + } + return null; } @@ -1176,11 +1226,11 @@ export function matchesKey(data: string, keyId: KeyId): boolean { case "f10": case "f11": case "f12": { - if (modifier !== 0) { - return false; + const functionKey = key as keyof typeof LEGACY_KEY_SEQUENCES & keyof typeof FUNCTIONAL_CODEPOINTS; + if (modifier === 0 && matchesLegacySequence(data, LEGACY_KEY_SEQUENCES[functionKey])) { + return true; } - const functionKey = key as keyof typeof LEGACY_KEY_SEQUENCES; - return matchesLegacySequence(data, LEGACY_KEY_SEQUENCES[functionKey]); + return matchesKittySequence(data, FUNCTIONAL_CODEPOINTS[functionKey], modifier); } } @@ -1277,6 +1327,8 @@ function formatParsedKey(codepoint: number, modifier: number, baseLayoutKey?: nu else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.end) keyName = "end"; else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.pageUp) keyName = "pageUp"; else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.pageDown) keyName = "pageDown"; + else if (effectiveCodepoint <= FUNCTIONAL_CODEPOINTS.f1 && effectiveCodepoint >= FUNCTIONAL_CODEPOINTS.f12) + keyName = `f${FUNCTIONAL_CODEPOINTS.f1 - effectiveCodepoint + 1}`; else if (effectiveCodepoint === ARROW_CODEPOINTS.up) keyName = "up"; else if (effectiveCodepoint === ARROW_CODEPOINTS.down) keyName = "down"; else if (effectiveCodepoint === ARROW_CODEPOINTS.left) keyName = "left"; @@ -1428,6 +1480,10 @@ export function decodeKittyPrintable(data: string): string | undefined { effectiveCodepoint = normalizeKittyFunctionalCodepoint(effectiveCodepoint); // Drop control characters or invalid codepoints. if (!Number.isFinite(effectiveCodepoint) || effectiveCodepoint < 32) return undefined; + // Drop unmapped Kitty functional keys (F13-F35, media keys, ...). + if (effectiveCodepoint >= KITTY_FUNCTIONAL_PUA_START && effectiveCodepoint <= KITTY_FUNCTIONAL_PUA_END) { + return undefined; + } try { return String.fromCodePoint(effectiveCodepoint); diff --git a/packages/tui/src/program-status.ts b/packages/tui/src/program-status.ts index 3b71e268d..919b6fc48 100644 --- a/packages/tui/src/program-status.ts +++ b/packages/tui/src/program-status.ts @@ -5,6 +5,8 @@ * Spec: https://www.superlogical.com/rex/docs/build/program-status */ +import { replaceControlCharacters } from "./utils.ts"; + export interface ProgramStatus { /** `clear` removes the status instead of reporting one. */ state: "idle" | "working" | "blocked" | "done" | "error" | "clear"; @@ -25,7 +27,6 @@ export function isProgramStatusReply(sequence: string): boolean { } const APP_PATTERN = /^[A-Za-z0-9_.+-]{1,32}$/; -const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]+/g; /** Decoded `msg` limit. Its base64 encoding stays under the 2732-byte encoded limit. */ const MAX_MESSAGE_BYTES = 2048; @@ -47,7 +48,7 @@ export function formatProgramStatus(status: ProgramStatus): string { const pairs = [`state=${status.state}`]; if (status.app !== undefined && APP_PATTERN.test(status.app)) pairs.push(`app=${status.app}`); if (status.state === "blocked" && status.kind) pairs.push(`kind=${status.kind}`); - const message = truncateUtf8((status.message ?? "").replace(CONTROL_CHARACTERS, " ").trim(), MAX_MESSAGE_BYTES); + const message = truncateUtf8(replaceControlCharacters(status.message ?? "").trim(), MAX_MESSAGE_BYTES); if (message) pairs.push(`msg=${Buffer.from(message, "utf8").toString("base64")}`); return `\x1b]7501;${pairs.join(":")}\x1b\\`; } diff --git a/packages/tui/src/terminal-image.ts b/packages/tui/src/terminal-image.ts index 201a40603..64703bf48 100644 --- a/packages/tui/src/terminal-image.ts +++ b/packages/tui/src/terminal-image.ts @@ -96,6 +96,13 @@ function detectCapabilitiesFromEnvironment(tmuxForwardsHyperlink: () => boolean) return { images: "kitty", trueColor: true, hyperlinks: true }; } + // Orca supports Kitty graphics but only advertises it through ORCA_IMAGE_PROTOCOL. Require + // both signals so Orca releases predating that advertisement keep the text fallback instead + // of reserving image rows that nothing renders. + if (termProgram === "orca" && process.env.ORCA_IMAGE_PROTOCOL === "kitty") { + return { images: "kitty", trueColor: true, hyperlinks: true }; + } + if (process.env.KITTY_WINDOW_ID || termProgram === "kitty") { return { images: "kitty", trueColor: true, hyperlinks: true }; } diff --git a/packages/tui/src/terminal.ts b/packages/tui/src/terminal.ts index 80d62b3e4..55dab58d5 100644 --- a/packages/tui/src/terminal.ts +++ b/packages/tui/src/terminal.ts @@ -10,6 +10,7 @@ import { type ProgramStatus, } from "./program-status.ts"; import { StdinBuffer } from "./stdin-buffer.ts"; +import { replaceControlCharacters } from "./utils.ts"; const TERMINAL_PROGRESS_KEEPALIVE_MS = 1000; const TERMINAL_PROGRESS_ACTIVE_SEQUENCE = "\x1b]9;4;3\x07"; @@ -59,12 +60,19 @@ export function refreshTerminalDimensions(): void { } } +// Some Apple Terminal setups send ESC+CR (Alt+Enter) for Shift+Enter (#11559). +function isNativeShiftEnterCandidate(data: string): boolean { + return data === "\r" || data === "\x1b\r"; +} + export function normalizeNativeShiftEnterInput( data: string, shouldDetectNativeShiftEnter: boolean, isShiftPressed: boolean, ): string { - if (shouldDetectNativeShiftEnter && data === "\r" && isShiftPressed) return NATIVE_SHIFT_ENTER_SEQUENCE; + if (shouldDetectNativeShiftEnter && isNativeShiftEnterCandidate(data) && isShiftPressed) { + return NATIVE_SHIFT_ENTER_SEQUENCE; + } return data; } @@ -396,7 +404,7 @@ export class ProcessTerminal implements Terminal { private forwardInputSequence(sequence: string): void { if (!this.inputHandler) return; const shouldDetectNativeShiftEnter = - sequence === "\r" && (isAppleTerminalSession() || process.platform === "win32"); + isNativeShiftEnterCandidate(sequence) && (isAppleTerminalSession() || process.platform === "win32"); const input = normalizeNativeShiftEnterInput( sequence, shouldDetectNativeShiftEnter, @@ -590,8 +598,9 @@ export class ProcessTerminal implements Terminal { } setTitle(title: string): void { - // OSC 0;title BEL - set terminal window title - process.stdout.write(`\x1b]0;${title}\x07`); + // OSC 0;title BEL - set terminal window title. A BEL or ESC in the title would end the sequence early and + // send the rest to the terminal as escape sequences (session names come from session files). + process.stdout.write(`\x1b]0;${replaceControlCharacters(title)}\x07`); } setProgramStatus(status: ProgramStatus): void { diff --git a/packages/tui/src/tui-alt-screen.ts b/packages/tui/src/tui-alt-screen.ts index 9596146a6..61d5b4825 100644 --- a/packages/tui/src/tui-alt-screen.ts +++ b/packages/tui/src/tui-alt-screen.ts @@ -678,7 +678,9 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { this.flashes.flash(message, durationMs); } - private shouldDeferViewportInputToOverlay(): boolean { + /** Focused overlays and components that capture viewport input receive viewport keys instead of the transcript. */ + private shouldDeferViewportInput(): boolean { + if (this.getFocusedComponent()?.capturesViewportInput) return true; return this.isOverlayFocused() && this.activeSearch?.overlay?.isFocused() !== true; } @@ -757,7 +759,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { return { consume: true }; } } - if (this.shouldDeferViewportInputToOverlay()) return undefined; + if (this.shouldDeferViewportInput()) return undefined; if (keybindings.matches(data, "tui.altScreen.pageUp")) { if (!isRelease) { this.scrollBy(-Math.max(1, this.getPrimaryScrollView().viewportHeight - PAGE_SCROLL_OVERLAP)); diff --git a/packages/tui/src/tui-main-screen.ts b/packages/tui/src/tui-main-screen.ts index c497acdbf..71229fcb5 100644 --- a/packages/tui/src/tui-main-screen.ts +++ b/packages/tui/src/tui-main-screen.ts @@ -5,6 +5,9 @@ import { deleteKittyImage, isImageLine } from "./terminal-image.ts"; import { type TUI, TuiBase, type TuiStopOptions } from "./tui.ts"; import { visibleWidth } from "./utils.ts"; +/** OSC 133 prompt markers at the start of a rendered line. Fullscreen uses them for prompt navigation. */ +const OSC133_ZONE_PREFIX = /^(?:\x1b\]133;[ABC](?:\x07|\x1b\\))+/; + const KITTY_SEQUENCE_PREFIX = "\x1b_G"; const MAX_RENDER_WRITE_CHARS = 1024 * 1024; @@ -262,7 +265,8 @@ export class TuiMainScreen extends TuiBase implements TUI { // Render all components to get new lines // Resolve fake cursors before compositing so overlays and line slicing see plain SGR codes - let newLines = this.resolveFakeCursors(this.render(width)); + // Drop OSC 133 markers: the terminal reads 133;A as a new shell prompt, which ends program status (OSC 7501) + let newLines = this.resolveFakeCursors(this.render(width).map((line) => line.replace(OSC133_ZONE_PREFIX, ""))); // Composite overlays into the rendered lines (before differential compare) if (this.hasOverlayEntries) { diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index d888940e9..ad11a36c5 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -134,6 +134,12 @@ export interface Component { */ wantsKeyRelease?: boolean; + /** + * If true while focused, fullscreen viewport keys (`tui.altScreen.*` scrolling and prompt jumps) go to this + * component instead of scrolling the transcript, as they do for a focused overlay. Default is false. + */ + capturesViewportInput?: boolean; + /** * Invalidate any cached rendering state. * Called when theme changes or when component needs to re-render from scratch. diff --git a/packages/tui/src/utils.ts b/packages/tui/src/utils.ts index 272366899..2a84d830d 100644 --- a/packages/tui/src/utils.ts +++ b/packages/tui/src/utils.ts @@ -326,6 +326,56 @@ export function stripTerminalSequences(str: string): string { return result; } +function extractAnsiCodes(str: string): string { + let result = ""; + let i = str.indexOf("\x1b"); + while (i !== -1) { + const length = ansiCodeLength(str, i); + if (length > 0) { + result += str.substring(i, i + length); + i += length; + } else { + i++; + } + i = str.indexOf("\x1b", i); + } + return result; +} + +// Same character set as String.prototype.trim() +const WHITESPACE_REGEX = /\s/; + +/** + * Whether a token's visible text is only whitespace. Codes closing a styled word attach to the following space, + * e.g. "\x1b[22m ", so they are skipped. Stops at the first visible non-whitespace text without allocating a + * stripped copy, because styled words reach this check on every wrap. + */ +function isWhitespaceToken(token: string): boolean { + let i = 0; + while (i < token.length) { + const code = token.charCodeAt(i); + if (code === 0x1b) { + const length = ansiCodeLength(token, i); + if (length === 0) return false; // A lone ESC is visible text + i += length; + } else if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) { + i++; + } else if (code < 0x80 || !WHITESPACE_REGEX.test(token[i]!)) { + return false; + } else { + i++; + } + } + return true; +} + +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]+/g; + +/** Replace each run of C0 and C1 control characters (ESC, BEL, newlines, ...) with one space. */ +export function replaceControlCharacters(str: string): string { + return str.replace(CONTROL_CHARACTERS, " "); +} + interface GraphemeCellRange { start: number; end: number; @@ -752,7 +802,7 @@ class AnsiCodeTracker { /** * Get reset codes for attributes that need to be turned off at line end. - * Underline must be closed to prevent bleeding into padding. + * Underline and strikethrough must be closed to prevent bleeding into padding. * Active OSC 8 hyperlinks must be closed and re-opened on the next line. * Returns empty string if no attributes need closing. */ @@ -761,6 +811,9 @@ class AnsiCodeTracker { if (this.underline) { result += "\x1b[24m"; // Underline off only } + if (this.strikethrough) { + result += "\x1b[29m"; // Strikethrough off only, re-opened at line start via getActiveCodes() + } if (this.activeHyperlink) { result += formatOsc8Close(this.activeHyperlink.terminator); // Re-opened at line start via getActiveCodes() } @@ -941,11 +994,13 @@ function wrapSingleLine(line: string, width: number): string[] { for (const token of tokens) { const tokenVisibleLength = visibleWidth(token); - const isWhitespace = token.trim() === ""; + const isWhitespace = isWhitespaceToken(token); // Token itself is too long - break it character by character if (tokenVisibleLength > width && !isWhitespace) { if (currentLine) { + // Trim before adding the reset, which would otherwise block the final trimEnd() + currentLine = currentLine.trimEnd(); // Add specific reset for underline only (preserves background) const lineEndReset = tracker.getLineEndReset(); if (lineEndReset) { @@ -972,6 +1027,12 @@ function wrapSingleLine(line: string, width: number): string[] { if (totalNeeded > width && currentVisibleLength > 0) { // Trim trailing whitespace, then add underline reset (not full reset, to preserve background) let lineToWrap = currentLine.trimEnd(); + if (isWhitespace) { + // The dropped space may carry codes (e.g. closing a styled word). Keep them on the wrapped line so + // the style ends there and is not re-opened on the next line. + lineToWrap += extractAnsiCodes(token); + updateTrackerFromText(token, tracker); + } const lineEndReset = tracker.getLineEndReset(); if (lineEndReset) { lineToWrap += lineEndReset; @@ -981,10 +1042,14 @@ function wrapSingleLine(line: string, width: number): string[] { // Don't start new line with whitespace currentLine = tracker.getActiveCodes(); currentVisibleLength = 0; - } else { - currentLine = tracker.getActiveCodes() + token; - currentVisibleLength = tokenVisibleLength; + continue; } + currentLine = tracker.getActiveCodes() + token; + currentVisibleLength = tokenVisibleLength; + } else if (isWhitespace && totalNeeded > width && token.includes("\x1b")) { + // Whitespace wider than an empty line. Plain spaces are removed by the final trimEnd(), but codes after + // them (e.g. a trailing reset) would block that and leave the line wider than width. Keep only the codes. + currentLine += extractAnsiCodes(token); } else { // Add to current line currentLine += token; diff --git a/packages/tui/test/editor.test.ts b/packages/tui/test/editor.test.ts index 552c654b4..5b2683d20 100644 --- a/packages/tui/test/editor.test.ts +++ b/packages/tui/test/editor.test.ts @@ -2132,6 +2132,36 @@ describe("Editor component", () => { }); describe("Autocomplete", () => { + it("moves the autocomplete selection with selection page keys (#7629)", async () => { + const editor = new Editor(createTestTUI(), defaultEditorTheme); + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion, + }); + + assert.strictEqual(editor.capturesViewportInput, false); + editor.handleInput("/"); + await flushAutocomplete(); + assert.strictEqual(editor.capturesViewportInput, true); + + editor.handleInput("\x1b[6~"); + editor.handleInput("\x1b[6~"); + editor.handleInput("\x1b[5~"); + editor.handleInput("\r"); + + assert.strictEqual(submitted, "/command-5"); + }); + it("triggers and debounces symbol completion after CJK punctuation", async (t) => { t.mock.timers.enable({ apis: ["setTimeout"] }); for (const before of [ diff --git a/packages/tui/test/keys.test.ts b/packages/tui/test/keys.test.ts index 0045610bc..4494508e1 100644 --- a/packages/tui/test/keys.test.ts +++ b/packages/tui/test/keys.test.ts @@ -7,6 +7,8 @@ import { describe, it } from "node:test"; import { decodeKittyPrintable, decodePrintableKey, + isKeyRelease, + isKeyRepeat, Key, matchesKey, parseKey, @@ -509,6 +511,26 @@ describe("matchesKey", () => { assert.strictEqual(matchesKey("\x1b[E", "clear"), true); }); + it("should match Kitty function keys F1-F12 with and without modifiers", () => { + assert.strictEqual(matchesKey("\x1b[P", "f1"), true); + assert.strictEqual(matchesKey("\x1b[Q", "f2"), true); + assert.strictEqual(matchesKey("\x1b[13~", "f3"), true); + assert.strictEqual(matchesKey("\x1b[S", "f4"), true); + assert.strictEqual(matchesKey("\x1b[1;5P", "ctrl+f1"), true); + assert.strictEqual(matchesKey("\x1b[1;5P", "f1"), false); + assert.strictEqual(matchesKey("\x1b[1;2S", "shift+f4"), true); + assert.strictEqual(matchesKey("\x1b[15;5~", "ctrl+f5"), true); + assert.strictEqual(matchesKey("\x1b[24;3~", "alt+f12"), true); + assert.strictEqual(matchesKey("\x1b[15;1:2~", "f5"), true); + assert.strictEqual(parseKey("\x1b[P"), "f1"); + assert.strictEqual(parseKey("\x1b[1;5P"), "ctrl+f1"); + assert.strictEqual(parseKey("\x1b[13~"), "f3"); + assert.strictEqual(parseKey("\x1b[21;2~"), "shift+f10"); + assert.strictEqual(parseKey("\x1b[1;5R"), undefined); + assert.strictEqual(isKeyRelease("\x1b[1;1:3P"), true); + assert.strictEqual(isKeyRepeat("\x1b[1;1:2S"), true); + }); + it("should match alt+arrows", () => { assert.strictEqual(matchesKey("\x1bp", "alt+up"), true); assert.strictEqual(matchesKey("\x1bp", "up"), false); @@ -525,6 +547,12 @@ describe("matchesKey", () => { }); describe("decodeKittyPrintable", () => { + it("ignores unmapped Kitty functional keys such as F13-F35", () => { + assert.strictEqual(decodeKittyPrintable("\x1b[57376u"), undefined); + assert.strictEqual(decodeKittyPrintable("\x1b[57398;2u"), undefined); + assert.strictEqual(decodePrintableKey("\x1b[57382u"), undefined); + }); + it("should decode Kitty keypad functional keys to printable characters", () => { assert.strictEqual(decodeKittyPrintable("\x1b[57399u"), "0"); assert.strictEqual(decodeKittyPrintable("\x1b[57400u"), "1"); diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index 38e6ceb5a..e0f5b4dd5 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -1,5 +1,7 @@ import assert from "node:assert"; import { afterEach, describe, it } from "node:test"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import type { Terminal as XtermTerminalType } from "@xterm/headless"; import { Chalk } from "chalk"; import { Markdown, type MarkdownTheme } from "../src/components/markdown.ts"; @@ -65,6 +67,63 @@ describe("Markdown component", () => { }); }); + describe("Block reuse", () => { + afterEach(() => resetCapabilitiesCache()); + + function countingTheme(calls: string[]): MarkdownTheme { + return { + ...defaultMarkdownTheme, + highlightCode: (code, lang) => { + calls.push(`${lang}:${code}`); + return code.split("\n").map((line) => chalk.green(line)); + }, + }; + } + + it("highlights a code block once across streamed text and resizes, and again after invalidate", () => { + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + markdown.setText("```ts\nx\n```\n\nafter"); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x"]); + markdown.invalidate(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("holds rendered blocks weakly, so a full garbage collection releases them", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + // A WeakRef keeps its target alive until the current job ends. + await new Promise((resolve) => setImmediate(resolve)); + gc(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("renders a block again when the block after it changes its spacing", () => { + const markdown = new Markdown("# Title", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("# Title\nfoo"); + assert.deepStrictEqual( + markdown.render(80).map((line) => stripAnsi(line).trim()), + ["Title", "", "foo"], + ); + }); + + it("follows changed reference link definitions", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: false }); + const markdown = new Markdown("[a][x]\n\n[x]: https://one.example", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("[a][x]\n\n[x]: https://two.example"); + assert.ok(stripAnsi(markdown.render(80).join("\n")).includes("https://two.example")); + }); + }); + describe("Lists", () => { it("should render simple nested list", () => { const markdown = new Markdown( @@ -966,6 +1025,29 @@ Inline \(A_{sf}\)`, ); }); + // Regression test for https://github.com/earendil-works/pi/issues/11483 + it("falls back to raw source when display math is wider than the content width", () => { + const source = [ + "\\[", + String.raw`\frac{\partial^2 u}{\partial x^2} + \frac{\partial^2 u}{\partial y^2} = \sum_{i=1}^{n} \frac{1}{i^2} + \int_0^1 f(x)\,dx`, + "\\]", + ].join("\n"); + const render = (width: number) => + new Markdown(source, 0, 0, defaultMarkdownTheme).render(width).map((line) => stripAnsi(line).trimEnd()); + + assert.deepStrictEqual(render(40), [ + "∂² u ∂² u n 1 1", + "──── + ──── = ∑ ── +∫ f(x) dx", + "∂ x² ∂ y² i=1 i² 0", + ]); + + const narrow = render(30); + assert.strictEqual(narrow[0], "\\["); + assert.strictEqual(narrow[narrow.length - 1], "\\]"); + assert.ok(!narrow.some((line) => line.includes("─"))); + assert.ok(narrow.every((line) => line.length <= 30)); + }); + // Regression tests for https://github.com/earendil-works/pi/issues/11159 describe("unclosed display math before tables (#11159)", () => { const table = ["| A | B |", "|---|---|", "| 1 | 2 |"]; @@ -1666,6 +1748,22 @@ bar`, }); }); + // #10925 + describe("styled words at the wrap boundary", () => { + it("should not start the next line with a space or carry styles into padding", () => { + const markdown = new Markdown("aaaa **bold** next ~~strk~~ more words", 1, 0, defaultMarkdownTheme, { + bgColor: (text) => `\x1b[44m${text}\x1b[49m`, + }); + + assert.deepStrictEqual(markdown.render(11), [ + "\x1b[44m aaaa \x1b[1mbold\x1b[22m \x1b[49m", + "\x1b[44m next \x1b[9mstrk\x1b[29m \x1b[49m", + "\x1b[44m more \x1b[49m", + "\x1b[44m words \x1b[49m", + ]); + }); + }); + // #10154 describe("CJK emphasis", () => { const tagTheme: MarkdownTheme = { @@ -1693,6 +1791,49 @@ bar`, resetCapabilitiesCache(); }); + it("should stop bare URLs at CJK punctuation and preserve the following text", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const punctuation of "。，；：！？、“”‘’（）【】《》「」『』…—·・〈〉［］｛｝．｡｢｣､･") { + for (const url of ["https://example.com", "https://example.com/中文?q=搜索#片段", "www.example.com"]) { + const source = `${url}${punctuation}后面的中文`; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + const href = url.startsWith("www.") ? `http://${url}` : url; + assert.ok(output.includes(`\x1b]8;;${href}\x1b\\`), source); + assert.ok(stripAnsi(output).includes(`\x1b]8;;\x1b\\${punctuation}后面的中文`), source); + assert.strictEqual(stripAnsi(output.replace(/\x1b\]8;;[^\x1b]*\x1b\\/g, "")).trimEnd(), source); + } + } + }); + + it("should preserve Chinese characters without punctuation in bare URLs", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const url of [ + "https://example.com紧接着中文", + "https://example.com/中文?q=搜索#片段", + "https://example.com/テストー한글", + ]) { + const output = new Markdown(url, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`)); + } + }); + + it("should preserve CJK punctuation in explicit links and angle autolinks", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const url = "https://example.com/中文，路径·テスト・한글…—。"; + for (const source of [`[文档](${url})`, `<${url}>`]) { + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`), source); + } + }); + + it("should preserve English punctuation trimming before Chinese punctuation", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const source = "https://example.com.，后面的中文"; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes("\x1b]8;;https://example.com\x1b\\")); + assert.ok(stripAnsi(output).includes("\x1b]8;;\x1b\\.，后面的中文")); + }); + it("should not duplicate URL for autolinked emails", () => { // Hyperlinks capability does not affect the mailto: display check. setCapabilities({ images: null, trueColor: false, hyperlinks: false }); diff --git a/packages/tui/test/select-list.test.ts b/packages/tui/test/select-list.test.ts index b5c24212c..b2ae5e564 100644 --- a/packages/tui/test/select-list.test.ts +++ b/packages/tui/test/select-list.test.ts @@ -113,4 +113,19 @@ describe("SelectList", () => { assert.ok(rendered[0].includes("…")); assert.equal(visibleIndexOf(rendered[0], "first"), visibleIndexOf(rendered[1], "second")); }); + + it("moves by one visible page with selection page keys (#7629)", () => { + const items = Array.from({ length: 12 }, (_, index) => ({ value: `item-${index}`, label: `Item ${index}` })); + const list = new SelectList(items, 5, testTheme); + const selections: string[] = []; + list.onSelectionChange = (item) => selections.push(item.value); + + for (const key of ["\x1b[6~", "\x1b[6~", "\x1b[6~", "\x1b[6~", "\x1b[5~"]) list.handleInput(key); + + assert.deepStrictEqual(selections, ["item-5", "item-10", "item-11", "item-6"]); + + list.setFilter("no match"); + list.handleInput("\x1b[6~"); + assert.deepStrictEqual(selections, ["item-5", "item-10", "item-11", "item-6"]); + }); }); diff --git a/packages/tui/test/settings-list.test.ts b/packages/tui/test/settings-list.test.ts index 07a7ff09f..08f94edc2 100644 --- a/packages/tui/test/settings-list.test.ts +++ b/packages/tui/test/settings-list.test.ts @@ -55,4 +55,26 @@ describe("SettingsList", () => { assert.deepStrictEqual(changes, [{ id: "tui-mode", value: "fullscreen" }]); }); + + it("moves by one visible page with selection page keys (#7629)", () => { + const list = new SettingsList( + Array.from({ length: 12 }, (_, index) => ({ + id: `setting-${index}`, + label: `Setting ${index}`, + currentValue: String(index), + })), + 5, + testTheme, + () => {}, + () => {}, + ); + + list.handleInput("\x1b[6~"); + assert.match(list.render(80).join("\n"), /> Setting 5/); + list.handleInput("\x1b[6~"); + list.handleInput("\x1b[6~"); + assert.match(list.render(80).join("\n"), /> Setting 11/); + list.handleInput("\x1b[5~"); + assert.match(list.render(80).join("\n"), /> Setting 6/); + }); }); diff --git a/packages/tui/test/terminal-image.test.ts b/packages/tui/test/terminal-image.test.ts index 6fd05f43d..0ae5bd8b4 100644 --- a/packages/tui/test/terminal-image.test.ts +++ b/packages/tui/test/terminal-image.test.ts @@ -34,6 +34,7 @@ import { visibleWidth } from "../src/utils.ts"; const ENV_KEYS = [ "TERM", "TERM_PROGRAM", + "ORCA_IMAGE_PROTOCOL", "TERMINAL_EMULATOR", "COLORTERM", "TMUX", @@ -343,6 +344,22 @@ describe("detectCapabilities", () => { }); }); + it("keeps images disabled for Orca without an image capability signal", () => { + withEnv({ TERM_PROGRAM: "Orca" }, () => { + const caps = detectCapabilities(); + assert.strictEqual(caps.images, null); + }); + }); + + it("enables images and hyperlinks for Orca advertising Kitty support", () => { + withEnv({ TERM_PROGRAM: "Orca", ORCA_IMAGE_PROTOCOL: "kitty" }, () => { + const caps = detectCapabilities(); + assert.strictEqual(caps.images, "kitty"); + assert.strictEqual(caps.trueColor, true); + assert.strictEqual(caps.hyperlinks, true); + }); + }); + it("enables hyperlinks for Ghostty", () => { withEnv({ TERM_PROGRAM: "ghostty" }, () => { const caps = detectCapabilities(); diff --git a/packages/tui/test/terminal.test.ts b/packages/tui/test/terminal.test.ts index 437994a38..8f8ce0e4f 100644 --- a/packages/tui/test/terminal.test.ts +++ b/packages/tui/test/terminal.test.ts @@ -36,6 +36,13 @@ describe("normalizeNativeShiftEnterInput", () => { assert.equal(normalizeNativeShiftEnterInput("\r", true, true), "\x1b[13;2u"); }); + it("rewrites ESC+CR to CSI-u Shift+Enter when Shift is pressed", () => { + // #11559 + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", true, true), "\x1b[13;2u"); + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", true, false), "\x1b\r"); + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", false, true), "\x1b\r"); + }); + it("leaves Return unchanged when native Shift detection is disabled", () => { assert.equal(normalizeNativeShiftEnterInput("\r", false, true), "\r"); }); @@ -395,6 +402,26 @@ describe("ProcessTerminal progress", () => { }); }); +describe("ProcessTerminal title", () => { + it("replaces control characters so the title cannot end the OSC sequence", () => { + const terminal = new ProcessTerminal(); + const writes: string[] = []; + const previousWrite = process.stdout.write; + + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + + try { + terminal.setTitle("pi - work\x07\x1b]52;c;Zm9v\x07\u009b2J - repo"); + assert.deepEqual(writes, ["\x1b]0;pi - work ]52;c;Zm9v 2J - repo\x07"]); + } finally { + process.stdout.write = previousWrite; + } + }); +}); + describe("ProcessTerminal dimensions", () => { it("falls back to COLUMNS and LINES before default dimensions", () => { const previousColumnsDescriptor = Object.getOwnPropertyDescriptor(process.stdout, "columns"); diff --git a/packages/tui/test/text.test.ts b/packages/tui/test/text.test.ts new file mode 100644 index 000000000..9c2e2bd61 --- /dev/null +++ b/packages/tui/test/text.test.ts @@ -0,0 +1,12 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Text } from "../src/components/text.ts"; + +describe("Text", () => { + it("keeps its wrapped lines when the same text is set again", () => { + const text = new Text("hello world", 0, 0); + const lines = text.render(5); + text.setText("hello world"); + assert.strictEqual(text.render(5), lines); + }); +}); diff --git a/packages/tui/test/tui-alt-screen.test.ts b/packages/tui/test/tui-alt-screen.test.ts index 41402d5c7..6ea551419 100644 --- a/packages/tui/test/tui-alt-screen.test.ts +++ b/packages/tui/test/tui-alt-screen.test.ts @@ -6,6 +6,7 @@ import { findAltScreenSearchMatches, } from "../src/alt-screen-search.ts"; import { Box } from "../src/components/box.ts"; +import { Editor } from "../src/components/editor.ts"; import { HStack } from "../src/components/h-stack.ts"; import { Image } from "../src/components/image.ts"; import { MouseRegion } from "../src/components/mouse-region.ts"; @@ -24,6 +25,7 @@ import { import { renderFakeCursor, type TuiMouseEvent } from "../src/tui.ts"; import { TuiAltScreen } from "../src/tui-alt-screen.ts"; import { stripTerminalSequences, visibleWidth } from "../src/utils.ts"; +import { defaultEditorTheme } from "./test-themes.ts"; import { VirtualTerminal } from "./virtual-terminal.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; @@ -2147,4 +2149,81 @@ describe("TuiAltScreen", () => { assert.ok(terminal.getViewport().some((line) => line.includes("↑ ↓"))); tui.stop(); }); + + it("defers viewport keys to a focused list instead of scrolling the transcript (#7629)", async () => { + const terminal = new VirtualTerminal(20, 6); + const tui = new TuiAltScreen(terminal); + const list = new SelectList( + Array.from({ length: 12 }, (_, index) => ({ value: `item-${index}`, label: `item-${index}` })), + 3, + { + selectedPrefix: (text) => text, + selectedText: (text) => text, + description: (text) => text, + scrollInfo: (text) => text, + noMatch: (text) => text, + }, + ); + tui.addChild(new Text(Array.from({ length: 12 }, (_, index) => `line ${index + 1}`).join("\n"), 0, 0)); + tui.addChild(list); + tui.setFocus(list); + tui.start(); + await terminal.waitForRender(); + const topBefore = tui.viewportTop; + + terminal.sendInput("\x1b[6~"); // PageDown + terminal.sendInput("\x1b[1;5H"); // Ctrl+Home + await terminal.waitForRender(); + assert.strictEqual(list.getSelectedItem()?.value, "item-3"); + assert.strictEqual(tui.viewportTop, topBefore); + tui.stop(); + }); + + it("gives viewport keys back to the transcript when editor autocomplete closes (#7629)", async () => { + const terminal = new VirtualTerminal(40, 8); + const tui = new TuiAltScreen(terminal); + const editor = new Editor(tui, defaultEditorTheme); + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion: (lines, cursorLine, _cursorCol, item) => ({ + lines: lines.map((line, index) => (index === cursorLine ? item.value : line)), + cursorLine, + cursorCol: item.value.length, + }), + }); + tui.addChild(new Text(Array.from({ length: 30 }, (_, index) => `line ${index + 1}`).join("\n"), 0, 0)); + tui.addChild(editor); + tui.setFocus(editor); + tui.start(); + await terminal.waitForRender(); + + terminal.sendInput("/"); + await new Promise((resolve) => setImmediate(resolve)); + await terminal.waitForRender(); + assert.strictEqual(editor.isShowingAutocomplete(), true); + const topWithMenu = tui.viewportTop; + terminal.sendInput("\x1b[6~"); // PageDown pages the menu + await terminal.waitForRender(); + assert.strictEqual(tui.viewportTop, topWithMenu); + + terminal.sendInput("\r"); + await terminal.waitForRender(); + assert.strictEqual(submitted, "/command-5"); + assert.strictEqual(editor.isShowingAutocomplete(), false); + const topWithoutMenu = tui.viewportTop; + terminal.sendInput("\x1b[5~"); // PageUp scrolls the transcript again + await terminal.waitForRender(); + assert.ok(tui.viewportTop < topWithoutMenu); + tui.stop(); + }); }); diff --git a/packages/tui/test/tui-render.test.ts b/packages/tui/test/tui-render.test.ts index 3749eac76..c01142a4a 100644 --- a/packages/tui/test/tui-render.test.ts +++ b/packages/tui/test/tui-render.test.ts @@ -209,6 +209,24 @@ describe("TUI bounded render output", () => { }); }); +describe("TUI OSC 133 markers", () => { + it("does not write OSC 133 prompt markers to the terminal", () => { + // Terminals read 133;A as a new shell prompt and drop the OSC 7501 program status. + const terminal = new BoundedWriteTerminal(); + const tui = new TuiMainScreen(terminal); + const component = new TestComponent(); + tui.addChild(component); + component.lines = ["\x1b]133;A\x07prompt", "\x1b]133;B\x07\x1b]133;C\x07output"]; + tui.renderNow(); + component.lines = [...component.lines, "\x1b]133;A\x1b\\reply", "\x1b]133;B\x1b\\\x1b]133;C\x1b\\done"]; + tui.renderNow(); + + const output = terminal.writes.join(""); + assert.ok(!output.includes("\x1b]133;"), "regular mode must not emit OSC 133 markers"); + for (const text of ["prompt", "output", "reply", "done"]) assert.ok(output.includes(text)); + }); +}); + /** Set each environment variable to `value`, returning a function that restores the previous state. */ function overrideEnv(names: readonly string[], value: string): () => void { const previousValues = names.map((name) => [name, process.env[name]] as const); diff --git a/packages/tui/test/wrap-ansi.test.ts b/packages/tui/test/wrap-ansi.test.ts index 618be2347..60cfc7248 100644 --- a/packages/tui/test/wrap-ansi.test.ts +++ b/packages/tui/test/wrap-ansi.test.ts @@ -51,6 +51,57 @@ describe("wrapTextWithAnsi", () => { }); }); + // #10925 + describe("styled words at the wrap boundary", () => { + it("keeps the closing code on the wrapped line and drops the following space", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[1mbold\x1b[22m next word", 9), [ + "aaaa \x1b[1mbold\x1b[22m", + "next word", + ]); + }); + + it("does not re-open a style closed by the dropped space", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[31mred\x1b[39m next", 8), [ + "aaaa \x1b[31mred\x1b[39m", + "next", + ]); + }); + + it("does not exceed width for styled whitespace wider than the line", () => { + for (const text of [ + `\x1b[41m${" ".repeat(20)}\x1b[0m`, + `ab\x1b[41m${" ".repeat(20)}\x1b[49mcd`, + `\x1b[33m${" ".repeat(20)}x\x1b[39m`, + ]) { + for (const line of wrapTextWithAnsi(text, 10)) { + assert.ok(visibleWidth(line) <= 10, JSON.stringify(line)); + } + } + }); + + it("trims trailing spaces before the line-end reset when a long word follows", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaa\x1b[9m abcdefghij\x1b[29m", 9), [ + "aaa\x1b[9m\x1b[29m", + "\x1b[9mabcdefghi\x1b[29m", + "\x1b[9mj\x1b[29m", + ]); + }); + + it("resets strikethrough at line end and restores it on the next line", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[9mstrike me\x1b[29m next", 12), [ + "aaaa \x1b[9mstrike\x1b[29m", + "\x1b[9mme\x1b[29m next", + ]); + }); + + it("preserves background when resetting strikethrough at line end", () => { + assert.deepStrictEqual(wrapTextWithAnsi("\x1b[41;9mstrike me\x1b[0m", 6), [ + "\x1b[41;9mstrike\x1b[29m", + "\x1b[9;41mme\x1b[0m", + ]); + }); + }); + describe("background color preservation", () => { it("should preserve background color across wrapped lines without full reset", () => { const bgBlue = "\x1b[44m"; diff --git a/pi-test.sh b/pi-test.sh index 95383308e..c77d7287d 100755 --- a/pi-test.sh +++ b/pi-test.sh @@ -51,6 +51,32 @@ if [[ "$NO_ENV" == "true" ]]; then unset AZURE_OPENAI_API_KEY unset AZURE_OPENAI_BASE_URL unset AZURE_OPENAI_RESOURCE_NAME + unset ANTHROPIC_AUTH_TOKEN + unset ANTHROPIC_FEDERATION_RULE_ID + unset ANTHROPIC_IDENTITY_TOKEN_FILE + unset ANTHROPIC_ORGANIZATION_ID + unset ANTHROPIC_SERVICE_ACCOUNT_ID + unset ANTHROPIC_WORKSPACE_ID + unset ANT_LING_API_KEY + unset BASETEN_API_KEY + unset CLOUDFLARE_API_KEY + unset DEEPSEEK_API_KEY + unset FIREWORKS_API_KEY + unset GOOGLE_CLOUD_API_KEY + unset KIMI_API_KEY + unset META_API_KEY + unset MOONSHOT_API_KEY + unset NVIDIA_API_KEY + unset QWEN_TOKEN_PLAN_API_KEY + unset QWEN_TOKEN_PLAN_CN_API_KEY + unset RADIUS_API_KEY + unset TOGETHER_API_KEY + unset TYPESAFE_API_KEY + unset XIAOMI_API_KEY + unset XIAOMI_TOKEN_PLAN_CN_API_KEY + unset XIAOMI_TOKEN_PLAN_AMS_API_KEY + unset XIAOMI_TOKEN_PLAN_SGP_API_KEY + unset ZAI_CODING_CN_API_KEY echo "Running without API keys..." fi diff --git a/scripts/browser-smoke-entry.ts b/scripts/browser-smoke-entry.ts index 4c6d07c38..9cf69450d 100644 --- a/scripts/browser-smoke-entry.ts +++ b/scripts/browser-smoke-entry.ts @@ -1,12 +1,17 @@ import { createHost, memoryLinkPair, replicatedState } from "@earendil-works/chord"; -import { createAssistantMessageEventStream, Type } from "@earendil-works/pi-ai"; +import { createAssistantMessageEventStream, validateToolArguments } from "@earendil-works/pi-ai"; import { complete, getModel, getProviders, streamSimple } from "@earendil-works/pi-ai/compat"; import { Agent, streamProxy } from "@earendil-works/pi-agent-core"; +import { obj, str } from "@earendil-works/pi-schema"; // Keep this entry browser-safe. It is bundled by scripts/check-browser-smoke.mjs // to catch accidental Node-only runtime imports in browser-facing package exports. const model = getModel("google", "gemini-2.5-flash"); -const schema = Type.Object({ prompt: Type.String() }); +const schema = obj({ prompt: str() }); +const args = validateToolArguments( + { name: "prompt", description: "", parameters: schema }, + { type: "toolCall", id: "1", name: "prompt", arguments: { prompt: 1 } }, +); const stream = createAssistantMessageEventStream(); const agent = new Agent({ initialState: { model }, streamFn: streamSimple }); @@ -17,6 +22,7 @@ console.log( getProviders().length, typeof complete, schema.type, + args.prompt, typeof stream.push, agent.hasQueuedMessages(), typeof streamProxy, diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index 48cc9d3b3..4aea32757 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -12,7 +12,7 @@ */ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { dirname, relative, resolve } from "node:path"; +import { dirname, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -26,6 +26,7 @@ const WORKSPACE = { "@earendil-works/pi-codemode": "packages/codemode/src", "@earendil-works/pi-telemetry": "packages/telemetry/src", "@earendil-works/pi-mcp": "packages/mcp/src", + "@earendil-works/pi-schema": "packages/schema/src", "@earendil-works/pi-tui": "packages/tui/src", }; @@ -34,15 +35,27 @@ const WORKSPACE = { * graph it is allowed to reach. */ const BUDGETS = { + "packages/schema": { + // The format checks (vendored from TypeBox, about 40 KB) load only through ./formats. + ".": { maxFiles: 10, forbid: ["schema/src/formats/"] }, + }, "packages/ai": { "./models": { // Request telemetry reaches pi-ai's telemetry.ts and pi-telemetry's root (index, noop, metrics, memory). // Request identity reaches utils/identity.ts and its version constant. maxFiles: 20, - forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/typebox-helpers.ts"], + forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/string-enum.ts"], allow: ["packages/telemetry/src/index.ts"], }, - "./utils/*": { maxFiles: 3, forbid: ["providers/", "api/", "index.ts"] }, + "./utils/*": { + maxFiles: 3, + forbid: ["providers/", "api/", "index.ts"], + overrides: { + // Tool argument validation runs on pi-schema: its validator, coercion and format checks (the checks TypeBox's + // validator ran by default). TypeBox, an external package, was not counted before. + "./utils/validation": { maxFiles: 42, forbid: ["providers/", "api/", "packages/ai/src/index.ts"] }, + }, + }, }, "packages/codemode": { // Node and Bun users never load the Cloudflare entry, which imports the `cloudflare:workers` built-in. @@ -60,24 +73,30 @@ const BUDGETS = { }, "packages/durable": { ".": { - // Tool argument validation reaches TypeBox; provider-session creation reaches pi-ai's lean UUID utility; the + // Tool argument validation reaches pi-schema (validator, coercion and format checks; 39 files that replace + // TypeBox, which this walk did not count); provider-session creation reaches pi-ai's lean UUID utility; the // memory and SQLite storages share scan order and cursor handling. Values from Chord's root (copyJson, // replicatedState) reach its host runtime through the root barrel. Chord's context reaches pi-telemetry's root // (index, noop, metrics, memory) for its NOOP telemetry default; harness/telemetry.ts is Durable's schema. // The Harness's request identity reaches pi-ai's utils/identity.ts and its version constant. Partial commits // parse streamed tool-call arguments with pi-ai's utils/streaming-json.ts (and its json-parse.ts) to append what // argument strings grew by (harness/partial-changes.ts). Chord's watches fold pending ops with Delta's - // pending queue (delta/pending.ts). The storages share the log (storage/log.ts). - maxFiles: 78, + // pending queue (delta/pending.ts). `registryReader` (harness/registry-reader.ts) reads a Chord v2 registry + // value through type-only imports. The Session caches hot Storage reads (session/cached-storage.ts) and keeps + // each conversation's newest entry and head marker (session/conversation-tails.ts). Group commit sends batches + // through `commit()`, or one `commit()` each (storage/commit-batch.ts), from a writer off the Session line + // (session/writer.ts) that gates reads by the keys of unpublished writes (session/keys.ts). + maxFiles: 123, forbid: [ "packages/ai/src/index.ts", - "packages/ai/src/utils/typebox-helpers.ts", + "packages/ai/src/utils/string-enum.ts", // Photon (WebAssembly) loads only through `./images`. "packages/durable/src/images/", ], }, - // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself. - "./tools": { maxFiles: 17, forbid: ["packages/durable/src/images/"] }, + // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself, + // and pi-schema's root for their schema builders (10 files; its format checks stay out). + "./tools": { maxFiles: 27, forbid: ["packages/durable/src/images/", "packages/schema/src/formats/"] }, }, }; @@ -147,22 +166,24 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { } const target = typeof declared === "string" ? declared : declared.import; for (const [name, distPath] of expand(pkgDir, entry, target)) { + // A wildcard entry's budget applies to every file it exports, unless `overrides` names that file. + const limits = budget.overrides?.[name] ?? budget; const source = sourceFor(pkgDir, distPath); if (!source) { console.error(`${pkgDir} export "${name}" points at ${distPath}, which has no source file`); failures += 1; continue; } - const graph = [...walk(source)].map((file) => relative(ROOT, file)); - if (graph.length > budget.maxFiles) { + const graph = [...walk(source)].map((file) => relative(ROOT, file).split(sep).join("/")); + if (graph.length > limits.maxFiles) { console.error( - `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${budget.maxFiles}\n` + + `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${limits.maxFiles}\n` + graph.map((file) => ` ${file}`).join("\n"), ); failures += 1; } - for (const pattern of budget.forbid ?? []) { - const hit = graph.filter((file) => file.includes(pattern) && !(budget.allow ?? []).some((allowed) => file.endsWith(allowed))); + for (const pattern of limits.forbid ?? []) { + const hit = graph.filter((file) => file.includes(pattern) && !(limits.allow ?? []).some((allowed) => file.endsWith(allowed))); if (hit.length > 0) { console.error(`${pkgDir} export "${name}" must not reach ${pattern}:\n${hit.map((f) => ` ${f}`).join("\n")}`); failures += 1; diff --git a/scripts/diff-model-catalog.mjs b/scripts/diff-model-catalog.mjs index e00563f76..1d2f10a4e 100644 --- a/scripts/diff-model-catalog.mjs +++ b/scripts/diff-model-catalog.mjs @@ -63,7 +63,11 @@ const currentThinkingOutput = join(temporaryRoot, "after-thinking"); let worktreeAdded = false; function generateCatalog(cwd, outputDir, pretty = false) { - const args = ["packages/ai/scripts/generate-models.ts", "--strict", "--json-only", "--json-output", outputDir]; + // Baselines from before the catalog moved still have the old generator. + const generator = existsSync(join(cwd, "packages/ai/catalog/generate.ts")) + ? "packages/ai/catalog/generate.ts" + : "packages/ai/scripts/generate-models.ts"; + const args = [generator, "--strict", "--json-only", "--json-output", outputDir]; if (pretty) args.push("--pretty"); run(process.execPath, args, { cwd, capture: true }); } diff --git a/scripts/issues/categories/models.md b/scripts/issues/categories/models.md index cc9d32953..774f24e99 100644 --- a/scripts/issues/categories/models.md +++ b/scripts/issues/categories/models.md @@ -6,7 +6,7 @@ owner: davidbrai Which models exist and which one is used. In: -- Built-in model catalog (generate-models, models.dev, pi.dev registry). +- Built-in model catalog (packages/ai/catalog, models.dev, pi.dev registry). - Model metadata: context window, max tokens, thinking level map, pricing. - models.json custom providers and models. - Model resolution: --model, --provider, defaultModel, startup fallback, scoped models, virtual models. diff --git a/scripts/model-catalog-protocol.test.ts b/scripts/model-catalog-protocol.test.ts index cdfcc0d50..0a7e3af77 100644 --- a/scripts/model-catalog-protocol.test.ts +++ b/scripts/model-catalog-protocol.test.ts @@ -55,25 +55,23 @@ test("selects the newest catalog a Pi version supports", () => { assert.equal(selectModelCatalog(index, undefined)?.revision, mixedApiRevision); }); -test("redirects Pi user agents to an explicit catalog version", () => { +test("serves Pi user agents the catalog for their version", () => { // Regression test for #9099: released clients identify themselves only by User-Agent. assert.deepEqual( parseModelCatalogRequest( "https://pi.dev/api/models/providers/openrouter?types=chat%2Cimage", "pi/0.84.4 (linux; node/v22.0.0; x64)", ), - { - kind: "redirect", - location: "https://pi.dev/api/models/providers/openrouter?types=chat%2Cimage&pi-version=0.84.4", - }, + { kind: "catalog", piVersion: "0.84.4", representation: "typed" }, ); assert.deepEqual(parseModelCatalogRequest("https://pi.dev/api/models", "pi/0.84.4"), { - kind: "redirect", - location: "https://pi.dev/api/models?pi-version=0.84.4", + kind: "catalog", + piVersion: "0.84.4", + representation: "legacy", }); }); -test("serves explicit and unversioned catalog requests without redirecting", () => { +test("serves explicit and unversioned catalog requests", () => { assert.deepEqual( parseModelCatalogRequest("https://pi.dev/api/models?pi-version=0.83.0", "pi/0.85.1 (linux; node/v22.0.0; x64)"), { kind: "catalog", piVersion: "0.83.0", representation: "legacy" }, diff --git a/scripts/model-catalog-protocol.ts b/scripts/model-catalog-protocol.ts index d151d36a5..173c62b19 100644 --- a/scripts/model-catalog-protocol.ts +++ b/scripts/model-catalog-protocol.ts @@ -53,7 +53,6 @@ export type ModelCatalogIndex = { export type ModelCatalogRequest = | { kind: "catalog"; piVersion: string | undefined; representation: ModelCatalogRepresentation } - | { kind: "redirect"; location: string } | { kind: "invalid"; error: string }; export function getModelCatalogArtifactKey(revision: string, artifact: ModelCatalogArtifact): string { @@ -97,10 +96,11 @@ export function parseModelCatalogRepresentation( /** * Decide how to answer a catalog request from its URL and User-Agent. * - * Released clients do not send `pi-version`. Responses are cached by URL, so - * instead of varying the response on the User-Agent, clients that identify - * as Pi are redirected to the equivalent URL with an explicit `pi-version`. - * Requests without a Pi User-Agent receive the default catalog. + * Released clients do not send `pi-version`, so clients that identify as Pi + * are served the catalog for the version in their User-Agent. The response + * therefore varies on the User-Agent, and the server must say so with `Vary`. + * An explicit `pi-version` takes precedence. Requests without a Pi User-Agent + * receive the default catalog. */ export function parseModelCatalogRequest(url: string | URL, userAgent: string | null | undefined): ModelCatalogRequest { const requestUrl = new URL(url); @@ -113,8 +113,7 @@ export function parseModelCatalogRequest(url: string | URL, userAgent: string | if (piVersion === null) { const userAgentVersion = PI_USER_AGENT_RE.exec(userAgent ?? "")?.[1]; if (userAgentVersion !== undefined && isValidModelCatalogPiVersion(userAgentVersion)) { - requestUrl.searchParams.set("pi-version", userAgentVersion); - return { kind: "redirect", location: requestUrl.toString() }; + return { kind: "catalog", piVersion: userAgentVersion, representation }; } return { kind: "catalog", piVersion: undefined, representation }; } diff --git a/scripts/npm-audit.mjs b/scripts/npm-audit.mjs index f636d2347..f2d391c90 100644 --- a/scripts/npm-audit.mjs +++ b/scripts/npm-audit.mjs @@ -10,15 +10,7 @@ const auditLevel = "moderate"; const severities = ["info", "low", "moderate", "high", "critical"]; /** @type {Record} */ -const acceptedAdvisories = { - "GHSA-86w9-cpqp-85rv": { - package: "node-forge", - reason: - "No fixed node-forge release exists. It is only reachable through @earendil-works/gondolin in the private, " + - "unpublished gondolin example extension. Gondolin only verifies leaf certificates against its own locally " + - "generated CA (public exponent 65537), so the low-exponent signature forgery does not apply.", - }, -}; +const acceptedAdvisories = {}; const result = spawnSync("npm", ["audit", "--omit=dev", "--json"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (result.error) { diff --git a/scripts/package-files.mjs b/scripts/package-files.mjs new file mode 100644 index 000000000..6e64aa95f --- /dev/null +++ b/scripts/package-files.mjs @@ -0,0 +1,30 @@ +// Cross-platform file operations for package build scripts, using only Node built-ins. +import { chmodSync, cpSync, globSync, mkdirSync, rmSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +const [command, ...paths] = process.argv.slice(2); +const usage = "Usage: package-files.mjs clean | copy | executable "; + +if (paths.length === 0) throw new Error(usage); + +switch (command) { + case "clean": + for (const path of paths) rmSync(path, { recursive: true, force: true }); + break; + case "copy": { + const destination = paths.pop(); + if (!destination || paths.length === 0) throw new Error(usage); + mkdirSync(destination, { recursive: true }); + for (const pattern of paths) { + const sources = globSync(pattern); + if (sources.length === 0) throw new Error(`No files match ${pattern}`); + for (const source of sources) cpSync(source, join(destination, basename(source)), { recursive: true }); + } + break; + } + case "executable": + for (const path of paths) chmodSync(path, statSync(path).mode | 0o111); + break; + default: + throw new Error(usage); +} diff --git a/scripts/package-files.test.mjs b/scripts/package-files.test.mjs new file mode 100644 index 000000000..9b455a792 --- /dev/null +++ b/scripts/package-files.test.mjs @@ -0,0 +1,82 @@ +// Exercises the file operations used by package build scripts without touching build output. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("./package-files.mjs", import.meta.url)); + +function workspace(t) { + const directory = mkdtempSync(join(tmpdir(), "pi-package-files-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function run(directory, ...args) { + return spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: "utf8" }); +} + +test("clean removes directories and files and ignores missing paths", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "dist", "nested"), { recursive: true }); + writeFileSync(join(directory, "dist", "nested", "file.js"), "output"); + writeFileSync(join(directory, "output.txt"), "output"); + assert.equal(run(directory, "clean", "dist", "output.txt", "missing").status, 0); + assert.equal(existsSync(join(directory, "dist")), false); + assert.equal(existsSync(join(directory, "output.txt")), false); +}); + +test("copy expands patterns, creates directories, and overwrites existing assets", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "source assets")); + writeFileSync(join(directory, "source assets", "dark.json"), "dark"); + writeFileSync(join(directory, "source assets", "light.json"), "light"); + writeFileSync(join(directory, "source assets", "ignored.txt"), "ignored"); + const args = ["copy", "source assets/*.json", "output/assets"]; + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "dark"); + assert.equal(readFileSync(join(directory, "output/assets/light.json"), "utf8"), "light"); + assert.equal(existsSync(join(directory, "output/assets/ignored.txt")), false); + writeFileSync(join(directory, "source assets", "dark.json"), "updated"); + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "updated"); +}); + +test("copy accepts multiple files and preserves directory names and nested contents", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "docs", "nested"), { recursive: true }); + writeFileSync(join(directory, "docs", "nested", "guide.md"), "guide"); + writeFileSync(join(directory, "README.md"), "readme"); + assert.equal(run(directory, "copy", "docs", "README.md", "dist").status, 0); + assert.equal(readFileSync(join(directory, "dist", "docs", "nested", "guide.md"), "utf8"), "guide"); + assert.equal(readFileSync(join(directory, "dist", "README.md"), "utf8"), "readme"); +}); + +test("copy fails when an asset pattern matches nothing", (t) => { + const result = run(workspace(t), "copy", "missing/*.json", "dist"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /No files match missing/); +}); + +test("executable adds executable bits without removing existing permissions", (t) => { + const directory = workspace(t); + for (const name of ["cli.js", "rpc.js"]) writeFileSync(join(directory, name), "entry", { mode: 0o640 }); + const mode = statSync(join(directory, "cli.js")).mode; + assert.equal(run(directory, "executable", "cli.js", "rpc.js").status, 0); + if (process.platform !== "win32") { + assert.equal(statSync(join(directory, "cli.js")).mode & 0o777, (mode | 0o111) & 0o777); + assert.equal(statSync(join(directory, "rpc.js")).mode & 0o111, 0o111); + } +}); + +test("invalid commands and missing arguments fail", (t) => { + const directory = workspace(t); + for (const args of [[], ["unknown", "dist"], ["clean"], ["copy", "dist"], ["executable"]]) { + const result = run(directory, ...args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Usage: package-files/); + } +}); diff --git a/tsconfig.json b/tsconfig.json index b530a76e4..a6a826276 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,12 @@ "@earendil-works/chord": ["./packages/chord/src/index.ts"], "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], + "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], + "@earendil-works/chord/next/plugins": ["./packages/chord/src/next-node/plugins/index.ts"], + "@earendil-works/chord/next/plugins/runtime": ["./packages/chord/src/next-node/plugins/runtime.ts"], + "@acme/app/*": ["./packages/chord/test/next/examples/49-plugin-api/app/*.ts"], + "@earendil-works/chord/next/http": ["./packages/chord/src/next-node/http/index.ts"], + "@earendil-works/chord/next/workers": ["./packages/chord/src/next-node/workers/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"], @@ -23,6 +29,8 @@ "@earendil-works/pi-codemode/portable": ["./packages/codemode/src/portable.ts"], "@earendil-works/pi-codemode/source": ["./packages/codemode/src/source.ts"], "@earendil-works/pi-codemode/worker": ["./packages/codemode/src/runtime/worker.ts"], + "@earendil-works/pi-schema": ["./packages/schema/src/index.ts"], + "@earendil-works/pi-schema/formats": ["./packages/schema/src/formats/index.ts"], "@earendil-works/pi-mcp": ["./packages/mcp/src/index.ts"], "@earendil-works/pi-mcp/oauth": ["./packages/mcp/src/oauth/index.ts"], "@earendil-works/pi-mcp/testing": ["./packages/mcp/src/testing/index.ts"], @@ -49,6 +57,6 @@ "@earendil-works/pi-agent-old/*": ["./packages/agent-old/src/*"] } }, - "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], - "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/gondolin/**"] + "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/ai/catalog/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], + "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/**", "packages/coding-agent/test/suite/regressions/8261-subagent-project-trust.test.ts"] }